From 5d4c1be28537626c2f3132bc62d0d6c77c8efd53 Mon Sep 17 00:00:00 2001 From: Vertex MG Team Date: Tue, 9 Sep 2025 00:15:42 -0700 Subject: [PATCH] Migrate llava notebook to use Model Garden SDK PiperOrigin-RevId: 804765256 --- .../model_garden_pytorch_llava.ipynb | 521 +++++++++--------- 1 file changed, 259 insertions(+), 262 deletions(-) diff --git a/notebooks/community/model_garden/model_garden_pytorch_llava.ipynb b/notebooks/community/model_garden/model_garden_pytorch_llava.ipynb index 09fd7cd04..11189036d 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_llava.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_llava.ipynb @@ -54,40 +54,48 @@ { "cell_type": "markdown", "metadata": { - "id": "cbDI9ag4oR4C" + "id": "3de7470326a2" }, "source": [ "## Overview\n", "\n", - "This notebook demonstrates deploying prebuilt [LLaVA models](https://huggingface.co/LLaVA-hf) with [vLLM](https://github.com/vllm-project/vllm) to improve serving throughput.\n", + "This notebook demonstrates how to deploy a **Pytorch-Llava** open model on Google Cloud Vertex AI.\n", "\n", + "### Objectives\n", "\n", - "### Objective\n", + "- Deploy Pytorch-Llava using containerized backends like [vLLM](https://github.com/vllm-project/vllm) on GPU.\n", + "- Use the deployed model to serve chat completion requests for both text and multimodal inputs.\n", "\n", - "- Download and deploy prebuilt LLaVA models\n", - "- Deploy LLaVA with [vLLM](https://github.com/vllm-project/vllm) to improve serving throughput\n", + "### File a Bug\n", "\n", - "### File a bug\n", - "\n", - "File a bug on [GitHub](https://github.com/GoogleCloudPlatform/vertex-ai-samples/issues/new) if you encounter any issue with the notebook.\n", + "If you encounter issues with this notebook, report them on [GitHub](https://github.com/GoogleCloudPlatform/vertex-ai-samples/issues/new).\n", "\n", "### Costs\n", "\n", "This tutorial uses billable components of Google Cloud:\n", "\n", - "* Vertex AI\n", - "* Cloud Storage\n", + "- Vertex AI\n", + "- Cloud Storage\n", "\n", - "Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing), [Cloud Storage pricing](https://cloud.google.com/storage/pricing), and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage." + "Refer to the [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage pricing](https://cloud.google.com/storage/pricing) pages for more information. Use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to estimate your projected costs." ] }, { "cell_type": "markdown", "metadata": { - "id": "hQJWRopioSKT" + "id": "jeYw-Czg-DFy" }, "source": [ - "## Before you begin" + "## Get Started" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "KgyhGvEzBDkj" + }, + "source": [ + "### Install Vertex AI SDK and other required packages" ] }, { @@ -95,84 +103,85 @@ "execution_count": null, "metadata": { "cellView": "form", - "id": "J_jmxcIZoSxU" + "id": "iCacdLqG-IsH" }, "outputs": [], "source": [ - "# @title Setup Google Cloud project\n", + "%pip install --upgrade --force-reinstall --quiet 'google-cloud-aiplatform>=1.106.0' 'openai' 'google-auth==2.27.0' 'requests==2.32.3'" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "HUKCrpBy-3yf" + }, + "source": [ + "### Authenticate the Notebook Environment (Colab only)\n", "\n", - "# @markdown 1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n", + "If you're running this notebook in Google Colab, run the following cell to authenticate." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form", + "id": "JXwCT1kn-3Gu" + }, + "outputs": [], + "source": [ + "import sys\n", "\n", - "# @markdown 2. **[Optional]** Set region. If not set, the region will be set automatically according to Colab Enterprise environment.\n", + "if \"google.colab\" in sys.modules:\n", + " from google.colab import auth\n", "\n", - "REGION = \"\" # @param {type:\"string\"}\n", + " auth.authenticate_user()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "AcW2nwB8-7yC" + }, + "source": [ + "### Set Google Cloud Project Information\n", "\n", - "# @markdown 3. If you want to run predictions with A100 80GB or H100 GPUs, we recommend using the regions listed below. **NOTE:** Make sure you have associated quota in selected regions. Click the links to see your current quota for each GPU type: [Nvidia A100 80GB](https://console.cloud.google.com/iam-admin/quotas?metric=aiplatform.googleapis.com%2Fcustom_model_serving_nvidia_a100_80gb_gpus), [Nvidia H100 80GB](https://console.cloud.google.com/iam-admin/quotas?metric=aiplatform.googleapis.com%2Fcustom_model_serving_nvidia_h100_gpus). You can request for quota following the instructions at [\"Request a higher quota\"](https://cloud.google.com/docs/quota/view-manage#requesting_higher_quota).\n", + "To get started with Vertex AI, ensure you have an existing Google Cloud project and that the [Vertex AI API is enabled](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com).\n", "\n", - "# @markdown > | Machine Type | Accelerator Type | Recommended Regions |\n", - "# @markdown | ----------- | ----------- | ----------- |\n", - "# @markdown | a2-ultragpu-1g | 1 NVIDIA_A100_80GB | us-central1, us-east4, europe-west4, asia-southeast1, us-east4 |\n", - "# @markdown | a3-highgpu-2g | 2 NVIDIA_H100_80GB | us-west1, asia-southeast1, europe-west4 |\n", - "# @markdown | a3-highgpu-4g | 4 NVIDIA_H100_80GB | us-west1, asia-southeast1, europe-west4 |\n", - "# @markdown | a3-highgpu-8g | 8 NVIDIA_H100_80GB | us-central1, europe-west4, us-west1, asia-southeast1 |\n", - "\n", - "# Upgrade Vertex AI SDK.\n", - "! pip3 install --upgrade --quiet 'google-cloud-aiplatform==1.103.0'\n", - "! git clone https://github.com/GoogleCloudPlatform/vertex-ai-samples.git\n", - "\n", - "import importlib\n", + "See the guide on [setting up your project and development environment](https://cloud.google.com/vertex-ai/docs/start/cloud-environment). Also confirm that [billing is enabled](https://cloud.google.com/billing/docs/how-to/modify-project).\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form", + "id": "eIVLp0oE--k-" + }, + "outputs": [], + "source": [ + "# Use the environment variable if the user doesn't provide Project ID.\n", "import os\n", - "from typing import Tuple\n", "\n", - "from google.cloud import aiplatform\n", - "\n", - "if os.environ.get(\"VERTEX_PRODUCT\") != \"COLAB_ENTERPRISE\":\n", - " ! pip install --upgrade tensorflow\n", - "! git clone https://github.com/GoogleCloudPlatform/vertex-ai-samples.git\n", - "\n", - "common_util = importlib.import_module(\n", - " \"vertex-ai-samples.notebooks.community.model_garden.docker_source_codes.notebook_util.common_util\"\n", - ")\n", - "\n", - "\n", - "LABEL = \"vllm_gpu\"\n", - "models, endpoints = {}, {}\n", - "\n", - "# Get the default cloud project id.\n", - "PROJECT_ID = os.environ[\"GOOGLE_CLOUD_PROJECT\"]\n", - "\n", - "# Get the default region for launching jobs.\n", - "if not REGION:\n", - " REGION = os.environ[\"GOOGLE_CLOUD_REGION\"]\n", - "\n", - "# Initialize Vertex AI API.\n", - "print(\"Initializing Vertex AI API.\")\n", - "aiplatform.init(project=PROJECT_ID, location=REGION)\n", - "\n", - "! gcloud config set project $PROJECT_ID\n", "import vertexai\n", "\n", - "vertexai.init(\n", - " project=PROJECT_ID,\n", - " location=REGION,\n", - ")\n", + "PROJECT_ID = \"[your-project-id]\" # @param {type: \"string\", placeholder: \"[your-project-id]\", isTemplate: true}\n", "\n", - "# @markdown You must provide a Hugging Face User Access Token (with read access) to access models from Hugging Face. You can follow the [Hugging Face documentation](https://huggingface.co/docs/hub/en/security-tokens) to create a **read** access token and put it in the `HF_TOKEN` field below.\n", + "if not PROJECT_ID or PROJECT_ID == \"[your-project-id]\":\n", + " PROJECT_ID = str(os.environ.get(\"GOOGLE_CLOUD_PROJECT\"))\n", "\n", - "HF_TOKEN = \"\" # @param {type:\"string\"}\n", - "assert HF_TOKEN, \"Provide a read HF_TOKEN to load models from Hugging Face.\"\n", + "REGION = os.environ.get(\"GOOGLE_CLOUD_REGION\", \"us-central1\")\n", "\n", - "\n", - "# @markdown Click \"Show code\" to see more details." + "vertexai.init(project=PROJECT_ID, location=REGION)" ] }, { "cell_type": "markdown", "metadata": { - "id": "cEIT7Ol_zs9L" + "id": "Q0CXrvcZH_aw" }, "source": [ - "## Deploy prebuilt LLaVA models" + "### Import libraries" ] }, { @@ -180,208 +189,202 @@ "execution_count": null, "metadata": { "cellView": "form", - "id": "ixW6s1W5_oWN" + "id": "3G2UXB82ICs6" }, "outputs": [], "source": [ - "# @title Select the model variants\n", - "\n", - "# @markdown Set the model to deploy.\n", - "base_model_name = \"llava-1.5-7b-hf\" # @param [\"llava-1.5-7b-hf\", \"llava-1.5-13b-hf\", \"llava-v1.6-mistral-7b-hf\", \"llava-v1.6-vicuna-7b-hf\"]\n", - "MODEL_ID = \"llava-hf/\" + base_model_name\n", - "PUBLISHER_MODEL_NAME = f\"publishers/liuhaotian/models/pytorch-llava@{base_model_name}\"\n", - "\n", - "accelerator_type = \"NVIDIA_L4\" # @param [\"NVIDIA_L4\", \"NVIDIA_TESLA_T4\", \"NVIDIA_TESLA_V100\", \"NVIDIA_TESLA_A100\", \"NVIDIA_A100_80GB\"]\n", - "accelerator_count = 1\n", - "if accelerator_type == \"NVIDIA_L4\":\n", - " machine_type = \"g2-standard-12\"\n", - "elif accelerator_type == \"NVIDIA_TESLA_T4\":\n", - " machine_type = \"n1-standard-8\"\n", - "elif accelerator_type == \"NVIDIA_TESLA_V100\":\n", - " machine_type = \"n1-standard-8\"\n", - "elif accelerator_type == \"NVIDIA_TESLA_A100\":\n", - " machine_type = \"a2-highgpu-1g\"\n", - "elif accelerator_type == \"NVIDIA_A100_80GB\":\n", - " machine_type = \"a2-ultragpu-1g\"\n", - "else:\n", - " raise ValueError(\n", - " f\"Recommended GPU setting not found for: {accelerator_type} and {base_model_name}.\"\n", - " )\n", - "\n", - "# @markdown Set use_dedicated_endpoint to False if you don't want to use [dedicated endpoint](https://cloud.google.com/vertex-ai/docs/general/deployment#create-dedicated-endpoint). Note that [dedicated endpoint does not support VPC Service Controls](https://cloud.google.com/vertex-ai/docs/predictions/choose-endpoint-type), uncheck the box if you are using VPC-SC.\n", - "use_dedicated_endpoint = True # @param {type:\"boolean\"}\n", - "\n", - "# The pre-built serving docker images.\n", - "VLLM_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-vllm-serve:20241007_2233_RC00\"\n", - "\n", - "common_util.check_quota(\n", - " project_id=PROJECT_ID,\n", - " region=REGION,\n", - " accelerator_type=accelerator_type,\n", - " accelerator_count=accelerator_count,\n", - " is_for_training=False,\n", + "from vertexai import model_garden" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "upYRiGtP_-iN" + }, + "source": [ + "## Deploy model" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "H2WC_0hXDVXc" + }, + "source": [ + "### Choose model variant" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "u41zbNa2EoFq" + }, + "source": [ + "You can proceed with the default model variant or select a different one." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form", + "id": "-fgC4NLSDkF7" + }, + "outputs": [], + "source": [ + "model_version = \"llava-1.5-7b-hf\" # @param [\"llama3-llava-next-8b-hf\", \"llava-1.5-13b-hf\", \"llava-1.5-7b-hf\", \"llava-v1.5-7b\", \"llava-v1.6-34b-hf\", \"llava-v1.6-mistral-7b-hf\", \"llava-v1.6-vicuna-13b-hf\", \"llava-v1.6-vicuna-7b-hf\"] {isTemplate:true}\n", + "MODEL_NAME = f\"liuhaotian/pytorch-llava@{model_version}\"" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "VRnUgU8LF3_i" + }, + "source": [ + "To see all deployable model variants available in Model Garden, use:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form", + "id": "-QLd-wshF6sB" + }, + "outputs": [], + "source": [ + "all_model_versions = model_garden.list_deployable_models(\n", + " model_filter=\"pytorch-llava\", list_hf_models=False\n", ")" ] }, + { + "cell_type": "markdown", + "metadata": { + "id": "N0UeFHa2GO63" + }, + "source": [ + "Once you've selected a model variant, initialize it:" + ] + }, { "cell_type": "code", "execution_count": null, "metadata": { "cellView": "form", - "id": "USB7dvYqvNdu" + "id": "GZiV3trBBcA3" }, "outputs": [], "source": [ - "# @title Deploy with customized configs\n", + "model = model_garden.OpenModel(MODEL_NAME)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "-0cL378wFlvf" + }, + "source": [ + "### Check the Deployment Configuration\n", "\n", - "# @markdown This section uploads prebuilt LLaVA models to Model Registry and deploys it to a Vertex AI Endpoint. It takes 15 to 30 minutes to finish depending on the size of the model.\n", + "Use the `list_deploy_options()` method to view the verified deployment configurations for your selected model. This helps ensure you have sufficient resources (e.g., GPU quota) available to deploy it." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form", + "id": "zm73g7vFFm9N" + }, + "outputs": [], + "source": [ + "deploy_options = model.list_deploy_options(concise=True)\n", + "print(deploy_options)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "WjV499VsGwrD" + }, + "source": [ + "### Deploy the Model\n", "\n", - "vllm_dtype = \"bfloat16\"\n", - "max_model_len = 4096\n", - "gpu_memory_utilization = 0.9\n", + "Now that you’ve reviewed the deployment options, use the `deploy()` method to serve the selected open model to a Vertex AI endpoint. Deployment time may vary depending on the model size and infrastructure requirements.\n", "\n", + "> **Note**: If the model requires accepting a license agreement (EULA), set the `accept_eula=True` flag in the deploy call. Set `use_dedicated_endpoint` to False if you don't want to use [dedicated endpoint](https://cloud.google.com/vertex-ai/docs/general/deployment#create-dedicated-endpoint)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form", + "id": "wX1itVTvXdEP" + }, + "outputs": [], + "source": [ + "use_dedicated_endpoint = True" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form", + "id": "MRmPFEPoGzsB" + }, + "outputs": [], + "source": [ + "endpoint = model.deploy(\n", + " accept_eula=True,\n", + " use_dedicated_endpoint=use_dedicated_endpoint,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "PHBtn8DQp-ID" + }, + "source": [ + "Alternatively, you can select one of the verified deployment configurations listed above." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form", + "id": "ADsJG8JYqI6c" + }, + "outputs": [], + "source": [ + "endpoint = model.deploy(\n", + " accept_eula=True,\n", + " use_dedicated_endpoint=use_dedicated_endpoint,\n", + " serving_container_image_uri=\"us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-vllm-serve:20250601_0916_RC01\",\n", + " machine_type=\"g2-standard-12\",\n", + " accelerator_type=\"NVIDIA_L4\",\n", + " accelerator_count=1,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "kqSUK2CwsImi" + }, + "source": [ + "To further customize your deployment, you can configure:\n", "\n", - "def deploy_model_vllm(\n", - " model_name: str,\n", - " model_id: str,\n", - " publisher: str,\n", - " publisher_model_id: str,\n", - " base_model_id: str = None,\n", - " machine_type: str = \"g2-standard-8\",\n", - " accelerator_type: str = \"NVIDIA_L4\",\n", - " accelerator_count: int = 1,\n", - " gpu_memory_utilization: float = 0.9,\n", - " max_model_len: int = 4096,\n", - " dtype: str = \"auto\",\n", - " enable_trust_remote_code: bool = False,\n", - " enforce_eager: bool = False,\n", - " enable_lora: bool = False,\n", - " enable_chunked_prefill: bool = False,\n", - " enable_prefix_cache: bool = False,\n", - " host_prefix_kv_cache_utilization_target: float = 0.0,\n", - " max_loras: int = 1,\n", - " max_cpu_loras: int = 8,\n", - " use_dedicated_endpoint: bool = False,\n", - " max_num_seqs: int = 256,\n", - " model_type: str = None,\n", - " enable_llama_tool_parser: bool = False,\n", - ") -> Tuple[aiplatform.Model, aiplatform.Endpoint]:\n", - " \"\"\"Deploys trained models with vLLM into Vertex AI.\"\"\"\n", - " endpoint = aiplatform.Endpoint.create(\n", - " display_name=f\"{model_name}-endpoint\",\n", - " dedicated_endpoint_enabled=use_dedicated_endpoint,\n", - " )\n", + "- **Compute Resources**: Machine type, replica count (min/max), accelerator type and quantity.\n", + "- **Infrastructure**: Use Spot VMs, reservation affinity, or dedicated endpoints.\n", + "- **Serving Container**: Customize container image, ports, health checks, and environment variables.\n", "\n", - " if not base_model_id:\n", - " base_model_id = model_id\n", - "\n", - " # See https://docs.vllm.ai/en/latest/models/engine_args.html for a list of possible arguments with descriptions.\n", - " vllm_args = [\n", - " \"python\",\n", - " \"-m\",\n", - " \"vllm.entrypoints.api_server\",\n", - " \"--host=0.0.0.0\",\n", - " \"--port=8080\",\n", - " f\"--model={model_id}\",\n", - " f\"--tensor-parallel-size={accelerator_count}\",\n", - " \"--swap-space=16\",\n", - " f\"--max-model-len={max_model_len}\",\n", - " f\"--dtype={dtype}\",\n", - " f\"--max-loras={max_loras}\",\n", - " f\"--max-cpu-loras={max_cpu_loras}\",\n", - " f\"--max-num-seqs={max_num_seqs}\",\n", - " \"--disable-log-stats\",\n", - " ]\n", - "\n", - " if gpu_memory_utilization:\n", - " vllm_args.append(f\"--gpu-memory-utilization={gpu_memory_utilization}\")\n", - "\n", - " if enable_trust_remote_code:\n", - " vllm_args.append(\"--trust-remote-code\")\n", - "\n", - " if enforce_eager:\n", - " vllm_args.append(\"--enforce-eager\")\n", - "\n", - " if enable_lora:\n", - " vllm_args.append(\"--enable-lora\")\n", - "\n", - " if enable_chunked_prefill:\n", - " vllm_args.append(\"--enable-chunked-prefill\")\n", - "\n", - " if enable_prefix_cache:\n", - " vllm_args.append(\"--enable-prefix-caching\")\n", - "\n", - " if 0 < host_prefix_kv_cache_utilization_target < 1:\n", - " vllm_args.append(\n", - " f\"--host-prefix-kv-cache-utilization-target={host_prefix_kv_cache_utilization_target}\"\n", - " )\n", - "\n", - " if model_type:\n", - " vllm_args.append(f\"--model-type={model_type}\")\n", - "\n", - " if enable_llama_tool_parser:\n", - " vllm_args.append(\"--enable-auto-tool-choice\")\n", - " vllm_args.append(\"--tool-call-parser=vertex-llama-3\")\n", - "\n", - " env_vars = {\n", - " \"MODEL_ID\": base_model_id,\n", - " \"DEPLOY_SOURCE\": \"notebook\",\n", - " }\n", - "\n", - " # HF_TOKEN is not a compulsory field and may not be defined.\n", - " try:\n", - " if HF_TOKEN:\n", - " env_vars[\"HF_TOKEN\"] = HF_TOKEN\n", - " except NameError:\n", - " pass\n", - "\n", - " model = aiplatform.Model.upload(\n", - " display_name=model_name,\n", - " serving_container_image_uri=VLLM_DOCKER_URI,\n", - " serving_container_args=vllm_args,\n", - " serving_container_ports=[8080],\n", - " serving_container_predict_route=\"/generate\",\n", - " serving_container_health_route=\"/ping\",\n", - " serving_container_environment_variables=env_vars,\n", - " serving_container_shared_memory_size_mb=(16 * 1024), # 16 GB\n", - " serving_container_deployment_timeout=7200,\n", - " model_garden_source_model_name=(\n", - " f\"publishers/{publisher}/models/{publisher_model_id}\"\n", - " ),\n", - " )\n", - " print(\n", - " f\"Deploying {model_name} on {machine_type} with {accelerator_count} {accelerator_type} GPU(s).\"\n", - " )\n", - " model.deploy(\n", - " endpoint=endpoint,\n", - " machine_type=machine_type,\n", - " accelerator_type=accelerator_type,\n", - " accelerator_count=accelerator_count,\n", - " deploy_request_timeout=1800,\n", - " system_labels={\n", - " \"NOTEBOOK_NAME\": \"model_garden_pytorch_llava.ipynb\",\n", - " \"NOTEBOOK_ENVIRONMENT\": common_util.get_deploy_source(),\n", - " },\n", - " )\n", - " print(\"endpoint_name:\", endpoint.name)\n", - "\n", - " return model, endpoint\n", - "\n", - "\n", - "models[LABEL], endpoints[LABEL] = deploy_model_vllm(\n", - " model_name=common_util.get_job_name_with_datetime(prefix=MODEL_ID),\n", - " model_id=MODEL_ID,\n", - " publisher=\"liuhaotian\",\n", - " publisher_model_id=\"pytorch-llava\",\n", - " machine_type=machine_type,\n", - " accelerator_type=accelerator_type,\n", - " accelerator_count=accelerator_count,\n", - " max_model_len=max_model_len,\n", - " gpu_memory_utilization=gpu_memory_utilization,\n", - " dtype=vllm_dtype,\n", - ")\n", - "\n", - "# @markdown Click \"Show code\" to see more details." + "See the [Model Garden SDK README](https://github.com/googleapis/python-aiplatform/blob/main/vertexai/model_garden/README.md) for advanced configuration options." ] }, { @@ -431,7 +434,7 @@ " \"top_p\": top_p,\n", " },\n", "]\n", - "response = endpoints[LABEL].predict(instances=instances)\n", + "response = endpoint.predict(instances=instances)\n", "\n", "for raw_prediction in response.predictions:\n", " prediction = raw_prediction.split(\"Output:\")\n", @@ -459,16 +462,10 @@ "outputs": [], "source": [ "# @title Delete the models and endpoints\n", - "# @markdown Delete the experiment models and endpoints to recycle the resources\n", - "# @markdown and avoid unnecessary continuous charges that may incur.\n", + "# @markdown Delete the endpoint.\n", "\n", - "# Undeploy model and delete endpoint.\n", - "for endpoint in endpoints.values():\n", - " endpoint.delete(force=True)\n", - "\n", - "# Delete models.\n", - "for model in models.values():\n", - " model.delete()" + "if endpoint:\n", + " endpoint.delete(force=True)" ] } ],