xiaohei/model_garden
1
0
Fork 0
mirror of https://github.com/GoogleCloudPlatform/vertex-ai-samples.git synced 2026-09-26 14:42:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add GPTQ to Falcon Instruct (#2574)

Browse Source 
* Add local inference for Falcon Instruct

* Lint

* Add memory necessary for falcon 7b

* nit: Typo

* add gptq quantization to falcon

* lint

* typo

* Comment out one setting.
This commit is contained in:
dstnluong-google
2023-12-13 19:49:39 +00:00
committed by GitHub
parent d2ad1dd763
commit 90219e0a76
1 changed files with 564 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
notebooks/community/model_garden/model_garden_pytorch_falcon_instruct_peft.ipynb
+564 -11
View File
@@ -60,13 +60,14 @@
"source": [
"## Overview\n",
"\n",
"This notebook demonstrates running inferences locally with prebuilt Falcon Instruct models, deploying prebuilt Falcon Instruct models, finetuning and deploying Falcon Instruct models with performance efficient finetuning libraries ([PEFT](https://github.com/huggingface/peft)), and evaluating PEFT-finetuned Falcon Instruct models in Vertex AI.\n",
"This notebook demonstrates running inferences locally with prebuilt Falcon Instruct models, deploying prebuilt Falcon Instruct models, finetuning and deploying Falcon Instruct models with performance efficient finetuning libraries ([PEFT](https://github.com/huggingface/peft)), quantizing and deploying Falcon Instruct models with [GPTQ](https://arxiv.org/abs/2210.17323), and evaluating PEFT-finetuned Falcon Instruct models in Vertex AI.\n",
"\n",
"### Objective\n",
"\n",
"- Run inferences locally on prebuilt Falcon Instruct models\n",
"- Deploy prebuilt Falcon Instruct models\n",
"- Finetune and deploy Falcon Instruct models with PEFT\n",
"- Quantize and deploy Falcon Instruct models with GPTQ\n",
"- Evaluate PEFT-finetuned Falcon Instruct models\n",
"\n",
"| Models | LoRA |\n",
@@ -119,11 +120,10 @@
"\n",
"if \"google.colab\" in sys.modules:\n",
" ! pip3 install --upgrade google-cloud-aiplatform\n",
" ! pip3 install google-cloud-language==2.10.0\n",
" from google.colab import auth as google_auth\n",
"\n",
" google_auth.authenticate_user()\n",
" # Install gdown for downloading example training images.\n",
" ! pip3 install gdown\n",
"\n",
" # Restart the notebook kernel after installs.\n",
" import IPython\n",
@@ -147,7 +147,7 @@
"\n",
"1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
"\n",
"1. [Enable the Vertex AI API and Compute Engine API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component).\n",
"1. [Enable the Vertex AI API, Compute Engine API, and Cloud Natural Language API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component,language.googleapis.com).\n",
"\n",
"1. [Create a Cloud Storage bucket](https://cloud.google.com/storage/docs/creating-buckets) for storing experiment outputs.\n",
"\n",
@@ -237,8 +237,10 @@
"outputs": [],
"source": [
"# The pre-built training, serving and evaluation docker images.\n",
"TRAIN_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-peft-train:20231020_0936_RC00\"\n",
"PREDICTION_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-peft-serve:20231026_1907_RC00\"\n",
"TRAIN_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-peft-train:20231207_0936_RC00\"\n",
"PREDICTION_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-peft-serve:20231129_0948_RC00\"\n",
"VLLM_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-vllm-serve:20231127_0916_RC00\"\n",
"VLLM_GPTQ_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-vllm-serve:gptq\"\n",
"EVAL_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-lm-evaluation-harness:20231011_0934_RC00\""
]
},
@@ -261,8 +263,9 @@
"source": [
"import os\n",
"from datetime import datetime\n",
"from typing import Tuple\n",
"\n",
"from google.cloud import aiplatform\n",
"from google.cloud import aiplatform, language\n",
"\n",
"\n",
"def get_job_name_with_datetime(prefix: str) -> str:\n",
@@ -281,7 +284,7 @@
" machine_type: str = \"n1-standard-8\",\n",
" accelerator_type: str = \"NVIDIA_TESLA_V100\",\n",
" accelerator_count: int = 1,\n",
") -> tuple[aiplatform.Model, aiplatform.Endpoint]:\n",
") -> Tuple[aiplatform.Model, aiplatform.Endpoint]:\n",
" \"\"\"Deploys trained models into Vertex AI.\"\"\"\n",
" endpoint = aiplatform.Endpoint.create(display_name=f\"{model_name}-endpoint\")\n",
" serving_env = {\n",
@@ -306,7 +309,130 @@
" deploy_request_timeout=1800,\n",
" service_account=service_account,\n",
" )\n",
" return model, endpoint"
" return model, endpoint\n",
"\n",
"\n",
"def deploy_model_vllm(\n",
" model_name: str,\n",
" model_id: str,\n",
" service_account: str,\n",
" machine_type: str = \"n1-standard-8\",\n",
" accelerator_type: str = \"NVIDIA_TESLA_V100\",\n",
" accelerator_count: int = 1,\n",
" quantization_method: str = \"\",\n",
") -> Tuple[aiplatform.Model, aiplatform.Endpoint]:\n",
" \"\"\"Deploys trained models with vLLM into Vertex AI.\"\"\"\n",
" endpoint = aiplatform.Endpoint.create(display_name=f\"{model_name}-endpoint\")\n",
"\n",
" vllm_args = [\n",
" \"--host=0.0.0.0\",\n",
" \"--port=7080\",\n",
" f\"--model={model_id}\",\n",
" f\"--tensor-parallel-size={accelerator_count}\",\n",
" \"--swap-space=16\",\n",
" \"--gpu-memory-utilization=0.9\",\n",
" \"--disable-log-stats\",\n",
" \"--dtype=float16\",\n",
" \"--trust-remote-code\",\n",
" ]\n",
" if quantization_method:\n",
" vllm_args.append(f\"--quantization={quantization_method}\")\n",
" if quantization_method == \"gptq\":\n",
" vllm_docker_uri = VLLM_GPTQ_DOCKER_URI\n",
" else:\n",
" vllm_docker_uri = VLLM_DOCKER_URI\n",
"\n",
" model = aiplatform.Model.upload(\n",
" display_name=model_name,\n",
" serving_container_image_uri=vllm_docker_uri,\n",
" serving_container_command=[\"python\", \"-m\", \"vllm.entrypoints.api_server\"],\n",
" serving_container_args=vllm_args,\n",
" serving_container_ports=[7080],\n",
" serving_container_predict_route=\"/generate\",\n",
" serving_container_health_route=\"/ping\",\n",
" )\n",
"\n",
" model.deploy(\n",
" endpoint=endpoint,\n",
" machine_type=machine_type,\n",
" accelerator_type=accelerator_type,\n",
" accelerator_count=accelerator_count,\n",
" deploy_request_timeout=1800,\n",
" service_account=service_account,\n",
" )\n",
" return model, endpoint\n",
"\n",
"\n",
"def moderate_text(text: str) -> language.ModerateTextResponse:\n",
" \"\"\"Calls Vertex AI APIs to analyze text moderations.\"\"\"\n",
" client = language.LanguageServiceClient()\n",
" document = language.Document(\n",
" content=text,\n",
" type_=language.Document.Type.PLAIN_TEXT,\n",
" )\n",
" return client.moderate_text(document=document)\n",
"\n",
"\n",
"def show_text_moderation(text: str, response: language.ModerateTextResponse) -> None:\n",
" \"\"\"Shows text moderation results.\"\"\"\n",
" import pandas as pd\n",
"\n",
" def confidence(category: language.ClassificationCategory) -> float:\n",
" return category.confidence\n",
"\n",
" columns = [\"category\", \"confidence\"]\n",
" categories = sorted(response.moderation_categories, key=confidence, reverse=True)\n",
" data = ((category.name, category.confidence) for category in categories)\n",
" df = pd.DataFrame(columns=columns, data=data)\n",
"\n",
" print(f\"Text analyzed:\\n{text}\")\n",
" print(df.to_markdown(index=False, tablefmt=\"presto\", floatfmt=\".0%\"))"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "06e73cb3f412"
},
"source": [
"## Run inferences locally with prebuilt Falcon Instruct models\n",
"\n",
"You will need at least 16GB of memory to swiftly run inference with Falcon-7B-Instruct."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "3ea64305957f"
},
"outputs": [],
"source": [
"import torch\n",
"import transformers\n",
"from transformers import AutoTokenizer\n",
"\n",
"model = \"tiiuae/falcon-7b-instruct\"\n",
"\n",
"tokenizer = AutoTokenizer.from_pretrained(model)\n",
"pipeline = transformers.pipeline(\n",
" \"text-generation\",\n",
" model=model,\n",
" tokenizer=tokenizer,\n",
" torch_dtype=torch.bfloat16,\n",
" trust_remote_code=True,\n",
" device_map=\"auto\",\n",
")\n",
"sequences = pipeline(\n",
" \"Girafatron is obsessed with giraffes, the most glorious animal on the face of this Earth. Girafatron believes all other animals are irrelevant when compared to the glorious majesty of the giraffe.\\nDaniel: Hello, Girafatron!\\nGirafatron:\",\n",
" max_length=200,\n",
" do_sample=True,\n",
" top_k=10,\n",
" num_return_sequences=1,\n",
" eos_token_id=tokenizer.eos_token_id,\n",
")\n",
"for seq in sequences:\n",
" print(f\"Result: {seq['generated_text']}\")"
]
},
{
@@ -797,6 +923,428 @@
" print(prediction)"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "YcbApnK4iyyu"
},
"source": [
"## Quantize and deploy Falcon Instruct models\n",
"\n",
"This section demonstrates post-training quantization of Falcon Instruct models with Vertex Custom Job. Quantization reduces the memory required by a model while attempting to retain the same performance. Read more about GPTQ in the following publication: [GPTQ: Accurate Post-Training Quantization for Generative Pre-trained Transformers](https://arxiv.org/abs/2210.17323)."
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "Rt9pFQV-jG1v"
},
"source": [
"### Deploy pre-quantized models with Google Cloud Text Moderation\n",
"Many GPTQ-quantized models are provided [here](https://huggingface.co/TheBloke?search_models=-gptq).\n",
"\n",
"This section uploads the model to Model Registry and deploys it on the Endpoint.\n",
"\n",
"The model deployment step will take 15 minutes to 1 hour to complete, depending on the model sizes.\n",
"\n",
"Notice that deploying a quantized requires much less GPU. We can deploy a quantized 40B model with only two L4s instead of four."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "_SFje2ifjNIx"
},
"outputs": [],
"source": [
"quantized_model_id = \"TheBloke/Falcon-7B-Instruct-GPTQ\" # @param [\"TheBloke/Falcon-7B-Instruct-GPTQ\", \"TheBloke/falcon-40b-instruct-GPTQ\"]\n",
"\n",
"quantization_method = \"gptq\"\n",
"\n",
"# Finds Vertex AI prediction supported accelerators and regions in\n",
"# https://cloud.google.com/vertex-ai/docs/predictions/configure-compute.\n",
"\n",
"# Sets 1 L4 (24G) to deploy Falcon Instruct 7B model.\n",
"machine_type = \"g2-standard-8\"\n",
"accelerator_type = \"NVIDIA_L4\"\n",
"accelerator_count = 1\n",
"\n",
"# Sets 2 L4's (24G) to deploy Falcon Instruct 40B model.\n",
"# machine_type = \"g2-standard-24\"\n",
"# accelerator_type = \"NVIDIA_L4\"\n",
"# accelerator_count = 2\n",
"\n",
"model_prequantized_vllm, endpoint_prequantized_vllm = deploy_model_vllm(\n",
" model_name=get_job_name_with_datetime(\n",
" prefix=\"falcon-instruct-serve-vllm-prequantized\"\n",
" ),\n",
" model_id=quantized_model_id,\n",
" service_account=SERVICE_ACCOUNT,\n",
" machine_type=machine_type,\n",
" accelerator_type=accelerator_type,\n",
" accelerator_count=accelerator_count,\n",
" quantization_method=quantization_method,\n",
")"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "7vpslc04szHW"
},
"source": [
"NOTE: After the deployment succeeds, the model weights will be downloaded on the fly. Thus additional 10 ~ 40 minutes (depending on the model sizes) of waiting time is needed **after** the above model deployment step succeeds and before you run the next step below. Otherwise you might see a `ServiceUnavailable: 503 502:Bad Gateway` error when you send requests to the endpoint.\n",
"\n",
"Once deployment succeeds, you can send requests to the endpoint with text prompts.\n",
"\n",
"Example:\n",
"\n",
"```\n",
"Human: What is a car?\n",
"Assistant: A car, or a motor car, is a road-connected human-transportation system used to move people or goods from one place to another. The term also encompasses a wide range of vehicles, including motorboats, trains, and aircrafts. Cars typically have four wheels, a cabin for passengers, and an engine or motor. They have been around since the early 19th century and are now one of the most popular forms of transportation, used for daily commuting, shopping, and other purposes.\n",
"```"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "HPFMjyMEs0qt"
},
"outputs": [],
"source": [
"# Loads an existing endpoint instance using the endpoint name:\n",
"# - Using `endpoint_name = endpoint_prequantized_vllm.name` allows us to get the\n",
"# endpoint name of the endpoint `endpoint_prequantized_vllm` created in the cell\n",
"# above.\n",
"# - Alternatively, you can set `endpoint_name = \"1234567890123456789\"` to load\n",
"# an existing endpoint with the ID 1234567890123456789.\n",
"# You may uncomment the code below to load an existing endpoint.\n",
"\n",
"# endpoint_name = endpoint_prequantized_vllm.name\n",
"# # endpoint_name = \"\" # @param {type:\"string\"}\n",
"# aip_endpoint_name = (\n",
"# f\"projects/{PROJECT_ID}/locations/{REGION}/endpoints/{endpoint_name}\"\n",
"# )\n",
"# endpoint_prequantized_vllm = aiplatform.Endpoint(aip_endpoint_name)\n",
"\n",
"\n",
"# Overides max_length and top_k parameters during inferences.\n",
"# If you encounter the issue like `ServiceUnavailable: 503 Took too long to respond when processing`,\n",
"# you can reduce the max length, such as set max_length as 20.\n",
"# If you are using L4 GPUs to serve Falcon Instruct 40B models, you should set\n",
"# max_length to around 1,000 tokens or fewer. If you need longer generated\n",
"# sequences, please file a request with Vertex to allowlist your project for a\n",
"# longer timeout threshold with Vertex endpoints.\n",
"instances = [\n",
" {\n",
" \"prompt\": \"What is a car?\",\n",
" \"max_tokens\": 50,\n",
" \"temperature\": 1.0,\n",
" \"top_p\": 1.0,\n",
" \"top_k\": 10,\n",
" },\n",
"]\n",
"response = endpoint_prequantized_vllm.predict(instances=instances)\n",
"\n",
"for prediction in response.predictions:\n",
" print(prediction)"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "-0Xjw3Gxs_Lv"
},
"source": [
"Text moderation analyzes a document against a list of safety attributes, which include \"harmful categories\" and topics that may be considered sensitive."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "GR2a8rzvs_kk"
},
"outputs": [],
"source": [
"for generated_text in response.predictions:\n",
" # Send a request to the API.\n",
" response = moderate_text(generated_text)\n",
" # Show the results.\n",
" show_text_moderation(generated_text, response)"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "ApFQMgVKmUbD"
},
"source": [
"### Quantize Falcon Instruct models\n",
"\n",
"Quantization reduces the amount of GPU required to serve a model by reducing the bit precision of the weights while minimizing drop in performance. Serving quantized models on VLLM requires models to be quantized to 4 bits. It is recommended to first search if a model has already been quantized and made publicly available: [GPTQ](https://huggingface.co/TheBloke?search_models=-gptq).\n",
"\n",
"Quantizing models with GPTQ will take around 40 minutes for Falcon Instruct 7B using 1 NVIDIA_L4 GPU and will take around 4 hours for Falcon Instruct 40B using 4 NVIDIA_L4 GPU.\n",
"\n",
"Finetuned models can also be quantized, so long as the LoRA weights are merged with the base model."
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "Wz8V8oNkxdEZ"
},
"source": [
"Set the base model id."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "Oeyvnj8BxX8K"
},
"outputs": [],
"source": [
"base_model_id = \"tiiuae/falcon-7b-instruct\" # @param [\"tiiuae/falcon-7b-instruct\", \"tiiuae/falcon-40b-instruct\"]"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "MEquWSCkmrJ6"
},
"outputs": [],
"source": [
"# Set up quantization job.\n",
"\n",
"# Set `finetuned_model_path` to the GCS path of the merged finetuned model\n",
"# from the section above, if not set, the base model will be quantized.\n",
"finetuned_model_path = \"\" # @param {type:\"string\"}\n",
"if finetuned_model_path:\n",
" prequantized_model_path = finetuned_model_path\n",
"else:\n",
" prequantized_model_path = base_model_id\n",
"\n",
"quantization_method = \"gptq\"\n",
"quantization_job_name = get_job_name_with_datetime(\n",
" f\"falcon-instruct-{quantization_method}-quantize\"\n",
")\n",
"\n",
"quantization_output_dir = os.path.join(MODEL_BUCKET, quantization_job_name)\n",
"quantization_output_dir_gcsfuse = quantization_output_dir.replace(\"gs://\", \"/gcs/\")\n",
"\n",
"# Worker pool spec.\n",
"\n",
"# Set 1 L4 (24G) for quantizing 7b model.\n",
"machine_type = \"g2-standard-32\"\n",
"accelerator_type = \"NVIDIA_L4\"\n",
"accelerator_count = 1\n",
"\n",
"# Set 4 L4 (24G) for quantizing 40b model.\n",
"# machine_type = \"g2-standard-48\"\n",
"# accelerator_type = \"NVIDIA_L4\"\n",
"# accelerator_count = 4\n",
"\n",
"# Quantization parameters.\n",
"quantization_precision_mode = \"4bit\"\n",
"\n",
"# The original datasets used in GPTQ paper.\n",
"gptq_dataset_name = \"c4\" # @param [\"wikitext2\",\"c4\",\"c4-new\",\"ptb\",\"ptb-new\"]\n",
"group_size = -1\n",
"damp_percent = 0.1\n",
"desc_act = True\n",
"quantization_args = [\n",
" \"--task=quantize-model\",\n",
" f\"--quantization_method={quantization_method}\",\n",
" f\"--pretrained_model_id={base_model_id}\",\n",
" f\"--quantization_precision_mode={quantization_precision_mode}\",\n",
" f\"--quantization_output_dir={quantization_output_dir_gcsfuse}\",\n",
" f\"--quantization_dataset_name={gptq_dataset_name}\",\n",
" f\"--group_size={group_size}\",\n",
" f\"--damp_percent={damp_percent}\",\n",
" f\"--desc_act={desc_act}\",\n",
" \"--cache_examples_on_gpu=False\",\n",
"]\n",
"\n",
"# Pass quantization arguments and launch job.\n",
"worker_pool_specs = [\n",
" {\n",
" \"machine_spec\": {\n",
" \"machine_type\": machine_type,\n",
" \"accelerator_type\": accelerator_type,\n",
" \"accelerator_count\": accelerator_count,\n",
" },\n",
" \"replica_count\": 1,\n",
" \"disk_spec\": {\n",
" \"boot_disk_type\": \"pd-ssd\",\n",
" \"boot_disk_size_gb\": 500,\n",
" },\n",
" \"container_spec\": {\n",
" \"image_uri\": TRAIN_DOCKER_URI,\n",
" \"env\": [\n",
" {\n",
" \"name\": \"PYTORCH_CUDA_ALLOC_CONF\",\n",
" \"value\": \"max_split_size_mb:32\",\n",
" },\n",
" ],\n",
" \"command\": [],\n",
" \"args\": quantization_args,\n",
" },\n",
" }\n",
"]\n",
"\n",
"print(f\"Quantizing {prequantized_model_path}.\")\n",
"quantize_job = aiplatform.CustomJob(\n",
" display_name=quantization_job_name,\n",
" project=PROJECT_ID,\n",
" worker_pool_specs=worker_pool_specs,\n",
" staging_bucket=STAGING_BUCKET,\n",
")\n",
"quantize_job.run()\n",
"\n",
"print(\"Quantized models were saved in: \", quantization_output_dir)"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "H7GIC4AYrG0E"
},
"source": [
"### Deploy quantized models with Google Cloud Text Moderation\n",
"This section uploads the model to Model Registry and deploys it on the Endpoint.\n",
"\n",
"The model deployment step will take 15 minutes to 1 hour to complete, depending on the model sizes.\n",
"\n",
"Notice that deploying a quantized model requires much less GPU. We can deploy a quantized 40B model with only two L4 GPUs instead of four."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "0SpJ_lbatIf7"
},
"outputs": [],
"source": [
"# Finds Vertex AI prediction supported accelerators and regions in\n",
"# https://cloud.google.com/vertex-ai/docs/predictions/configure-compute.\n",
"\n",
"# Sets 1 L4 (24G) to deploy Falcon Instruct 7B.\n",
"machine_type = \"g2-standard-8\"\n",
"accelerator_type = \"NVIDIA_L4\"\n",
"accelerator_count = 1\n",
"\n",
"# Sets 2 L4's (24G) to deploy Falcon Instruct 70B models.\n",
"# machine_type = \"g2-standard-24\"\n",
"# accelerator_type = \"NVIDIA_L4\"\n",
"# accelerator_count = 2\n",
"\n",
"model_quantized_vllm, endpoint_quantized_vllm = deploy_model_vllm(\n",
" model_name=get_job_name_with_datetime(\n",
" prefix=\"falcon-instruct-serve-vllm-quantized\"\n",
" ),\n",
" model_id=quantization_output_dir,\n",
" service_account=SERVICE_ACCOUNT,\n",
" machine_type=machine_type,\n",
" accelerator_type=accelerator_type,\n",
" accelerator_count=accelerator_count,\n",
" quantization_method=quantization_method,\n",
")"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "Dzq7aM6ctPD9"
},
"source": [
"NOTE: After the deployment succeeds, the model weights will be downloaded on the fly. Thus additional 10 ~ 40 minutes (depending on the model sizes) of waiting time is needed **after** the above model deployment step succeeds and before you run the next step below. Otherwise you might see a `ServiceUnavailable: 503 502:Bad Gateway` error when you send requests to the endpoint.\n",
"\n",
"Once deployment succeeds, you can send requests to the endpoint with text prompts.\n",
"\n",
"Example:\n",
"\n",
"```\n",
"Human: What is a car?\n",
"Assistant: A car, or a motor car, is a road-connected human-transportation system used to move people or goods from one place to another. The term also encompasses a wide range of vehicles, including motorboats, trains, and aircrafts. Cars typically have four wheels, a cabin for passengers, and an engine or motor. They have been around since the early 19th century and are now one of the most popular forms of transportation, used for daily commuting, shopping, and other purposes.\n",
"```"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "twm_awnMtQa2"
},
"outputs": [],
"source": [
"# Loads an existing endpoint instance using the endpoint name:\n",
"# - Using `endpoint_name = endpoint_quantized_vllm.name` allows us to get the\n",
"# endpoint name of the endpoint `endpoint_quantized_vllm` created in the cell\n",
"# above.\n",
"# - Alternatively, you can set `endpoint_name = \"1234567890123456789\"` to load\n",
"# an existing endpoint with the ID 1234567890123456789.\n",
"# You may uncomment the code below to load an existing endpoint.\n",
"\n",
"# endpoint_name = endpoint_quantized_vllm.name\n",
"# # endpoint_name = \"\" # @param {type:\"string\"}\n",
"# aip_endpoint_name = (\n",
"# f\"projects/{PROJECT_ID}/locations/{REGION}/endpoints/{endpoint_name}\"\n",
"# )\n",
"# endpoint_quantized_vllm = aiplatform.Endpoint(aip_endpoint_name)\n",
"\n",
"\n",
"# Overides max_length and top_k parameters during inferences.\n",
"# If you encounter the issue like `ServiceUnavailable: 503 Took too long to respond when processing`,\n",
"# you can reduce the max length, such as set max_length as 20.\n",
"# If you are using L4 GPUs to serve Falcon Instruct 40B models, you should set\n",
"# max_length to around 1,000 tokens or fewer. If you need longer generated\n",
"# sequences, please file a request with Vertex to allowlist your project for a\n",
"# longer timeout threshold with Vertex endpoints.\n",
"instances = [\n",
" {\n",
" \"prompt\": \"What is a car?\",\n",
" \"max_tokens\": 50,\n",
" \"temperature\": 1.0,\n",
" \"top_p\": 1.0,\n",
" \"top_k\": 10,\n",
" },\n",
"]\n",
"response = endpoint_quantized_vllm.predict(instances=instances)\n",
"\n",
"for prediction in response.predictions:\n",
" print(prediction)"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "AeUE4K_CteAj"
},
"source": [
"Text moderation analyzes a document against a list of safety attributes, which include \"harmful categories\" and topics that may be considered sensitive."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "fAoVzA69teui"
},
"outputs": [],
"source": [
"for generated_text in response.predictions:\n",
" # Send a request to the API.\n",
" response = moderate_text(generated_text)\n",
" # Show the results.\n",
" show_text_moderation(generated_text, response)"
]
},
{
"cell_type": "markdown",
"metadata": {
@@ -991,17 +1539,22 @@
},
"outputs": [],
"source": [
"# Delete custom train and evaluation jobs.\n",
"# Delete custom train, quantization, and evaluation jobs.\n",
"train_job.delete()\n",
"quantize_job.delete()\n",
"eval_job.delete()\n",
"\n",
"# Undeploy models and delete endpoints.\n",
"endpoint_without_peft.delete(force=True)\n",
"endpoint_with_peft.delete(force=True)\n",
"endpoint_prequantized_vllm.delete(force=True)\n",
"endpoint_quantized_vllm.delete(force=True)\n",
"\n",
"# Delete models.\n",
"model_without_peft.delete()\n",
"model_with_peft.delete()"
"model_with_peft.delete()\n",
"model_prequantized_vllm.delete()\n",
"model_quantized_vllm.delete()"
]
}
],