xiaohei/model_garden
1
0
Fork 0
mirror of https://github.com/GoogleCloudPlatform/vertex-ai-samples.git synced 2026-09-26 14:42:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add Qwen2.5 related updates to the notebook

Browse Source 
PiperOrigin-RevId: 683423186
This commit is contained in:
Vertex MG Team
2024-10-07 19:46:47 -07:00
committed by Copybara-Service
parent d68494bd3b
commit 75cf2fe9b9
1 changed files with 26 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
notebooks/community/model_garden/model_garden_pytorch_qwen2_deployment.ipynb
+26 -11
View File
@@ -54,13 +54,13 @@
"source": [
"## Overview\n",
"\n",
"This notebook demonstrates deploying prebuilt [Qwen2 models](https://huggingface.co/collections/Qwen/qwen2-6659360b33528ced941e557f) with [vLLM](https://github.com/vllm-project/vllm) to improve serving throughput.\n",
"This notebook demonstrates deploying prebuilt [Qwen2](https://huggingface.co/collections/Qwen/qwen2-6659360b33528ced941e557f) and [Qwen2.5](https://huggingface.co/collections/Qwen/qwen25-66e81a666513e518adb90d9e) models with [vLLM](https://github.com/vllm-project/vllm) to improve serving throughput.\n",
"\n",
"\n",
"### Objective\n",
"\n",
"- Download and deploy prebuilt Qwen2 models\n",
"- Deploy Qwen2 with [vLLM](https://github.com/vllm-project/vllm) to improve serving throughput\n",
"- Download and deploy prebuilt Qwen2/Qwen2.5 models\n",
"- Deploy Qwen2/Qwen2.5 with [vLLM](https://github.com/vllm-project/vllm) to improve serving throughput\n",
"\n",
"### Costs\n",
"\n",
@@ -117,8 +117,6 @@
"! git clone https://github.com/GoogleCloudPlatform/vertex-ai-samples.git\n",
"\n",
"models, endpoints = {}, {}\n",
"# Dedicated endpoint not supported yet\n",
"use_dedicated_endpoint = False\n",
"\n",
"common_util = importlib.import_module(\n",
" \"vertex-ai-samples.community-content.vertex_model_garden.model_oss.notebook_util.common_util\"\n",
@@ -199,23 +197,22 @@
"source": [
"# @title Deploy\n",
"\n",
"# @markdown This section uploads prebuilt Qwen2 models to Model Registry and deploys it to a Vertex AI Endpoint. It takes 15 to 30 minutes to finish depending on the size of the model.\n",
"# @markdown This section uploads prebuilt Qwen2/Qwen2.5 models to Model Registry and deploys it to a Vertex AI Endpoint. It takes 15 to 30 minutes to finish depending on the size of the model.\n",
"\n",
"MODEL_ID = \"Qwen2-0.5B-Instruct\" # @param [\"Qwen2-0.5B-Instruct\", \"Qwen2-1.5B-Instruct\", \"Qwen2-7B-Instruct\"] {isTemplate: true}\n",
"MODEL_ID = \"Qwen2.5-0.5B-Instruct\" # @param [\"Qwen2-0.5B-Instruct\", \"Qwen2-1.5B-Instruct\", \"Qwen2-7B-Instruct\", \"Qwen2.5-0.5B-Instruct\", \"Qwen2.5-1.5B-Instruct\", \"Qwen2.5-7B-Instruct\", \"Qwen2.5-14B-Instruct\", \"Qwen2.5-32B-Instruct\"] {isTemplate: true}\n",
"model_path_prefix = \"Qwen\"\n",
"model_id = os.path.join(model_path_prefix, MODEL_ID)\n",
"\n",
"# The pre-built serving docker image for vLLM.\n",
"VLLM_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-vllm-serve:20240721_0916_RC00\"\n",
"VLLM_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-vllm-serve:20240919_0916_RC00\"\n",
"\n",
"accelerator_type = \"NVIDIA_L4\" # @param [\"NVIDIA_L4\"] {isTemplate: true}\n",
"vllm_dtype = \"bfloat16\"\n",
"gpu_memory_utilization = 0.85\n",
"max_model_len = 32768 # Maximum context length.\n",
"\n",
"if \"0.5B\" in MODEL_ID or \"1.5B\" in MODEL_ID:\n",
" if accelerator_type == \"NVIDIA_L4\":\n",
" # Sets 1 L4 (12G) to deploy Qwen2-0.5B and Qwen2-1.5B models.\n",
" # Sets 1 L4 (12G) to deploy Qwen2/2.5-0.5B and Qwen2/2.5-1.5B models.\n",
" accelerator_count = 1\n",
" # Sets machine type to g2-standard-12 for 1 L4\n",
" machine_type = \"g2-standard-12\"\n",
@@ -226,7 +223,7 @@
" )\n",
"elif \"7B\" in MODEL_ID:\n",
" if accelerator_type == \"NVIDIA_L4\":\n",
" # Sets 2 L4 (24G) to deploy Qwen2-7B model.\n",
" # Sets 2 L4 (24G) to deploy Qwen2/2.5-7B model.\n",
" accelerator_count = 2\n",
" # Sets machine type to g2-standard-24 for 2 L4's\n",
" machine_type = \"g2-standard-24\"\n",
@@ -235,9 +232,27 @@
" \"Recommended machine settings not found for accelerator type: %s\"\n",
" % accelerator_type\n",
" )\n",
"elif \"14B\" in MODEL_ID or \"32B\" in MODEL_ID:\n",
" if accelerator_type == \"NVIDIA_L4\":\n",
" # Sets 4 L4 (48G) to deploy Qwen2.5-14B and Qwen2.5-32B models.\n",
" accelerator_count = 4\n",
" # Sets machine type to g2-standard-48 for 4 L4's\n",
" machine_type = \"g2-standard-48\"\n",
" else:\n",
" raise ValueError(\n",
" \"Recommended machine settings not found for accelerator type: %s\"\n",
" % accelerator_type\n",
" )\n",
"else:\n",
" raise ValueError(\"Invalid model id: %s\" % MODEL_ID)\n",
"\n",
"gpu_memory_utilization = 0.85\n",
"if \"32B\" in MODEL_ID:\n",
" gpu_memory_utilization = 0.95\n",
"\n",
"# Dedicated endpoint not supported yet.\n",
"use_dedicated_endpoint = False\n",
"\n",
"common_util.check_quota(\n",
" project_id=PROJECT_ID,\n",
" region=REGION,\n",