xiaohei/model_garden
1
0
Fork 0
mirror of https://github.com/GoogleCloudPlatform/vertex-ai-samples.git synced 2026-09-26 14:42:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update vLLM version in Llama 3.1 and Guard deployment notebooks. (#3335)

Browse Source
This commit is contained in:
Kathy Yu
2024-07-27 00:25:53 +00:00
committed by GitHub
parent 2111ade4a6
commit 462f0b6276
2 changed files with 9 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
notebooks/community/model_garden/model_garden_llama_guard_deployment.ipynb
+5 -3
View File
@@ -193,7 +193,7 @@
"! gsutil -m cp -R $VERTEX_AI_MODEL_GARDEN_LLAMA_GUARD $MODEL_BUCKET\n",
"\n",
"# The pre-built serving docker images.\n",
"VLLM_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-vllm-serve:20240724_0916_RC00\"\n",
"VLLM_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-vllm-serve:20240726_1329_RC00\"\n",
"\n",
"\n",
"def deploy_model_vllm(\n",
@@ -229,6 +229,8 @@
" \"--disable-log-stats\",\n",
" \"--enforce-eager\",\n",
" \"--disable-custom-all-reduce\",\n",
" \"--enable-chunked-prefill\",\n",
" \"--max-num-seqs=12\",\n",
" ]\n",
"\n",
" env_vars = {\n",
@@ -319,8 +321,8 @@
" is_for_training=False,\n",
")\n",
"\n",
"gpu_memory_utilization = 0.85\n",
"max_model_len = 8192 # Maximum context length.\n",
"gpu_memory_utilization = 0.9\n",
"max_model_len = 32768 # Maximum context length.\n",
"\n",
"models[\"vllm_gpu\"], endpoints[\"vllm_gpu\"] = deploy_model_vllm(\n",
" model_name=common_util.get_job_name_with_datetime(prefix=MODEL_ID),\n",
notebooks/community/model_garden/model_garden_pytorch_llama3_1_deployment.ipynb
+4 -3
View File
@@ -223,7 +223,7 @@
"\n",
"# The pre-built serving docker images.\n",
"HEXLLM_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai-restricted/vertex-vision-model-garden-dockers/hex-llm-serve:llama3.1\"\n",
"VLLM_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-vllm-serve:20240724_0916_RC00\"\n",
"VLLM_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-vllm-serve:20240726_1329_RC00\"\n",
"\n",
"SERVICE_ENDPOINT = \"aiplatform.googleapis.com\"\n",
"\n",
@@ -261,6 +261,7 @@
" f\"--hbm_utilization_factor={hbm_utilization_factor}\",\n",
" f\"--max_running_seqs={max_running_seqs}\",\n",
" f\"--max_model_len={max_model_len}\",\n",
" \"--max-num-seqs=12\",\n",
" ]\n",
" hexllm_envs = {\n",
" \"PJRT_DEVICE\": \"TPU\",\n",
@@ -530,8 +531,8 @@
" is_for_training=False,\n",
")\n",
"\n",
"gpu_memory_utilization = 0.8\n",
"max_model_len = 8192 # Maximum context length.\n",
"gpu_memory_utilization = 0.9\n",
"max_model_len = 32768 # Maximum context length.\n",
"\n",
"model, endpoint = deploy_model_vllm(\n",
" model_name=common_util.get_job_name_with_datetime(prefix=\"llama_3_1-vllm-serve\"),\n",