xiaohei/model_garden
1
0
Fork 0
mirror of https://github.com/GoogleCloudPlatform/vertex-ai-samples.git synced 2026-09-26 14:42:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update swap space setting for CodeLLaMA notebook. (#2462)

Browse Source
This commit is contained in:
Kathy Yu
2023-11-07 00:13:51 +00:00
committed by GitHub
parent 4873c54cf0
commit d27adf8c47
1 changed files with 12 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
notebooks/community/model_garden/model_garden_pytorch_codellama.ipynb
+12 -1
View File
@@ -300,6 +300,7 @@
" model_id: str,\n",
" service_account: str,\n",
" precision: str = \"float16\",\n",
" swap_space: int = 16,\n",
" machine_type: str = \"n1-standard-8\",\n",
" accelerator_type: str = \"NVIDIA_TESLA_V100\",\n",
" accelerator_count: int = 1,\n",
@@ -312,7 +313,7 @@
" \"--port=7080\",\n",
" f\"--model={model_id}\",\n",
" f\"--tensor-parallel-size={accelerator_count}\",\n",
" \"--swap-space=16\",\n",
" f\"--swap-space={swap_space}\",\n",
" \"--gpu-memory-utilization=0.95\",\n",
" f\"--dtype={precision}\",\n",
" \"--disable-log-stats\",\n",
@@ -569,6 +570,7 @@
"accelerator_type = \"NVIDIA_TESLA_V100\"\n",
"accelerator_count = 1\n",
"vllm_precision = \"float16\"\n",
"vllm_swap_space = 16\n",
"\n",
"# Sets 1 L4 (24G) to deploy 7B models.\n",
"# L4 serving is more cost efficient than V100 serving.\n",
@@ -576,12 +578,14 @@
"# accelerator_type = \"NVIDIA_L4\"\n",
"# accelerator_count = 1\n",
"# vllm_precision = \"bfloat16\"\n",
"# vllm_swap_space = 16\n",
"\n",
"# Sets A100 (40G) to deploy 7B models.\n",
"# machine_type = \"a2-highgpu-1g\"\n",
"# accelerator_type = \"NVIDIA_TESLA_A100\"\n",
"# accelerator_count = 1\n",
"# vllm_precision = \"bfloat16\"\n",
"# vllm_swap_space = 16\n",
"\n",
"# Sets 2 V100 (16G) to deploy 13B models.\n",
"# V100 serving has better throughput and latency performance than L4 serving.\n",
@@ -589,6 +593,7 @@
"# accelerator_type = \"NVIDIA_TESLA_V100\"\n",
"# accelerator_count = 2\n",
"# vllm_precision = \"float16\"\n",
"# vllm_swap_space = 16\n",
"\n",
"# Sets 2 L4 (24G) to deploy 13B models.\n",
"# L4 serving is more cost efficient than V100 serving.\n",
@@ -596,12 +601,14 @@
"# accelerator_type = \"NVIDIA_L4\"\n",
"# accelerator_count = 2\n",
"# vllm_precision = \"bfloat16\"\n",
"# vllm_swap_space = 16\n",
"\n",
"# Sets A100 (40G) to deploy 13B models.\n",
"# machine_type = \"a2-highgpu-1g\"\n",
"# accelerator_type = \"NVIDIA_TESLA_A100\"\n",
"# accelerator_count = 1\n",
"# vllm_precision = \"bfloat16\"\n",
"# vllm_swap_space = 16\n",
"\n",
"# Sets 8 V100 (16G) to deploy 34B models.\n",
"# V100 serving has better throughput and latency performance than L4 serving.\n",
@@ -609,6 +616,7 @@
"# accelerator_type = \"NVIDIA_TESLA_V100\"\n",
"# accelerator_count = 8\n",
"# vllm_precision = \"float16\"\n",
"# vllm_swap_space = 12\n",
"\n",
"# Sets 4 L4 (24G) to deploy 34B models.\n",
"# L4 serving is more cost efficient than V100 serving.\n",
@@ -616,18 +624,21 @@
"# accelerator_type = \"NVIDIA_L4\"\n",
"# accelerator_count = 4\n",
"# vllm_precision = \"bfloat16\"\n",
"# vllm_swap_space = 16\n",
"\n",
"# Sets 2 A100 (40G) to deploy 34B models.\n",
"# machine_type = \"a2-highgpu-2g\"\n",
"# accelerator_type = \"NVIDIA_TESLA_A100\"\n",
"# accelerator_count = 2\n",
"# vllm_precision = \"bfloat16\"\n",
"# vllm_swap_space = 16\n",
"\n",
"model_vllm, endpoint_vllm = deploy_model_vllm(\n",
" model_name=get_job_name_with_datetime(prefix=\"code-llama-serve-vllm\"),\n",
" model_id=model_id,\n",
" service_account=SERVICE_ACCOUNT,\n",
" precision=vllm_precision,\n",
" swap_space=vllm_swap_space,\n",
" machine_type=machine_type,\n",
" accelerator_type=accelerator_type,\n",
" accelerator_count=accelerator_count,\n",