mirror of
https://github.com/GoogleCloudPlatform/vertex-ai-samples.git
synced 2026-09-26 14:42:04 +00:00
refactor: refactor llama3 deployment nb (#3100)
Co-authored-by: Rayan Dasoriya <dasoriya@google.com>
This commit is contained in:
Rayan Dasoriya
co-authored by
Rayan Dasoriya
Rayan Dasoriya
parent
70ebb04f06
commit
cec3f9dd55
@@ -92,16 +92,23 @@
|
||||
"# @title Setup Google Cloud project\n",
|
||||
"# Import the necessary packages\n",
|
||||
"\n",
|
||||
"import json\n",
|
||||
"! git clone https://github.com/GoogleCloudPlatform/vertex-ai-samples.git\n",
|
||||
"\n",
|
||||
"import importlib\n",
|
||||
"import os\n",
|
||||
"import re\n",
|
||||
"from datetime import datetime\n",
|
||||
"from typing import Tuple\n",
|
||||
"\n",
|
||||
"from google.cloud import aiplatform\n",
|
||||
"\n",
|
||||
"common_util = importlib.import_module(\n",
|
||||
" \"vertex-ai-samples.community-content.vertex_model_garden.model_oss.notebook_util.common_util\"\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# @markdown 1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
|
||||
"\n",
|
||||
"# @markdown 2. [Optional] [Create a Cloud Storage bucket](https://cloud.google.com/storage/docs/creating-buckets) for storing experiment outputs. Set the BUCKET_URI for the experiment environment. The specified Cloud Storage bucket (`BUCKET_URI`) should be located in the specified region (`REGION`). Note that a multi-region bucket (eg. \"us\") is not considered a match for a single region covered by the multi-region range (eg. \"us-central1\"). If not set, a unique GCS bucket will be created instead.\n",
|
||||
"# @markdown 2. [Optional] [Create a Cloud Storage bucket](https://cloud.google.com/storage/docs/creating-buckets) for storing experiment outputs. Set the BUCKET_URI for the experiment environment. The specified Cloud Storage bucket (`BUCKET_URI`) should be located in the same region as where the notebook was launched. Note that a multi-region bucket (eg. \"us\") is not considered a match for a single region covered by the multi-region range (eg. \"us-central1\"). If not set, a unique GCS bucket will be created instead.\n",
|
||||
"\n",
|
||||
"# Get the default cloud project id.\n",
|
||||
"PROJECT_ID = os.environ[\"GOOGLE_CLOUD_PROJECT\"]\n",
|
||||
@@ -115,7 +122,7 @@
|
||||
"\n",
|
||||
"# Cloud Storage bucket for storing the experiment artifacts.\n",
|
||||
"# A unique GCS bucket will be created for the purpose of this notebook. If you\n",
|
||||
"# prefer using your own GCS bucket, please change the value yourself below.\n",
|
||||
"# prefer using your own GCS bucket, change the value yourself below.\n",
|
||||
"now = datetime.now().strftime(\"%Y%m%d%H%M%S\")\n",
|
||||
"BUCKET_URI = \"gs://\" # @param {type:\"string\"}\n",
|
||||
"\n",
|
||||
@@ -169,6 +176,12 @@
|
||||
"assert (\n",
|
||||
" VERTEX_AI_MODEL_GARDEN_LLAMA3\n",
|
||||
"), \"Please click the agreement of LLaMA3 in Vertex AI Model Garden, and get the GCS path of LLaMA3 model artifacts.\"\n",
|
||||
"parsed_gcs_url = re.search(\"gs://.*?(?=[ ]|$)\", VERTEX_AI_MODEL_GARDEN_LLAMA3)\n",
|
||||
"if parsed_gcs_url:\n",
|
||||
" VERTEX_AI_MODEL_GARDEN_LLAMA3 = parsed_gcs_url.group()\n",
|
||||
"assert VERTEX_AI_MODEL_GARDEN_LLAMA3.startswith(\n",
|
||||
" \"gs://\"\n",
|
||||
"), \"VERTEX_AI_MODEL_GARDEN_LLAMA3 is expected to be a GCS URI and must start with `gs://`.\"\n",
|
||||
"print(\n",
|
||||
" \"Copying LLaMA3 model artifacts from\",\n",
|
||||
" VERTEX_AI_MODEL_GARDEN_LLAMA3,\n",
|
||||
@@ -182,13 +195,6 @@
|
||||
"VLLM_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-vllm-serve:20240508_0916_RC02\"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def get_job_name_with_datetime(prefix: str) -> str:\n",
|
||||
" \"\"\"Gets the job name with date time when triggering deployment jobs in\n",
|
||||
" Vertex AI.\n",
|
||||
" \"\"\"\n",
|
||||
" return prefix + datetime.now().strftime(\"_%Y%m%d_%H%M%S\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def deploy_model_vllm(\n",
|
||||
" model_name: str,\n",
|
||||
" model_id: str,\n",
|
||||
@@ -237,101 +243,7 @@
|
||||
" )\n",
|
||||
" print(\"endpoint_name:\", endpoint.name)\n",
|
||||
"\n",
|
||||
" return model, endpoint\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"SERVICE_ENDPOINT = \"aiplatform.googleapis.com\"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def get_quota(project_id: str, region: str, resource_id: str) -> int:\n",
|
||||
" \"\"\"Returns the quota for a resource in a region. Returns -1 if can not figure out the quota.\"\"\"\n",
|
||||
" quota_list_output = !gcloud alpha services quota list --service=$SERVICE_ENDPOINT --consumer=projects/$project_id --filter=\"$SERVICE_ENDPOINT/$resource_id\" --format=json\n",
|
||||
" # Use '.s' on the command output because it is an SList type.\n",
|
||||
" quota_data = json.loads(quota_list_output.s)\n",
|
||||
" if len(quota_data) == 0 or \"consumerQuotaLimits\" not in quota_data[0]:\n",
|
||||
" return -1\n",
|
||||
" if (\n",
|
||||
" len(quota_data[0][\"consumerQuotaLimits\"]) == 0\n",
|
||||
" or \"quotaBuckets\" not in quota_data[0][\"consumerQuotaLimits\"][0]\n",
|
||||
" ):\n",
|
||||
" return -1\n",
|
||||
" all_regions_data = quota_data[0][\"consumerQuotaLimits\"][0][\"quotaBuckets\"]\n",
|
||||
" for region_data in all_regions_data:\n",
|
||||
" if (\n",
|
||||
" region_data.get(\"dimensions\")\n",
|
||||
" and region_data[\"dimensions\"][\"region\"] == region\n",
|
||||
" ):\n",
|
||||
" if \"effectiveLimit\" in region_data:\n",
|
||||
" return int(region_data[\"effectiveLimit\"])\n",
|
||||
" else:\n",
|
||||
" return 0\n",
|
||||
" return -1\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def get_resource_id(accelerator_type: str, is_for_training: bool) -> str:\n",
|
||||
" \"\"\"Returns the resource id for a given accelerator type and the use case.\n",
|
||||
" Args:\n",
|
||||
" accelerator_type: The accelerator type.\n",
|
||||
" is_for_training: Whether the resource is used for training. Set false\n",
|
||||
" for serving use case.\n",
|
||||
" Returns:\n",
|
||||
" The resource id.\n",
|
||||
" \"\"\"\n",
|
||||
" training_accelerator_map = {\n",
|
||||
" \"NVIDIA_TESLA_V100\": \"custom_model_training_nvidia_v100_gpus\",\n",
|
||||
" \"NVIDIA_L4\": \"custom_model_training_nvidia_l4_gpus\",\n",
|
||||
" \"NVIDIA_TESLA_A100\": \"custom_model_training_nvidia_a100_gpus\",\n",
|
||||
" }\n",
|
||||
" serving_accelerator_map = {\n",
|
||||
" \"NVIDIA_TESLA_V100\": \"custom_model_serving_nvidia_v100_gpus\",\n",
|
||||
" \"NVIDIA_L4\": \"custom_model_serving_nvidia_l4_gpus\",\n",
|
||||
" \"NVIDIA_TESLA_A100\": \"custom_model_serving_nvidia_a100_gpus\",\n",
|
||||
" }\n",
|
||||
" if is_for_training:\n",
|
||||
" if accelerator_type in training_accelerator_map:\n",
|
||||
" return training_accelerator_map[accelerator_type]\n",
|
||||
" else:\n",
|
||||
" raise ValueError(\n",
|
||||
" f\"Could not find accelerator type: {accelerator_type} for training.\"\n",
|
||||
" )\n",
|
||||
" else:\n",
|
||||
" if accelerator_type in serving_accelerator_map:\n",
|
||||
" return serving_accelerator_map[accelerator_type]\n",
|
||||
" else:\n",
|
||||
" raise ValueError(\n",
|
||||
" f\"Could not find accelerator type: {accelerator_type} for serving.\"\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def check_quota(\n",
|
||||
" project_id: str,\n",
|
||||
" region: str,\n",
|
||||
" accelerator_type: str,\n",
|
||||
" accelerator_count: int,\n",
|
||||
" is_for_training: bool,\n",
|
||||
"):\n",
|
||||
" \"\"\"Checks if the project and the region has the required quota.\"\"\"\n",
|
||||
" resource_id = get_resource_id(accelerator_type, is_for_training)\n",
|
||||
" quota = get_quota(project_id, region, resource_id)\n",
|
||||
" quota_request_instruction = (\n",
|
||||
" \"Either use \"\n",
|
||||
" \"a different region or request additional quota. Follow \"\n",
|
||||
" \"instructions here \"\n",
|
||||
" \"https://cloud.google.com/docs/quotas/view-manage#requesting_higher_quota\"\n",
|
||||
" \" to check quota in a region or request additional quota for \"\n",
|
||||
" \"your project.\"\n",
|
||||
" )\n",
|
||||
" if quota == -1:\n",
|
||||
" raise ValueError(\n",
|
||||
" f\"\"\"Quota not found for: {resource_id} in {region}.\n",
|
||||
" {quota_request_instruction}\"\"\"\n",
|
||||
" )\n",
|
||||
" if quota < accelerator_count:\n",
|
||||
" raise ValueError(\n",
|
||||
" f\"\"\"Quota not enough for {resource_id} in {region}:\n",
|
||||
" {quota} < {accelerator_count}.\n",
|
||||
" {quota_request_instruction}\"\"\"\n",
|
||||
" )"
|
||||
" return model, endpoint\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -356,9 +268,10 @@
|
||||
"\n",
|
||||
"# @markdown This section uploads prebuilt LLaMA3 models to Model Registry and deploys it to a Vertex AI Endpoint. It takes 15 minutes to 1 hour to finish depending on the size of the model.\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# @markdown NVIDIA_L4 GPUs are used for demonstration. The serving efficiency of L4 GPUs is inferior to that of A100 GPUs, but L4 GPUs are nevertheless good serving solutions if you do not have A100 quota.\n",
|
||||
"\n",
|
||||
"# @markdown Llama 3 uses a context length of 8,192 tokens, double the context length of Llama 2. Please see this [Meta blog post](https://ai.meta.com/blog/meta-llama-3/) for more details.\n",
|
||||
"\n",
|
||||
"# @markdown Set the model to deploy.\n",
|
||||
"\n",
|
||||
"base_model_name = \"llama3-8b-chat-hf\" # @param [\"llama3-8b-hf\", \"llama3-8b-chat-hf\", \"llama3-70b-hf\", \"llama3-70b-chat-hf\"] {isTemplate:true}\n",
|
||||
@@ -401,7 +314,7 @@
|
||||
" raise ValueError(\n",
|
||||
" f\"Recommended GPU setting not found for: {accelerator_type} and {base_model_name}.\"\n",
|
||||
" )\n",
|
||||
"check_quota(\n",
|
||||
"common_util.check_quota(\n",
|
||||
" project_id=PROJECT_ID,\n",
|
||||
" region=REGION,\n",
|
||||
" accelerator_type=accelerator_type,\n",
|
||||
@@ -412,8 +325,12 @@
|
||||
"gpu_memory_utilization = 0.85\n",
|
||||
"max_model_len = 8192 # Maximum context length.\n",
|
||||
"\n",
|
||||
"# Ensure max_model_len does not exceed the limit\n",
|
||||
"if max_model_len > 8192:\n",
|
||||
" raise ValueError(\"max_model_len cannot exceed 8192\")\n",
|
||||
"\n",
|
||||
"model, endpoint = deploy_model_vllm(\n",
|
||||
" model_name=get_job_name_with_datetime(prefix=\"llama3-serve\"),\n",
|
||||
" model_name=common_util.get_job_name_with_datetime(prefix=\"llama3-serve\"),\n",
|
||||
" model_id=model_id,\n",
|
||||
" service_account=SERVICE_ACCOUNT,\n",
|
||||
" machine_type=machine_type,\n",
|
||||
@@ -436,7 +353,7 @@
|
||||
"source": [
|
||||
"# @title Predict\n",
|
||||
"\n",
|
||||
"# @markdown Once deployment succeeds, you can send requests to the endpoint with text prompts. Sampling parameters supported by vLLM can be found [here](https://github.com/vllm-project/vllm/blob/2e8e49fce3775e7704d413b2f02da6d7c99525c9/vllm/sampling_params.py#L23-L64).\n",
|
||||
"# @markdown Once deployment succeeds, you can send requests to the endpoint with text prompts. Sampling parameters supported by vLLM can be found [here](https://docs.vllm.ai/en/latest/dev/sampling_params.html).\n",
|
||||
"\n",
|
||||
"# @markdown Example:\n",
|
||||
"\n",
|
||||
|
||||
Reference in New Issue
Block a user