xiaohei/model_garden
1
0
Fork 0
mirror of https://github.com/GoogleCloudPlatform/vertex-ai-samples.git synced 2026-09-26 14:42:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update the notebook to use the Vertex SDK to send requests to the deployed endpoint instead of openai SDK.

Browse Source 
PiperOrigin-RevId: 685713862
This commit is contained in:
Dustin Luong
2024-10-14 08:23:23 -07:00
committed by Copybara-Service
parent eb166ed3df
commit a1d4026020
1 changed files with 34 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files
notebooks/community/model_garden/model_garden_pytorch_llama3_2_deployment.ipynb
+34 -65
View File
@@ -988,46 +988,30 @@
"source": [
"# @title Chat completion for text-only models\n",
"\n",
"ENDPOINT_RESOURCE_NAME = \"projects/{}/locations/{}/endpoints/{}\".format(\n",
" PROJECT_ID, REGION, endpoints[\"ref_gpu\"].name\n",
")\n",
"\n",
"# @title Chat Completions Inference\n",
"\n",
"# @markdown Once deployment succeeds, you can send requests to the endpoint using the OpenAI SDK.\n",
"\n",
"# @markdown First you will need to install the SDK and some auth-related dependencies.\n",
"\n",
"! pip install -qU openai google-auth requests\n",
"# @markdown Once deployment succeeds, you can send requests to the endpoint using the Vertex SDK.\n",
"\n",
"# @markdown Next fill out some request parameters:\n",
"\n",
"user_message = \"How is your day going?\" # @param {type: \"string\"}\n",
"user_message = \"Tell me about large language models\" # @param {type: \"string\"}\n",
"# @markdown If you encounter the issue like `ServiceUnavailable: 503 Took too long to respond when processing`, you can reduce the maximum number of output tokens, such as set `max_tokens` as 20.\n",
"max_tokens = 50 # @param {type: \"integer\"}\n",
"temperature = 1.0 # @param {type: \"number\"}\n",
"top_p = 0.9 # @param {type: \"number\"}\n",
"\n",
"# @markdown Now we can send a request.\n",
"\n",
"import google.auth\n",
"import openai\n",
"instances = [\n",
" {\n",
" \"messages\": [{\"role\": \"user\", \"content\": user_message}],\n",
" \"temperature\": temperature,\n",
" \"max_tokens\": max_tokens,\n",
" \"top_p\": top_p,\n",
" }\n",
"]\n",
"\n",
"creds, project = google.auth.default()\n",
"auth_req = google.auth.transport.requests.Request()\n",
"creds.refresh(auth_req)\n",
"model_response = endpoints[\"ref_gpu\"].predict(instances=instances)\n",
"\n",
"BASE_URL = (\n",
" f\"https://{REGION}-aiplatform.googleapis.com/v1beta1/{ENDPOINT_RESOURCE_NAME}\"\n",
")\n",
"client = openai.OpenAI(base_url=BASE_URL, api_key=creds.token)\n",
"\n",
"model_response = client.chat.completions.create(\n",
" model=\"\",\n",
" messages=[{\"role\": \"user\", \"content\": user_message}],\n",
" temperature=temperature,\n",
" max_tokens=max_tokens,\n",
")\n",
"print(model_response)\n",
"print(model_response.predictions[0])\n",
"\n",
"# @markdown Click \"Show Code\" to see more details."
]
@@ -1043,17 +1027,8 @@
"source": [
"# @title Chat completion for vision models\n",
"\n",
"ENDPOINT_RESOURCE_NAME = \"projects/{}/locations/{}/endpoints/{}\".format(\n",
" PROJECT_ID, REGION, endpoints[\"ref_gpu\"].name\n",
")\n",
"# @markdown Once deployment succeeds, you can send requests to the endpoint using the Vertex SDK.\n",
"\n",
"# @title Chat Completions Inference\n",
"\n",
"# @markdown Once deployment succeeds, you can send requests to the endpoint using the OpenAI SDK.\n",
"\n",
"# @markdown First you will need to install the SDK and some auth-related dependencies.\n",
"\n",
"! pip install -qU openai google-auth requests\n",
"# @markdown Next fill out some request parameters:\n",
"\n",
"user_image1 = \"https://upload.wikimedia.org/wikipedia/commons/thumb/8/84/Male_and_female_chicken_sitting_together.jpg/440px-Male_and_female_chicken_sitting_together.jpg\" # @param {type: \"string\"}\n",
@@ -1062,37 +1037,31 @@
"# @markdown If you encounter the issue like `ServiceUnavailable: 503 Took too long to respond when processing`, you can reduce the maximum number of output tokens, such as set `max_tokens` as 20.\n",
"max_tokens = 50 # @param {type: \"integer\"}\n",
"temperature = 1.0 # @param {type: \"number\"}\n",
"top_p = 0.9 # @param {type: \"number\"}\n",
"\n",
"# @markdown Now we can send a request.\n",
"\n",
"import google.auth\n",
"import openai\n",
"instances = [\n",
" {\n",
" \"messages\": [\n",
" {\n",
" \"role\": \"user\",\n",
" \"content\": [\n",
" {\"type\": \"image_url\", \"image_url\": {\"url\": user_image1}},\n",
" {\"type\": \"image_url\", \"image_url\": {\"url\": user_image2}},\n",
" {\"type\": \"text\", \"text\": user_message},\n",
" ],\n",
" },\n",
" ],\n",
" \"temperature\": temperature,\n",
" \"max_tokens\": max_tokens,\n",
" \"top_p\": top_p,\n",
" }\n",
"]\n",
"\n",
"creds, project = google.auth.default()\n",
"auth_req = google.auth.transport.requests.Request()\n",
"creds.refresh(auth_req)\n",
"model_response = endpoints[\"ref_gpu\"].predict(instances=instances)\n",
"\n",
"BASE_URL = (\n",
" f\"https://{REGION}-aiplatform.googleapis.com/v1beta1/{ENDPOINT_RESOURCE_NAME}\"\n",
")\n",
"client = openai.OpenAI(base_url=BASE_URL, api_key=creds.token)\n",
"\n",
"model_response = client.chat.completions.create(\n",
" model=\"\",\n",
" messages=[\n",
" {\n",
" \"role\": \"user\",\n",
" \"content\": [\n",
" {\"type\": \"image_url\", \"image_url\": {\"url\": user_image1}},\n",
" {\"type\": \"image_url\", \"image_url\": {\"url\": user_image2}},\n",
" {\"type\": \"text\", \"text\": user_message},\n",
" ],\n",
" }\n",
" ],\n",
" temperature=temperature,\n",
" max_tokens=max_tokens,\n",
")\n",
"print(model_response)\n",
"print(model_response.predictions[0])\n",
"\n",
"# @markdown Click \"Show Code\" to see more details."
]