xiaohei/model_garden
1
0
Fork 0
mirror of https://github.com/GoogleCloudPlatform/vertex-ai-samples.git synced 2026-09-26 14:42:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

request timeout set to 30 minutes

Browse Source 
PiperOrigin-RevId: 781362956
This commit is contained in:
Vertex MG Team
2025-07-09 22:10:38 -07:00
committed by Copybara-Service
parent f5e9d7a9ed
commit b77fb73ad0
1 changed files with 257 additions and 254 deletions
Download Patch File Download Diff File Expand all files Collapse all files
notebooks/community/model_garden/model_garden_integration_with_agent.ipynb
+257 -254
View File
@@ -32,18 +32,18 @@
"source": [
"# Vertex AI Model Garden Integration With Agents\n",
"\n",
"\u003ctable\u003e\u003ctbody\u003e\u003ctr\u003e\n",
" \u003ctd style=\"text-align: center\"\u003e\n",
" \u003ca href=\"https://console.cloud.google.com/vertex-ai/colab/import/https:%2F%2Fraw.githubusercontent.com%2FGoogleCloudPlatform%2Fvertex-ai-samples%2Fmain%2Fnotebooks%2Fcommunity%2Fmodel_garden%2Fmodel_garden_integration_with_agent.ipynb\"\u003e\n",
" \u003cimg alt=\"Google Cloud Colab Enterprise logo\" src=\"https://lh3.googleusercontent.com/JmcxdQi-qOpctIvWKgPtrzZdJJK-J3sWE1RsfjZNwshCFgE_9fULcNpuXYTilIR2hjwN\" width=\"32px\"\u003e\u003cbr\u003e Run in Colab Enterprise\n",
" \u003c/a\u003e\n",
" \u003c/td\u003e\n",
" \u003ctd style=\"text-align: center\"\u003e\n",
" \u003ca href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_integration_with_agent.ipynb\"\u003e\n",
" \u003cimg alt=\"GitHub logo\" src=\"https://github.githubassets.com/assets/GitHub-Mark-ea2971cee799.png\" width=\"32px\"\u003e\u003cbr\u003e View on GitHub\n",
" \u003c/a\u003e\n",
" \u003c/td\u003e\n",
"\u003c/tr\u003e\u003c/tbody\u003e\u003c/table\u003e"
"<table><tbody><tr>\n",
" <td style=\"text-align: center\">\n",
" <a href=\"https://console.cloud.google.com/vertex-ai/colab/import/https:%2F%2Fraw.githubusercontent.com%2FGoogleCloudPlatform%2Fvertex-ai-samples%2Fmain%2Fnotebooks%2Fcommunity%2Fmodel_garden%2Fmodel_garden_integration_with_agent.ipynb\">\n",
" <img alt=\"Google Cloud Colab Enterprise logo\" src=\"https://lh3.googleusercontent.com/JmcxdQi-qOpctIvWKgPtrzZdJJK-J3sWE1RsfjZNwshCFgE_9fULcNpuXYTilIR2hjwN\" width=\"32px\"><br> Run in Colab Enterprise\n",
" </a>\n",
" </td>\n",
" <td style=\"text-align: center\">\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_integration_with_agent.ipynb\">\n",
" <img alt=\"GitHub logo\" src=\"https://github.githubassets.com/assets/GitHub-Mark-ea2971cee799.png\" width=\"32px\"><br> View on GitHub\n",
" </a>\n",
" </td>\n",
"</tr></tbody></table>"
]
},
{
@@ -87,8 +87,8 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "J_jmxcIZoSxU",
"cellView": "form"
"cellView": "form",
"id": "J_jmxcIZoSxU"
},
"outputs": [],
"source": [
@@ -106,7 +106,7 @@
"\n",
"# @markdown 4. If you want to run predictions with A100 80GB or H100 GPUs, we recommend using the regions listed below. **NOTE:** Make sure you have associated quota in selected regions. Click the links to see your current quota for each GPU type: [Nvidia A100 80GB](https://console.cloud.google.com/iam-admin/quotas?metric=aiplatform.googleapis.com%2Fcustom_model_serving_nvidia_a100_80gb_gpus), [Nvidia H100 80GB](https://console.cloud.google.com/iam-admin/quotas?metric=aiplatform.googleapis.com%2Fcustom_model_serving_nvidia_h100_gpus). You can request for quota following the instructions at [\"Request a higher quota\"](https://cloud.google.com/docs/quota/view-manage#requesting_higher_quota).\n",
"\n",
"# @markdown \u003e | Machine Type | Accelerator Type | Recommended Regions |\n",
"# @markdown > | Machine Type | Accelerator Type | Recommended Regions |\n",
"# @markdown | ----------- | ----------- | ----------- |\n",
"# @markdown | a2-ultragpu-1g | 1 NVIDIA_A100_80GB | us-central1, us-east4, europe-west4, asia-southeast1, us-east4 |\n",
"# @markdown | a3-highgpu-2g | 2 NVIDIA_H100_80GB | us-west1, asia-southeast1, europe-west4 |\n",
@@ -114,7 +114,7 @@
"# @markdown | a3-highgpu-8g | 8 NVIDIA_H100_80GB | us-central1, europe-west4, us-west1, asia-southeast1 |\n",
"\n",
"# Upgrade Vertex AI SDK.\n",
"! pip3 install --upgrade --quiet 'google-cloud-aiplatform\u003e=1.84.0'\n",
"! pip3 install --upgrade --quiet 'google-cloud-aiplatform>=1.84.0'\n",
"! pip3 install -qU openai google-auth requests\n",
"! pip3 install --upgrade --quiet \\\n",
" \"google-cloud-aiplatform[agent_engines,langchain]\" \\\n",
@@ -236,7 +236,7 @@
"from langchain_core.tools import tool\n",
"\n",
"@tool\n",
"def get_weather(city: str) -\u003e str:\n",
"def get_weather(city: str) -> str:\n",
" \"\"\"Simulates a web search. Use it get information on weather.\n",
"\n",
" Args:\n",
@@ -251,7 +251,7 @@
"\n",
"\n",
"@tool\n",
"def get_current_time(city: str) -\u003e str:\n",
"def get_current_time(city: str) -> str:\n",
" \"\"\"Simulates getting the current time for a city.\n",
"\n",
" Args:\n",
@@ -322,7 +322,7 @@
" )\n",
"\n",
"\n",
"def call_tools(model_output: str = None) -\u003e List[Any]:\n",
"def call_tools(model_output: str = None) -> List[Any]:\n",
" \"\"\"Execute the tool calls.\"\"\"\n",
" if not model_output:\n",
" return []\n",
@@ -351,20 +351,26 @@
" except Exception as ex: # pylint: disable=broad-except\n",
" print(str(ex))\n",
"\n",
" return format_tool_calls\n"
" return format_tool_calls"
]
},
{
"cell_type": "markdown",
"source": [
"## Deploy And Test OSS LLM Models For Agents"
],
"metadata": {
"id": "de6hEdbjNvWR"
}
},
"source": [
"## Deploy And Test OSS LLM Models For Agents"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "gbfFZLT5KkUV"
},
"outputs": [],
"source": [
"# @title Deploy OSS LLMs\n",
"# @markdown This section will show how to deploy OSS LLMs properly for agent integration.\n",
@@ -389,118 +395,120 @@
"# @markdown For simplicity, we deploy models with traditional endpoints by default here.\n",
"\n",
"MODEL_ID = \"Qwen3-32B\" # @param [\"Qwen3-32B\", \"llama-4-scout-17b-16e-instruct\", \"llama-3.3-70b-instruct\", \"DeepSeek-R1-Distill-Llama-70B\", \"gemma-3-27b-it\"] {isTemplate: true}\n",
"accelerator_type = \"NVIDIA_H100_80GB\" # @param [\"NVIDIA_L4\", \"NVIDIA_H100_80GB\"] {isTemplate: true}\n",
"accelerator_type = (\n",
" \"NVIDIA_H100_80GB\" # @param [\"NVIDIA_L4\", \"NVIDIA_H100_80GB\"] {isTemplate: true}\n",
")\n",
"\n",
"if \"Qwen\" in MODEL_ID:\n",
" publisher_model_name = f\"publishers/qwen/models/qwen3@{MODEL_ID.lower()}\"\n",
" publisher_model_name = f\"publishers/qwen/models/qwen3@{MODEL_ID.lower()}\"\n",
"\n",
" if accelerator_type == \"NVIDIA_L4\":\n",
" accelerator_count = 4\n",
" # Sets machine type to g2-standard-48 for 4 L4's\n",
" machine_type = \"g2-standard-48\"\n",
" elif accelerator_type == \"NVIDIA_H100_80GB\":\n",
" accelerator_count = 2\n",
" machine_type = \"a3-highgpu-2g\"\n",
" else:\n",
" raise ValueError(\n",
" \"Recommended machine settings not found for accelerator type: %s\"\n",
" % accelerator_type\n",
" )\n",
" if accelerator_type == \"NVIDIA_L4\":\n",
" accelerator_count = 4\n",
" # Sets machine type to g2-standard-48 for 4 L4's\n",
" machine_type = \"g2-standard-48\"\n",
" elif accelerator_type == \"NVIDIA_H100_80GB\":\n",
" accelerator_count = 2\n",
" machine_type = \"a3-highgpu-2g\"\n",
" else:\n",
" raise ValueError(\n",
" \"Recommended machine settings not found for accelerator type: %s\"\n",
" % accelerator_type\n",
" )\n",
"elif \"llama-4\" in MODEL_ID:\n",
" publisher_model_name = f\"publishers/meta/models/llama4@{MODEL_ID.lower()}\"\n",
" if accelerator_type == \"NVIDIA_H100_80GB\":\n",
" accelerator_count = 8\n",
" machine_type = \"a3-highgpu-8g\"\n",
" else:\n",
" raise ValueError(\n",
" \"Recommended machine settings not found for accelerator type: %s\"\n",
" % accelerator_type\n",
" )\n",
" publisher_model_name = f\"publishers/meta/models/llama4@{MODEL_ID.lower()}\"\n",
" if accelerator_type == \"NVIDIA_H100_80GB\":\n",
" accelerator_count = 8\n",
" machine_type = \"a3-highgpu-8g\"\n",
" else:\n",
" raise ValueError(\n",
" \"Recommended machine settings not found for accelerator type: %s\"\n",
" % accelerator_type\n",
" )\n",
"elif \"llama-3\" in MODEL_ID:\n",
" publisher_model_name = f\"publishers/meta/models/llama3-3@{MODEL_ID.lower()}\"\n",
" if accelerator_type == \"NVIDIA_H100_80GB\":\n",
" accelerator_count = 4\n",
" machine_type = \"a3-highgpu-4g\"\n",
" else:\n",
" raise ValueError(\n",
" \"Recommended machine settings not found for accelerator type: %s\"\n",
" % accelerator_type\n",
" )\n",
" publisher_model_name = f\"publishers/meta/models/llama3-3@{MODEL_ID.lower()}\"\n",
" if accelerator_type == \"NVIDIA_H100_80GB\":\n",
" accelerator_count = 4\n",
" machine_type = \"a3-highgpu-4g\"\n",
" else:\n",
" raise ValueError(\n",
" \"Recommended machine settings not found for accelerator type: %s\"\n",
" % accelerator_type\n",
" )\n",
"elif \"DeepSeek-R1\" in MODEL_ID:\n",
" publisher_model_name = (\n",
" f\"publishers/deepseek-ai/models/deepseek-r1@{MODEL_ID.lower()}\"\n",
" )\n",
" if accelerator_type == \"NVIDIA_L4\":\n",
" accelerator_count = 8\n",
" # Sets machine type to g2-standard-96 for 8 L4's\n",
" machine_type = \"g2-standard-96\"\n",
" elif accelerator_type == \"NVIDIA_H100_80GB\":\n",
" accelerator_count = 4\n",
" machine_type = \"a3-highgpu-4g\"\n",
" else:\n",
" raise ValueError(\n",
" \"Recommended machine settings not found for accelerator type: %s\"\n",
" % accelerator_type\n",
" publisher_model_name = (\n",
" f\"publishers/deepseek-ai/models/deepseek-r1@{MODEL_ID.lower()}\"\n",
" )\n",
" if accelerator_type == \"NVIDIA_L4\":\n",
" accelerator_count = 8\n",
" # Sets machine type to g2-standard-96 for 8 L4's\n",
" machine_type = \"g2-standard-96\"\n",
" elif accelerator_type == \"NVIDIA_H100_80GB\":\n",
" accelerator_count = 4\n",
" machine_type = \"a3-highgpu-4g\"\n",
" else:\n",
" raise ValueError(\n",
" \"Recommended machine settings not found for accelerator type: %s\"\n",
" % accelerator_type\n",
" )\n",
"elif \"gemma-3\" in MODEL_ID:\n",
" publisher_model_name = f\"publishers/google/models/gemma3@{MODEL_ID.lower()}\"\n",
" if accelerator_type == \"NVIDIA_L4\":\n",
" accelerator_count = 4\n",
" # Sets machine type to g2-standard-48 for 4 L4's\n",
" machine_type = \"g2-standard-48\"\n",
" elif accelerator_type == \"NVIDIA_H100_80GB\":\n",
" accelerator_count = 2\n",
" machine_type = \"a3-highgpu-2g\"\n",
" else:\n",
" raise ValueError(\n",
" \"Recommended machine settings not found for accelerator type: %s\"\n",
" % accelerator_type\n",
" )\n",
" publisher_model_name = f\"publishers/google/models/gemma3@{MODEL_ID.lower()}\"\n",
" if accelerator_type == \"NVIDIA_L4\":\n",
" accelerator_count = 4\n",
" # Sets machine type to g2-standard-48 for 4 L4's\n",
" machine_type = \"g2-standard-48\"\n",
" elif accelerator_type == \"NVIDIA_H100_80GB\":\n",
" accelerator_count = 2\n",
" machine_type = \"a3-highgpu-2g\"\n",
" else:\n",
" raise ValueError(\n",
" \"Recommended machine settings not found for accelerator type: %s\"\n",
" % accelerator_type\n",
" )\n",
"else:\n",
" raise ValueError(\"Unsupported model: %s\" % MODEL_ID)\n",
" raise ValueError(\"Unsupported model: %s\" % MODEL_ID)\n",
"\n",
"from vertexai.preview import model_garden\n",
"\n",
"model = model_garden.OpenModel(publisher_model_name)\n",
"\n",
"if \"Qwen3-32B\" == MODEL_ID:\n",
" container_spec = model.list_deploy_options()[0].container_spec\n",
" updated_args = container_spec.args[:-2] + [\n",
" f\"--tp={accelerator_count}\",\n",
" \"--tool-call-parser=qwen25\",\n",
" ]\n",
" container_spec.args = updated_args\n",
" accept_eula = False\n",
" container_spec = model.list_deploy_options()[0].container_spec\n",
" updated_args = container_spec.args[:-2] + [\n",
" f\"--tp={accelerator_count}\",\n",
" \"--tool-call-parser=qwen25\",\n",
" ]\n",
" container_spec.args = updated_args\n",
" accept_eula = False\n",
"elif \"llama-4-scout-17b-16e-instruct\" == MODEL_ID:\n",
" container_spec = model.list_deploy_options()[1].container_spec\n",
" container_spec.image_uri = \"us-docker.pkg.dev/deeplearning-platform-release/vertex-model-garden/sglang-serve.cu124.0-4.ubuntu2204.py310:20250515-1800-rc0\"\n",
" updated_args = container_spec.args[:-1] + [\n",
" f\"--tp={accelerator_count}\",\n",
" \"--tool-call-parser=pythonic\",\n",
" ]\n",
" container_spec.args = updated_args\n",
" accept_eula = True\n",
" container_spec = model.list_deploy_options()[1].container_spec\n",
" container_spec.image_uri = \"us-docker.pkg.dev/deeplearning-platform-release/vertex-model-garden/sglang-serve.cu124.0-4.ubuntu2204.py310:20250515-1800-rc0\"\n",
" updated_args = container_spec.args[:-1] + [\n",
" f\"--tp={accelerator_count}\",\n",
" \"--tool-call-parser=pythonic\",\n",
" ]\n",
" container_spec.args = updated_args\n",
" accept_eula = True\n",
"elif \"llama-3.3-70b-instruct\" == MODEL_ID:\n",
" container_spec = model.list_deploy_options()[0].container_spec\n",
" accept_eula = True\n",
" container_spec = model.list_deploy_options()[0].container_spec\n",
" accept_eula = True\n",
"elif \"DeepSeek-R1\" in MODEL_ID:\n",
" container_spec = model.list_deploy_options()[0].container_spec\n",
" updated_args = container_spec.args[:-1] + [f\"--tp={accelerator_count}\"]\n",
" container_spec.args = updated_args\n",
" accept_eula = False\n",
" container_spec = model.list_deploy_options()[0].container_spec\n",
" updated_args = container_spec.args[:-1] + [f\"--tp={accelerator_count}\"]\n",
" container_spec.args = updated_args\n",
" accept_eula = False\n",
"elif \"gemma-3-27b-it\" == MODEL_ID:\n",
" container_spec = model.list_deploy_options()[0].container_spec\n",
" updated_args = []\n",
" for arg in container_spec.args:\n",
" if arg.startswith(\"--tensor-parallel-size\"):\n",
" updated_args.append(f\"--tensor-parallel-size={accelerator_count}\")\n",
" else:\n",
" updated_args.append(arg)\n",
" container_spec.args = updated_args\n",
" container_spec = model.list_deploy_options()[0].container_spec\n",
" updated_args = []\n",
" for arg in container_spec.args:\n",
" if arg.startswith(\"--tensor-parallel-size\"):\n",
" updated_args.append(f\"--tensor-parallel-size={accelerator_count}\")\n",
" else:\n",
" updated_args.append(arg)\n",
" container_spec.args = updated_args\n",
"\n",
" accept_eula = True\n",
" accept_eula = True\n",
"else:\n",
" raise ValueError(\"Unsupported model: %s\" % MODEL_ID)\n",
" raise ValueError(\"Unsupported model: %s\" % MODEL_ID)\n",
"\n",
"print(\"The container spec are:\")\n",
"print(container_spec)\n",
@@ -523,7 +531,7 @@
" accelerator_count=accelerator_count,\n",
" use_dedicated_endpoint=False,\n",
" spot=False,\n",
" deploy_request_timeout=deploy_request_timeout,\n",
" deploy_request_timeout=1800,\n",
" accept_eula=False,\n",
")\n",
"print(\"Finished to deploy models to endpoints.\")\n",
@@ -536,16 +544,16 @@
"print(\"The deployed endpoint resource name is:\")\n",
"print(endpoint_resource_name)\n",
"# @markdown Click \"Show Code\" to see more details."
],
"metadata": {
"id": "gbfFZLT5KkUV",
"cellView": "form"
},
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "_MNVrfJomCd0"
},
"outputs": [],
"source": [
"# @title Test The Endpoint\n",
"# endpoint_resource_name = (\n",
@@ -555,9 +563,11 @@
"\n",
"location = endpoint_resource_name.split(\"/\")[3]\n",
"if endpoint.gca_resource.dedicated_endpoint_enabled:\n",
" base_url = f\"https://{endpoint.gca_resource.dedicated_endpoint_dns}/v1beta1/{endpoint.resource_name}\"\n",
" base_url = f\"https://{endpoint.gca_resource.dedicated_endpoint_dns}/v1beta1/{endpoint.resource_name}\"\n",
"else:\n",
" base_url = f\"https://{location}-aiplatform.googleapis.com/v1beta1/{endpoint.resource_name}\"\n",
" base_url = (\n",
" f\"https://{location}-aiplatform.googleapis.com/v1beta1/{endpoint.resource_name}\"\n",
" )\n",
"\n",
"# @markdown Predict locally with some requests.\n",
"\n",
@@ -585,40 +595,34 @@
")\n",
"\n",
"if stream:\n",
" usage = None\n",
" contents = []\n",
" for chunk in model_response:\n",
" if chunk.usage is not None:\n",
" usage = chunk.usage\n",
" continue\n",
" print(chunk.choices[0].delta.content, end=\"\")\n",
" contents.append(chunk.choices[0].delta.content)\n",
" print(f\"\\n\\n{usage}\")\n",
" usage = None\n",
" contents = []\n",
" for chunk in model_response:\n",
" if chunk.usage is not None:\n",
" usage = chunk.usage\n",
" continue\n",
" print(chunk.choices[0].delta.content, end=\"\")\n",
" contents.append(chunk.choices[0].delta.content)\n",
" print(f\"\\n\\n{usage}\")\n",
"else:\n",
" print(model_response.choices[0].message.content)"
],
"metadata": {
"id": "_MNVrfJomCd0",
"cellView": "form"
},
"execution_count": null,
"outputs": []
" print(model_response.choices[0].message.content)"
]
},
{
"cell_type": "markdown",
"source": [
"## Integrate OSS LLMs With ADK"
],
"metadata": {
"id": "T9UiFwbiowOV"
}
},
"source": [
"## Integrate OSS LLMs With ADK"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "Aa4e1-6FvRAP",
"cellView": "form"
"cellView": "form",
"id": "Aa4e1-6FvRAP"
},
"outputs": [],
"source": [
@@ -629,7 +633,7 @@
"# @markdown tools, then the integration with ADK can work properly.\n",
"\n",
"# @markdown 1. Create `agent.py` by loading VMG endpoints and example tool functions.\n",
"agent_app = '''\n",
"agent_app = r'''\n",
"\"\"\"This is a sample agent for model garden agents.\"\"\"\n",
"\n",
"import datetime\n",
@@ -644,7 +648,7 @@
"_MODEL_GARDEN_ENDPOINT_REGEX = r\"projects\\/.+\\/locations\\/.+\\/endpoints\\/.+\"\n",
"\n",
"\n",
"def get_weather(city: str) -\u003e str:\n",
"def get_weather(city: str) -> str:\n",
" \"\"\"Simulates a web search. Use it get information on weather.\n",
"\n",
" Args:\n",
@@ -658,7 +662,7 @@
" return \"It's 80 degrees and sunny.\"\n",
"\n",
"\n",
"def get_current_time(city: str) -\u003e str:\n",
"def get_current_time(city: str) -> str:\n",
" \"\"\"Simulates getting the current time for a city.\n",
"\n",
" Args:\n",
@@ -680,7 +684,7 @@
" )\n",
"\n",
"\n",
"def _get_auth_headers() -\u003e dict[str, str]:\n",
"def _get_auth_headers() -> dict[str, str]:\n",
" \"\"\"Gets the auth headers for the model garden endpoint.\"\"\"\n",
" creds, _ = google.auth.default(\n",
" scopes=[\"https://www.googleapis.com/auth/cloud-platform\"]\n",
@@ -731,14 +735,14 @@
")\n",
"'''\n",
"with BUILD_DIR.joinpath(\"agent.py\").open(\"w\") as f:\n",
" f.write(agent_app)\n",
" f.write(agent_app)\n",
"\n",
"# @markdown 2. Create `__init__.py` to load agent.py for ADK apps.\n",
"initialize = \"\"\"\n",
"from . import agent\n",
"\"\"\"\n",
"with BUILD_DIR.joinpath(\"__init__.py\").open(\"w\") as f:\n",
" f.write(initialize)\n",
" f.write(initialize)\n",
"\n",
"# @markdown 3. Create `Dockerfile` to build agent app dockers.\n",
"dockerfile_content = \"\"\"\n",
@@ -762,7 +766,7 @@
"CMD adk web --port=8000 --trace_to_cloud \"/app/agents\"\n",
"\"\"\"\n",
"with BUILD_DIR.joinpath(\"Dockerfile\").open(\"w\") as f:\n",
" f.write(dockerfile_content)\n",
" f.write(dockerfile_content)\n",
"\n",
"# @markdown 4. Build agent web app dockers.\n",
"VMG_AGENT_UI_CONTAINER_IMAGE_URI = (\n",
@@ -776,6 +780,12 @@
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "L2AGmQZmuVam"
},
"outputs": [],
"source": [
"# @title Deploy Agent Web App Dockers To Cloud Run\n",
"\n",
@@ -788,16 +798,16 @@
" --memory=1024Mi \\\n",
" --set-env-vars=\"GOOGLE_MODEL_GARDEN_ENDPOINT={endpoint_resource_name}\"\n",
"# @markdown Click \"Show Code\" to see more details."
],
"metadata": {
"id": "L2AGmQZmuVam",
"cellView": "form"
},
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "d5WGF_jG0CEn"
},
"outputs": [],
"source": [
"# @title Test With Web App\n",
"# @markdown After the deployment, there will be a service URL. You can click\n",
@@ -806,25 +816,25 @@
"# @markdown to help you test, evaluate, debug, and showcase your agent(s).\n",
"\n",
"# @markdown ![ADK WEB UI](https://raw.githubusercontent.com/google/adk-python/main/assets/adk-web-dev-ui-function-call.png)"
],
"metadata": {
"cellView": "form",
"id": "d5WGF_jG0CEn"
},
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "markdown",
"source": [
"## Integrate OSS LLMs With Agent Engine"
],
"metadata": {
"id": "ZI9NZvpPQcgN"
}
},
"source": [
"## Integrate OSS LLMs With Agent Engine"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "8A_DhwDrQbOW"
},
"outputs": [],
"source": [
"# @title Build Agents With Endpoints And Agent Engine\n",
"\n",
@@ -840,9 +850,11 @@
"endpoint = aiplatform.Endpoint(endpoint_resource_name)\n",
"location = endpoint_resource_name.split(\"/\")[3]\n",
"if endpoint.gca_resource.dedicated_endpoint_enabled:\n",
" base_url = f\"https://{endpoint.gca_resource.dedicated_endpoint_dns}/v1beta1/{endpoint.resource_name}\"\n",
" base_url = f\"https://{endpoint.gca_resource.dedicated_endpoint_dns}/v1beta1/{endpoint.resource_name}\"\n",
"else:\n",
" base_url = f\"https://{location}-aiplatform.googleapis.com/v1beta1/{endpoint.resource_name}\"\n",
" base_url = (\n",
" f\"https://{location}-aiplatform.googleapis.com/v1beta1/{endpoint.resource_name}\"\n",
" )\n",
"\n",
"\n",
"def model_builder(\n",
@@ -853,23 +865,23 @@
" location: str, # Specified via vertexai.init\n",
" **kwargs,\n",
"):\n",
" # Note: the credential expires after 1 hour by default.\n",
" # After expiration, it must be refreshed.\n",
" creds, _ = google.auth.default(\n",
" scopes=[\"https://www.googleapis.com/auth/cloud-platform\"]\n",
" )\n",
" auth_req = google.auth.transport.requests.Request()\n",
" creds.refresh(auth_req)\n",
" # Note: the credential expires after 1 hour by default.\n",
" # After expiration, it must be refreshed.\n",
" creds, _ = google.auth.default(\n",
" scopes=[\"https://www.googleapis.com/auth/cloud-platform\"]\n",
" )\n",
" auth_req = google.auth.transport.requests.Request()\n",
" creds.refresh(auth_req)\n",
"\n",
" if model_kwargs is None:\n",
" model_kwargs = {}\n",
" if model_kwargs is None:\n",
" model_kwargs = {}\n",
"\n",
" return ChatOpenAI(\n",
" model=\"\",\n",
" base_url=base_url,\n",
" api_key=creds.token,\n",
" **model_kwargs,\n",
" )\n",
" return ChatOpenAI(\n",
" model=\"\",\n",
" base_url=base_url,\n",
" api_key=creds.token,\n",
" **model_kwargs,\n",
" )\n",
"\n",
"\n",
"# @markdown Use the following parameters to generate different answers:\n",
@@ -885,25 +897,25 @@
" \"history\": lambda x: x[\"history\"],\n",
" \"input\": lambda x: x[\"input\"],\n",
" \"ai_prompt\": lambda x: x[\"ai_prompt\"],\n",
" \"agent_scratchpad\": lambda x: format_to_tool_messages(\n",
" x[\"intermediate_steps\"]\n",
" ),\n",
"} | prompts.ChatPromptTemplate.from_messages([\n",
" prompts.MessagesPlaceholder(variable_name=\"system_prompt\"),\n",
" prompts.MessagesPlaceholder(variable_name=\"history\"),\n",
" (\"user\", \"{input}\"),\n",
" prompts.MessagesPlaceholder(variable_name=\"ai_prompt\"),\n",
" prompts.MessagesPlaceholder(variable_name=\"agent_scratchpad\"),\n",
"])\n",
" \"agent_scratchpad\": lambda x: format_to_tool_messages(x[\"intermediate_steps\"]),\n",
"} | prompts.ChatPromptTemplate.from_messages(\n",
" [\n",
" prompts.MessagesPlaceholder(variable_name=\"system_prompt\"),\n",
" prompts.MessagesPlaceholder(variable_name=\"history\"),\n",
" (\"user\", \"{input}\"),\n",
" prompts.MessagesPlaceholder(variable_name=\"ai_prompt\"),\n",
" prompts.MessagesPlaceholder(variable_name=\"agent_scratchpad\"),\n",
" ]\n",
")\n",
"\n",
"# Initialize session history\n",
"store = {}\n",
"\n",
"\n",
"def get_session_history(session_id: str):\n",
" if session_id not in store:\n",
" store[session_id] = ChatMessageHistory()\n",
" return store[session_id]\n",
" if session_id not in store:\n",
" store[session_id] = ChatMessageHistory()\n",
" return store[session_id]\n",
"\n",
"\n",
"agent = agent_engines.LangchainAgent(\n",
@@ -917,44 +929,44 @@
" \"extra_body\": {},\n",
" },\n",
")"
],
"metadata": {
"cellView": "form",
"id": "8A_DhwDrQbOW"
},
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "tsdsFCe00IGg"
},
"outputs": [],
"source": [
"# @title Test The Agent Locally\n",
"# @markdown You can check the agent result before deploying to Vertex AI.\n",
"\n",
"\n",
"def test_agent(running_agent, query, session_id):\n",
" response = running_agent.query(\n",
" input={\n",
" \"system_prompt\": [(\"system\", system_prompt_with_tools)],\n",
" \"input\": query,\n",
" \"ai_prompt\": [],\n",
" },\n",
" config={\"configurable\": {\"session_id\": session_id}},\n",
" )\n",
" tool_results = call_tools(response[\"output\"])\n",
"\n",
" if tool_results:\n",
" temporal_result = response[\"output\"] + json.dumps(tool_results)\n",
" response = running_agent.query(\n",
" input={\n",
" \"system_prompt\": [(\"system\", system_prompt_without_tools)],\n",
" \"ai_prompt\": [(\"ai\", temporal_result)],\n",
" \"system_prompt\": [(\"system\", system_prompt_with_tools)],\n",
" \"input\": query,\n",
" \"ai_prompt\": [],\n",
" },\n",
" config={\"configurable\": {\"session_id\": session_id}},\n",
" )\n",
" tool_results = call_tools(response[\"output\"])\n",
"\n",
" print(response[\"output\"])\n",
" if tool_results:\n",
" temporal_result = response[\"output\"] + json.dumps(tool_results)\n",
" response = running_agent.query(\n",
" input={\n",
" \"system_prompt\": [(\"system\", system_prompt_without_tools)],\n",
" \"ai_prompt\": [(\"ai\", temporal_result)],\n",
" \"input\": query,\n",
" },\n",
" config={\"configurable\": {\"session_id\": session_id}},\n",
" )\n",
"\n",
" print(response[\"output\"])\n",
"\n",
"\n",
"query = \"What is the weather and current time in SF?\" # @param {type:\"string\"}\n",
@@ -965,16 +977,16 @@
" query=query,\n",
" session_id=session_id,\n",
")"
],
"metadata": {
"cellView": "form",
"id": "tsdsFCe00IGg"
},
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "KLQreX5X0v6V"
},
"outputs": [],
"source": [
"# @title Deploy Agent On Vertex AI\n",
"\n",
@@ -1003,19 +1015,17 @@
"# @markdown The agent resource name will be used below.\n",
"\n",
"remote_agent_resource_name = remote_agent.resource_name\n",
"print(\n",
" \"The deployed remote agent resource name is: \", remote_agent_resource_name\n",
")"
],
"metadata": {
"cellView": "form",
"id": "KLQreX5X0v6V"
},
"execution_count": null,
"outputs": []
"print(\"The deployed remote agent resource name is: \", remote_agent_resource_name)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "5XnQ8uBNWpOW"
},
"outputs": [],
"source": [
"# @title Test The Remote Agent\n",
"\n",
@@ -1031,13 +1041,7 @@
" query=query,\n",
" session_id=session_id,\n",
")"
],
"metadata": {
"cellView": "form",
"id": "5XnQ8uBNWpOW"
},
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "markdown",
@@ -1079,18 +1083,17 @@
" shutil.rmtree(TUTORIAL_DIR)\n",
"\n",
"if delete_agent_engine:\n",
" remote_agent.delete()\n"
" remote_agent.delete()"
]
}
],
"metadata": {
"colab": {
"name": "model_garden_integration_with_agent.ipynb",
"toc_visible": true,
"provenance": [],
"collapsed_sections": [
"tAelDidov5AW"
]
],
"name": "model_garden_integration_with_agent.ipynb",
"toc_visible": true
},
"kernelspec": {
"display_name": "Python 3",