mirror of
https://github.com/GoogleCloudPlatform/vertex-ai-samples.git
synced 2026-09-26 22:51:56 +00:00
Compare commits
@@ -238,7 +238,7 @@ def _get_notebook_python_version(notebook_path: str) -> str:
|
||||
|
||||
# Look for the python version specification pattern
|
||||
re_match = re.search(
|
||||
"python version = (\d+\.\d+)", markdown, flags=re.IGNORECASE
|
||||
r"python version = (\d+\.\d+)", markdown, flags=re.IGNORECASE
|
||||
)
|
||||
if re_match:
|
||||
# get the version number
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
notebooks/official/vizier/gapic-vizier-multi-objective-optimization.ipynb
|
||||
notebooks/official/pipelines/lightweight_functions_component_io_kfp.ipynb
|
||||
notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-locally-trained-models.ipynb
|
||||
notebooks/official/custom/custom-tabular-bq-managed-dataset.ipynb
|
||||
.cloud-build/tests/python_version_test.ipynb
|
||||
|
||||
@@ -333,6 +333,7 @@ def vqa_predict(
|
||||
image: Any,
|
||||
language_code: str = "en",
|
||||
new_width: int = 1000,
|
||||
use_dedicated_endpoint: bool = False,
|
||||
) -> Sequence[str]:
|
||||
"""Predicts the answer to a question about an image using an Endpoint."""
|
||||
# Resize and convert image to base64 string.
|
||||
@@ -356,7 +357,9 @@ def vqa_predict(
|
||||
"image": resized_image_base64,
|
||||
})
|
||||
|
||||
response = endpoint.predict(instances=instances)
|
||||
response = endpoint.predict(
|
||||
instances=instances, use_dedicated_endpoint=use_dedicated_endpoint
|
||||
)
|
||||
return [pred.get("response") for pred in response.predictions]
|
||||
|
||||
|
||||
@@ -366,6 +369,7 @@ def caption_predict(
|
||||
image: Any,
|
||||
caption_prompt: bool = False,
|
||||
new_width: int = 1000,
|
||||
use_dedicated_endpoint: bool = False,
|
||||
) -> str:
|
||||
"""Predicts a caption for a given image using an Endpoint."""
|
||||
# Resize and convert image to base64 string.
|
||||
@@ -380,7 +384,9 @@ def caption_predict(
|
||||
instance["prompt"] = caption_prompt_format.format(language_code)
|
||||
|
||||
instances = [instance]
|
||||
response = endpoint.predict(instances=instances)
|
||||
response = endpoint.predict(
|
||||
instances=instances, use_dedicated_endpoint=use_dedicated_endpoint
|
||||
)
|
||||
return response.predictions[0].get("response")
|
||||
|
||||
|
||||
@@ -389,6 +395,7 @@ def ocr_predict(
|
||||
ocr_prompt: str,
|
||||
image: Any,
|
||||
new_width: int = 1000,
|
||||
use_dedicated_endpoint: bool = False,
|
||||
) -> str:
|
||||
"""Extracts text from a given image using an Endpoint."""
|
||||
# Resize and convert image to base64 string.
|
||||
@@ -400,7 +407,9 @@ def ocr_predict(
|
||||
instance["prompt"] = ocr_prompt
|
||||
instances = [instance]
|
||||
|
||||
response = endpoint.predict(instances=instances)
|
||||
response = endpoint.predict(
|
||||
instances=instances, use_dedicated_endpoint=use_dedicated_endpoint
|
||||
)
|
||||
return response.predictions[0].get("response")
|
||||
|
||||
|
||||
@@ -409,6 +418,7 @@ def detect_predict(
|
||||
detect_prompt: str,
|
||||
image: Any,
|
||||
new_width: int = 1000,
|
||||
use_dedicated_endpoint: bool = False,
|
||||
) -> str:
|
||||
"""Predicts the answer to a question about an image using an Endpoint."""
|
||||
# Resize and convert image to base64 string.
|
||||
@@ -420,7 +430,9 @@ def detect_predict(
|
||||
instance["prompt"] = detect_prompt
|
||||
instances = [instance]
|
||||
|
||||
response = endpoint.predict(instances=instances)
|
||||
response = endpoint.predict(
|
||||
instances=instances, use_dedicated_endpoint=use_dedicated_endpoint
|
||||
)
|
||||
return response.predictions[0].get("response")
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,465 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "Pr9TgOcV9vAXeqGiyTaTI5kS",
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "Pr9TgOcV9vAXeqGiyTaTI5kS"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Copyright 2025 Google LLC\n",
|
||||
"#\n",
|
||||
"# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
|
||||
"# you may not use this file except in compliance with the License.\n",
|
||||
"# You may obtain a copy of the License at\n",
|
||||
"#\n",
|
||||
"# https://www.apache.org/licenses/LICENSE-2.0\n",
|
||||
"#\n",
|
||||
"# Unless required by applicable law or agreed to in writing, software\n",
|
||||
"# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
|
||||
"# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
|
||||
"# See the License for the specific language governing permissions and\n",
|
||||
"# limitations under the License."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "M1CpgYundFwz",
|
||||
"metadata": {
|
||||
"id": "M1CpgYundFwz"
|
||||
},
|
||||
"source": [
|
||||
"# Get started with your deployed model on GKE\n",
|
||||
"\n",
|
||||
"<table><tbody><tr>\n",
|
||||
" <td style=\"text-align: center\">\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/colab/import/https:%2F%2Fraw.githubusercontent.com%2FGoogleCloudPlatform%2Fvertex-ai-samples%2Fmain%2Fnotebooks%2Fcommunity%2Fmodel_garden%2Fgke_model_ui_deployment_notebook_auto.ipynb\">\n",
|
||||
" <img alt=\"Google Cloud Colab Enterprise logo\" src=\"https://lh3.googleusercontent.com/JmcxdQi-qOpctIvWKgPtrzZdJJK-J3sWE1RsfjZNwshCFgE_9fULcNpuXYTilIR2hjwN\" width=\"32px\"><br> Run in Colab Enterprise\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td style=\"text-align: center\">\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/gke_model_ui_deployment_notebook_auto.ipynb\">\n",
|
||||
" <img alt=\"GitHub logo\" src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" width=\"32px\"><br> View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
"</tr></tbody></table>"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "t2jj2XOgkS4F",
|
||||
"metadata": {
|
||||
"id": "t2jj2XOgkS4F"
|
||||
},
|
||||
"source": [
|
||||
"# Overview\n",
|
||||
"\n",
|
||||
"This notebook will guide you through the initial step of testing your recently\n",
|
||||
"deployed model with text prompts. Depending on your deployed model's inference\n",
|
||||
"setup, the notebook utilizes either Text Generation Inference\n",
|
||||
"[TGI](https://huggingface.co/docs/text-generation-inference/en/index) or\n",
|
||||
"[vLLM](https://developers.googleblog.com/en/inference-with-gemma-using-dataflow-and-vllm/#:~:text=model%20frameworks%20simple.-,What%20is%20vLLM%3F,-vLLM%20is%20an),\n",
|
||||
"two efficient serving frameworks that enhance the performance of your GPU model.\n",
|
||||
"Ready to see your deployed model respond? Run the cells below and start\n",
|
||||
"experimenting with different prompts!\n",
|
||||
"\n",
|
||||
"### Prerequisites\n",
|
||||
"\n",
|
||||
"Before proceeding with this notebook, ensure you have already deployed a model\n",
|
||||
"using the Google Cloud Console. You can find an overview of AI and Machine\n",
|
||||
"Learning services on\n",
|
||||
"[GKE AI/ML](https://console.cloud.google.com/kubernetes/aiml/overview).\n",
|
||||
"\n",
|
||||
"### Objective\n",
|
||||
"\n",
|
||||
"Enable prompt-based testing of the AI model deployed on GKE\n",
|
||||
"\n",
|
||||
"### GPUs\n",
|
||||
"\n",
|
||||
"GPUs let you accelerate specific workloads running on your nodes, such as\n",
|
||||
"machine learning and data processing. GKE provides a range of machine type\n",
|
||||
"options for node configuration, including machine types with NVIDIA H100, L4,\n",
|
||||
"and A100 GPUs.\n",
|
||||
"\n",
|
||||
"### Understanding the Inference Frameworks\n",
|
||||
"\n",
|
||||
"Your model is running on one of two popular and efficient serving frameworks:\n",
|
||||
"vLLM or Text Generation Inference (TGI). The following sections provide a brief\n",
|
||||
"overview of each to give you context on the underlying technology powering your\n",
|
||||
"model.\n",
|
||||
"\n",
|
||||
"#### TGI\n",
|
||||
"\n",
|
||||
"TGI is a highly optimized open-source LLM serving framework that can increase\n",
|
||||
"serving throughput on GPUs. TGI includes features such as:\n",
|
||||
"\n",
|
||||
"* Optimized transformer implementation with PagedAttention\n",
|
||||
"* Continuous batching to improve the overall serving throughput\n",
|
||||
"* Tensor parallelism and distributed serving on multiple GPUs\n",
|
||||
"\n",
|
||||
"To learn more, refer to the\n",
|
||||
"[TGI documentation](https://github.com/huggingface/text-generation-inference/blob/main/README.md)\n",
|
||||
"\n",
|
||||
"#### vLLM\n",
|
||||
"\n",
|
||||
"vLLM is another fast and easy-to-use library for LLM inference and serving. It's\n",
|
||||
"known for its high throughput and efficiency, and it leverages PagedAttention.\n",
|
||||
"Key features include:\n",
|
||||
"\n",
|
||||
"* PagedAttention: Efficient memory management for handling long sequences and\n",
|
||||
" dynamic workloads.\n",
|
||||
"* Continuous batching: Maximizes GPU utilization by batching incoming\n",
|
||||
" requests.\n",
|
||||
"* High-throughput serving: Designed for production-level serving with low\n",
|
||||
" latency.\n",
|
||||
"* Optimized CUDA kernels.\n",
|
||||
"\n",
|
||||
"To learn more, refer to the\n",
|
||||
"[vLLM documentation](https://cloud.google.com/vertex-ai/generative-ai/docs/open-models/vllm/use-vllm)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "XMf-T58TkDy1",
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "XMf-T58TkDy1"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title # Connect to Google Cloud Project\n",
|
||||
"# @markdown #### Run this cell to configure your Google Cloud environment for Kubernetes (GKE) operations.\n",
|
||||
"# @markdown\n",
|
||||
"# @markdown #### Actions:\n",
|
||||
"# @markdown 1. **Connects to Project:** Retrieves and sets your Google Cloud project ID.\n",
|
||||
"# @markdown 3. **Installs `kubectl`:** Installs the Kubernetes command-line tool.\n",
|
||||
"\n",
|
||||
"import os\n",
|
||||
"\n",
|
||||
"# Get the default cloud project id.\n",
|
||||
"PROJECT_ID = os.environ[\"GOOGLE_CLOUD_PROJECT\"]\n",
|
||||
"\n",
|
||||
"# Set up gcloud.\n",
|
||||
"! gcloud config set project \"$PROJECT_ID\"\n",
|
||||
"! gcloud services enable container.googleapis.com\n",
|
||||
"\n",
|
||||
"# Add kubectl to the set of available tools.\n",
|
||||
"! mkdir -p /tools/google-cloud-sdk/.install\n",
|
||||
"! gcloud components install kubectl --quiet"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "IKGTaN84p8rX",
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "IKGTaN84p8rX"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title # Chat completion for text-only models { vertical-output: true}\n",
|
||||
"# @markdown You may send prompts to the model server for prediction.\n",
|
||||
"# @markdown\n",
|
||||
"# @markdown * **user_prompt (string):** This is the text prompt you provide to the language model. It's the question or instruction e (e.g., \"Explain neural networks\").\n",
|
||||
"# @markdown * **temperature (number):** This parameter controls the randomness of the model's output. It influences how the model selects the next token in the sequence it generates. Typical values range from 0.2 to 1.0.\n",
|
||||
"# @markdown * **max_tokens (number):** This parameter refers to the maximum number of tokens (words or sub-word units) that the model is allowed to generate in its response.\n",
|
||||
"# @markdown\n",
|
||||
"\n",
|
||||
"import json\n",
|
||||
"import subprocess\n",
|
||||
"\n",
|
||||
"import ipywidgets as widgets\n",
|
||||
"from IPython.display import Markdown, clear_output, display\n",
|
||||
"\n",
|
||||
"CLUSTER = \"\" # @param {type:\"string\", isTemplate:true}\n",
|
||||
"REGION = \"\" # @param {type:\"string\", isTemplate:true}\n",
|
||||
"NAMESPACE = \"\" # @param {type:\"string\", isTemplate:true}\n",
|
||||
"DEPLOYMENT = \"\" # @param {type:\"string\", isTemplate:true}\n",
|
||||
"DEPLOYMENT_APP_LABEL = \"\" # @param {type:\"string\", isTemplate:true}\n",
|
||||
"\n",
|
||||
"SERVICE = f\"{DEPLOYMENT}-service\"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def _run_kubectl(cmd):\n",
|
||||
" \"\"\"Executes a kubectl command and returns its stdout.\"\"\"\n",
|
||||
" result = subprocess.run(cmd, capture_output=True, text=True, check=True, timeout=60)\n",
|
||||
" return result.stdout.strip()\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def fetch_cluster_credential(cluster, region, project_id):\n",
|
||||
" try:\n",
|
||||
" # Ensure credentials for the target cluster\n",
|
||||
" cred_cmd = [\n",
|
||||
" \"gcloud\",\n",
|
||||
" \"container\",\n",
|
||||
" \"clusters\",\n",
|
||||
" \"get-credentials\",\n",
|
||||
" cluster,\n",
|
||||
" f\"--location={region}\",\n",
|
||||
" f\"--project={project_id}\",\n",
|
||||
" ]\n",
|
||||
" _run_kubectl(cred_cmd)\n",
|
||||
" except Exception as e:\n",
|
||||
" # Original code prints error and returns empty dict\n",
|
||||
" print(f\"Error fetching cluster credentials: {e}\")\n",
|
||||
" return {}\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def get_deployment_pod_name(deployment, namespace, deployment_app_label):\n",
|
||||
" \"\"\"Finds the running pod name for a given deployment and namespace.\"\"\"\n",
|
||||
"\n",
|
||||
" cmd = [\n",
|
||||
" \"kubectl\",\n",
|
||||
" \"get\",\n",
|
||||
" \"pods\",\n",
|
||||
" \"-n\",\n",
|
||||
" namespace,\n",
|
||||
" \"-o\",\n",
|
||||
" \"json\",\n",
|
||||
" \"-l\",\n",
|
||||
" f\"app={deployment_app_label}\",\n",
|
||||
" \"--field-selector=status.phase=Running\",\n",
|
||||
" ]\n",
|
||||
" try:\n",
|
||||
" pods_json = _run_kubectl(cmd)\n",
|
||||
" pods = json.loads(pods_json)\n",
|
||||
" if pods.get(\"items\"):\n",
|
||||
" return pods[\"items\"][0][\"metadata\"][\"name\"]\n",
|
||||
" print(f\"No running pods found for {deployment} in {namespace}.\")\n",
|
||||
" return None\n",
|
||||
" except (\n",
|
||||
" subprocess.CalledProcessError,\n",
|
||||
" json.JSONDecodeError,\n",
|
||||
" IndexError,\n",
|
||||
" KeyError,\n",
|
||||
" ) as e:\n",
|
||||
" print(f\"Error getting pod name for {deployment} in {namespace}: {e}\")\n",
|
||||
" return None\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def check_inference_label(pod_name, namespace):\n",
|
||||
" \"\"\"Checks if the specified pod has the vLLM inference server label.\"\"\"\n",
|
||||
"\n",
|
||||
" cmd = [\"kubectl\", \"get\", \"pod\", pod_name, \"-n\", namespace, \"-o\", \"json\"]\n",
|
||||
" try:\n",
|
||||
" pod_json = _run_kubectl(cmd)\n",
|
||||
" labels = json.loads(pod_json).get(\"metadata\", {}).get(\"labels\", {})\n",
|
||||
" return labels.get(\"ai.gke.io/inference-server\") == \"vllm\"\n",
|
||||
" except (subprocess.CalledProcessError, json.JSONDecodeError, KeyError) as e:\n",
|
||||
" print(f\"Error checking labels for pod {pod_name} in {namespace}: {e}\")\n",
|
||||
" return False\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def get_service_endpoint(service, namespace):\n",
|
||||
" \"\"\"Retrieve the service endpoint of the deployment\"\"\"\n",
|
||||
" endpoint_cmd = [\n",
|
||||
" \"kubectl\",\n",
|
||||
" \"get\",\n",
|
||||
" \"endpoints\",\n",
|
||||
" service,\n",
|
||||
" \"-n\",\n",
|
||||
" namespace,\n",
|
||||
" ]\n",
|
||||
" try:\n",
|
||||
" endpoint_output = _run_kubectl(endpoint_cmd).splitlines()\n",
|
||||
" if len(endpoint_output) < 2 or len(endpoint_output[1].split()) < 2:\n",
|
||||
" print(f\"Endpoint data incomplete for {service}.\")\n",
|
||||
" return None\n",
|
||||
" endpoint = endpoint_output[1].split()[\n",
|
||||
" 1\n",
|
||||
" ] # Assumes format: NAME ENDPOINTS AGE -> service ip:port,... age\n",
|
||||
" return endpoint\n",
|
||||
" except subprocess.CalledProcessError as e:\n",
|
||||
" print(f\"Error getting endpoints for {service}: {e}\")\n",
|
||||
" return None\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def process_response(request, pod_name, pod_endpoint, is_vllm_inference, namespace):\n",
|
||||
" \"\"\"Sends a request to the pod and processes the response.\"\"\"\n",
|
||||
"\n",
|
||||
" json_data_escaped = json.dumps(request).replace(\"'\", \"'\\\\''\")\n",
|
||||
" curl_cmd = (\n",
|
||||
" f\"kubectl exec -n {namespace} -t {pod_name} -- curl -s -X POST\"\n",
|
||||
" f' http://{pod_endpoint}/generate -H \"Content-Type: application/json\"'\n",
|
||||
" f\" -d '{json_data_escaped}' 2> /dev/null\"\n",
|
||||
" )\n",
|
||||
" try:\n",
|
||||
" response_raw = _run_kubectl([\"bash\", \"-c\", curl_cmd])\n",
|
||||
" if not response_raw:\n",
|
||||
" return f\"Error: Empty response from pod {pod_name}.\"\n",
|
||||
" first_line = response_raw.splitlines()[0]\n",
|
||||
" data = json.loads(first_line)\n",
|
||||
"\n",
|
||||
" if is_vllm_inference: # vLLM format\n",
|
||||
" predictions = data.get(\"predictions\")\n",
|
||||
" if isinstance(predictions, (list, tuple)) and predictions:\n",
|
||||
" return predictions[0]\n",
|
||||
" return f\"Error: Unexpected vLLM format. Raw: {first_line}\"\n",
|
||||
" else: # TGI format\n",
|
||||
" generated_text = data.get(\"generated_text\")\n",
|
||||
" if generated_text is not None:\n",
|
||||
" return generated_text\n",
|
||||
" return f\"Error: Unexpected TGI format. Raw: {first_line}\"\n",
|
||||
" except Exception as e:\n",
|
||||
" return f\"Unexpected error during response processing: {e}\"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# --- Widgets Setup ---\n",
|
||||
"user_prompt_widget = widgets.Textarea(\n",
|
||||
" value=\"What is AI?\",\n",
|
||||
" description=\"User Prompt:\",\n",
|
||||
" layout=widgets.Layout(width=\"95%\", height=\"100px\"),\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"temperature_widget = widgets.FloatSlider(\n",
|
||||
" value=0.50, min=0.0, max=1.0, step=0.01, description=\"Temperature:\"\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"max_tokens_widget = widgets.IntSlider(\n",
|
||||
" value=250, min=1, max=2048, step=1, description=\"Max Tokens:\"\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"submit_button = widgets.Button(description=\"Submit\")\n",
|
||||
"output_area_response = widgets.Output()\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# --- Submit Button Logic ---\n",
|
||||
"def on_submit_clicked(b):\n",
|
||||
" \"\"\"Handles the submit button click event.\"\"\"\n",
|
||||
" with output_area_response:\n",
|
||||
" clear_output()\n",
|
||||
"\n",
|
||||
" fetch_cluster_credential(CLUSTER, REGION, PROJECT_ID)\n",
|
||||
"\n",
|
||||
" # retrieve deployment pod\n",
|
||||
" pod_name = get_deployment_pod_name(DEPLOYMENT, NAMESPACE, DEPLOYMENT_APP_LABEL)\n",
|
||||
" if not pod_name:\n",
|
||||
" display(\n",
|
||||
" Markdown(f\"**Error:** Could not find running pod for `{DEPLOYMENT}`.\")\n",
|
||||
" )\n",
|
||||
" return\n",
|
||||
"\n",
|
||||
" # build the request message\n",
|
||||
" is_vllm = check_inference_label(pod_name, NAMESPACE)\n",
|
||||
" request = {\n",
|
||||
" \"max_tokens\": max_tokens_widget.value,\n",
|
||||
" \"temperature\": temperature_widget.value,\n",
|
||||
" \"prompt\" if is_vllm else \"inputs\": user_prompt_widget.value,\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" # retrieve service endpoint for the deployment\n",
|
||||
" endpoint = get_service_endpoint(SERVICE, NAMESPACE)\n",
|
||||
" if not endpoint:\n",
|
||||
" display(Markdown(f\"**Error getting endpoints for `{SERVICE}`:**\\n\"))\n",
|
||||
" return\n",
|
||||
"\n",
|
||||
" # prompt test the deployment endpoint\n",
|
||||
" try:\n",
|
||||
" response = process_response(request, pod_name, endpoint, is_vllm, NAMESPACE)\n",
|
||||
" display(Markdown(f\"**Response:**\\n\\n{response}\"))\n",
|
||||
" except Exception as e:\n",
|
||||
" display(Markdown(f\"**Unexpected Error:**\\n```\\n{e}\\n```\"))\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# --- Display Widgets ---\n",
|
||||
"submit_button.on_click(on_submit_clicked)\n",
|
||||
"display(\n",
|
||||
" user_prompt_widget,\n",
|
||||
" temperature_widget,\n",
|
||||
" max_tokens_widget,\n",
|
||||
" submit_button,\n",
|
||||
" output_area_response,\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "5b6ZM2K3fux0",
|
||||
"metadata": {
|
||||
"id": "5b6ZM2K3fux0"
|
||||
},
|
||||
"source": [
|
||||
"# Next Steps: Integrating the GKE Service Endpoint\n",
|
||||
"\n",
|
||||
"After successfully deploying a model on Google Kubernetes Engine (GKE) and\n",
|
||||
"verifying it via a notebook, the next step is to integrate it into various\n",
|
||||
"applications. This involves making HTTP requests to the service's endpoint from\n",
|
||||
"your application code.\n",
|
||||
"\n",
|
||||
"### Exposing the Service\n",
|
||||
"\n",
|
||||
"To make your deployed model accessible to applications, you'll need to expose\n",
|
||||
"its service endpoint. Google Kubernetes Engine offers several ways to do this:\n",
|
||||
"\n",
|
||||
"1. **Ingress:** Configure an Ingress resource to route external HTTP(S) traffic\n",
|
||||
" to your service. Set up Ingress for either an internal Load Balancer\n",
|
||||
" (accessible only within your VPC) or an external Load Balancer (accessible\n",
|
||||
" from the internet).\n",
|
||||
" [Learn more about GKE Ingress](https://cloud.google.com/kubernetes-engine/docs/concepts/ingress).\n",
|
||||
"2. **Gateway API:** A more modern and feature-rich API for managing traffic\n",
|
||||
" routing in Kubernetes. Similar to Ingress, Gateway API allows you to define\n",
|
||||
" how external and internal traffic should be directed to your services.\n",
|
||||
" [Explore GKE Gateway API](https://cloud.google.com/kubernetes-engine/docs/concepts/gateway-api).\n",
|
||||
"\n",
|
||||
"### Setting Up Autoscaling\n",
|
||||
"\n",
|
||||
"Ensure your model serving can handle varying traffic by configuring the\n",
|
||||
"Horizontal Pod Autoscaler (HPA). HPA automatically scales the number of Pods\n",
|
||||
"based on resource utilization or custom metrics, optimizing performance and\n",
|
||||
"cost.\n",
|
||||
"[See how to configure HPA](https://cloud.google.com/kubernetes-engine/docs/how-to/horizontal-pod-autoscaling).\n",
|
||||
"\n",
|
||||
"### Setting Up Monitoring\n",
|
||||
"\n",
|
||||
"Monitor the health and performance of your deployed model using Google Cloud\n",
|
||||
"Managed Service for Prometheus. Configure your model serving to expose\n",
|
||||
"Prometheus metrics for comprehensive insights.\n",
|
||||
"[Get started with Google Cloud Managed Prometheus](https://cloud.google.com/kubernetes-engine/docs/how-to/configure-automatic-application-monitoring).\n",
|
||||
"\n",
|
||||
"### Additional Resources:\n",
|
||||
"\n",
|
||||
"* #### Kubernetes Documentation:\n",
|
||||
"\n",
|
||||
" * Services:\n",
|
||||
" https://kubernetes.io/docs/concepts/services-networking/service/\n",
|
||||
"\n",
|
||||
"* #### Google Cloud Documentation:\n",
|
||||
"\n",
|
||||
" * Google Kubernetes Engine (GKE):\n",
|
||||
" https://cloud.google.com/kubernetes-engine\n",
|
||||
" * Cloud Load Balancing:\n",
|
||||
" https://cloud.google.com/load-balancing/docs/ingress\n",
|
||||
" * Gateway API on GKE:\n",
|
||||
" https://cloud.google.com/kubernetes-engine/docs/concepts/gateway-api\n",
|
||||
" * Learn about GPUs in GKE:\n",
|
||||
" https://cloud.google.com/kubernetes-engine/docs/concepts/gpus\n",
|
||||
"\n",
|
||||
"* #### Python requests Library:\n",
|
||||
"\n",
|
||||
" * https://requests.readthedocs.io/en/latest/\n",
|
||||
"\n",
|
||||
"* #### LangChain with Google Integrations:\n",
|
||||
"\n",
|
||||
" * The Langchain documentation is very useful:\n",
|
||||
" https://python.langchain.com/docs/integrations/providers/google/"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"name": "gke_model_ui_deployment_notebook_auto.ipynb",
|
||||
"toc_visible": true
|
||||
},
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"name": "python3"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 0
|
||||
}
|
||||
@@ -6,7 +6,7 @@
|
||||
"id": "DZ1j6RRg-Td6",
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "DZ1j6RRg-Td6"
|
||||
"id": "f705f4be70e9"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -29,7 +29,7 @@
|
||||
"cell_type": "markdown",
|
||||
"id": "99c1c3fc2ca5",
|
||||
"metadata": {
|
||||
"id": "99c1c3fc2ca5"
|
||||
"id": "778cc1227be8"
|
||||
},
|
||||
"source": [
|
||||
"# Vertex AI Model Garden - Advanced Features\n",
|
||||
@@ -52,7 +52,7 @@
|
||||
"cell_type": "markdown",
|
||||
"id": "f9-tJ6RfDLIs",
|
||||
"metadata": {
|
||||
"id": "f9-tJ6RfDLIs"
|
||||
"id": "0779b48f654e"
|
||||
},
|
||||
"source": [
|
||||
"## Overview\n",
|
||||
@@ -90,7 +90,7 @@
|
||||
"cell_type": "markdown",
|
||||
"id": "47GcOrZjosOx",
|
||||
"metadata": {
|
||||
"id": "47GcOrZjosOx"
|
||||
"id": "69453bf7230e"
|
||||
},
|
||||
"source": [
|
||||
"## Before you begin"
|
||||
@@ -100,7 +100,7 @@
|
||||
"cell_type": "markdown",
|
||||
"id": "1D_pWejJPHP3",
|
||||
"metadata": {
|
||||
"id": "1D_pWejJPHP3"
|
||||
"id": "bf3706e69f61"
|
||||
},
|
||||
"source": [
|
||||
"### Request for quota\n",
|
||||
@@ -118,10 +118,9 @@
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "L3dqbxovo5t6",
|
||||
"language": "python",
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "L3dqbxovo5t6"
|
||||
"id": "2b585189a670"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -149,6 +148,8 @@
|
||||
"# Install and import the necessary packages\n",
|
||||
"! pip install -q openai google-auth requests\n",
|
||||
"\n",
|
||||
"! pip3 install --upgrade --quiet 'google-cloud-aiplatform>=1.93.1'\n",
|
||||
"\n",
|
||||
"! git clone https://github.com/GoogleCloudPlatform/vertex-ai-samples.git\n",
|
||||
"\n",
|
||||
"import datetime\n",
|
||||
@@ -224,7 +225,7 @@
|
||||
"cell_type": "markdown",
|
||||
"id": "SeGqxuMfRBS5",
|
||||
"metadata": {
|
||||
"id": "SeGqxuMfRBS5"
|
||||
"id": "4782dd003acb"
|
||||
},
|
||||
"source": [
|
||||
"### Access Llama 3.1, 3.2, and 3.3 models on Vertex AI for serving"
|
||||
@@ -236,7 +237,7 @@
|
||||
"id": "BxlzWU2KQqmw",
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "BxlzWU2KQqmw"
|
||||
"id": "798068fc0355"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -275,7 +276,7 @@
|
||||
"cell_type": "markdown",
|
||||
"id": "JpNBJJgjWL7j",
|
||||
"metadata": {
|
||||
"id": "JpNBJJgjWL7j"
|
||||
"id": "10ed490e28e5"
|
||||
},
|
||||
"source": [
|
||||
"## Prefix Caching <a name=\"prefix-caching\"></a>\n",
|
||||
@@ -303,7 +304,7 @@
|
||||
"cell_type": "markdown",
|
||||
"id": "9gZJ8cB27e1m",
|
||||
"metadata": {
|
||||
"id": "9gZJ8cB27e1m"
|
||||
"id": "30ddb93fdd7b"
|
||||
},
|
||||
"source": [
|
||||
"### Try out Prefix Caching with Hex-LLM\n",
|
||||
@@ -319,7 +320,7 @@
|
||||
"id": "RpmoA2nXjdCd",
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "RpmoA2nXjdCd"
|
||||
"id": "b56d82c1aa6f"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -508,7 +509,7 @@
|
||||
"id": "5QoK8c0R9U3B",
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "5QoK8c0R9U3B"
|
||||
"id": "96c5afed49b4"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -519,9 +520,7 @@
|
||||
"\n",
|
||||
"if use_dedicated_endpoint:\n",
|
||||
" DEDICATED_ENDPOINT_DNS = endpoints[\"hexllm_tpu\"].gca_resource.dedicated_endpoint_dns\n",
|
||||
"ENDPOINT_RESOURCE_NAME = \"projects/{}/locations/{}/endpoints/{}\".format(\n",
|
||||
" PROJECT_ID, REGION, endpoints[\"hexllm_tpu\"].name\n",
|
||||
")\n",
|
||||
"ENDPOINT_RESOURCE_NAME = endpoints[\"hexllm_tpu\"].resource_name\n",
|
||||
"\n",
|
||||
"# @markdown Once deployment succeeds, you can send requests to the endpoint using the OpenAI SDK.\n",
|
||||
"\n",
|
||||
@@ -576,7 +575,7 @@
|
||||
"id": "29rn5ATmB2YC",
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "29rn5ATmB2YC"
|
||||
"id": "9a95c9f90358"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -649,7 +648,7 @@
|
||||
"cell_type": "markdown",
|
||||
"id": "KjbM8E9DGuuR",
|
||||
"metadata": {
|
||||
"id": "KjbM8E9DGuuR"
|
||||
"id": "12ad6d1ff725"
|
||||
},
|
||||
"source": [
|
||||
"#### Delete the models and endpoints"
|
||||
@@ -661,7 +660,7 @@
|
||||
"id": "JpLU7GRQGuuR",
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "JpLU7GRQGuuR"
|
||||
"id": "1ab4e3bb74b4"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -687,7 +686,7 @@
|
||||
"cell_type": "markdown",
|
||||
"id": "XZ33HhYmOxCS",
|
||||
"metadata": {
|
||||
"id": "XZ33HhYmOxCS"
|
||||
"id": "7a8a9a1b2ddf"
|
||||
},
|
||||
"source": [
|
||||
"### Try out Prefix Caching with vLLM\n",
|
||||
@@ -710,7 +709,7 @@
|
||||
"id": "E8OiHHNNE_wj",
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "E8OiHHNNE_wj"
|
||||
"id": "4425cc0bdedc"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -914,7 +913,7 @@
|
||||
"id": "zex1oXl36A70",
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "zex1oXl36A70"
|
||||
"id": "bcbafec839cd"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -922,9 +921,7 @@
|
||||
"\n",
|
||||
"if use_dedicated_endpoint:\n",
|
||||
" DEDICATED_ENDPOINT_DNS = endpoints[\"vllm_gpu\"].gca_resource.dedicated_endpoint_dns\n",
|
||||
"ENDPOINT_RESOURCE_NAME = \"projects/{}/locations/{}/endpoints/{}\".format(\n",
|
||||
" PROJECT_ID, REGION, endpoints[\"vllm_gpu\"].name\n",
|
||||
")\n",
|
||||
"ENDPOINT_RESOURCE_NAME = endpoints[\"vllm_gpu\"].resource_name\n",
|
||||
"\n",
|
||||
"# @markdown Once deployment succeeds, you can send requests to the endpoint using the OpenAI SDK.\n",
|
||||
"\n",
|
||||
@@ -979,7 +976,7 @@
|
||||
"id": "gDOC_nfsJeUR",
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "gDOC_nfsJeUR"
|
||||
"id": "e984f43422d5"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -1052,7 +1049,7 @@
|
||||
"cell_type": "markdown",
|
||||
"id": "GdGxaTirJeUR",
|
||||
"metadata": {
|
||||
"id": "GdGxaTirJeUR"
|
||||
"id": "dff0d10dcc20"
|
||||
},
|
||||
"source": [
|
||||
"#### Delete the models and endpoints"
|
||||
@@ -1064,7 +1061,7 @@
|
||||
"id": "OgoqXE-VJeUR",
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "OgoqXE-VJeUR"
|
||||
"id": "5b8751773e7f"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -1090,7 +1087,7 @@
|
||||
"cell_type": "markdown",
|
||||
"id": "w4Guijaw_NEs",
|
||||
"metadata": {
|
||||
"id": "w4Guijaw_NEs"
|
||||
"id": "863775857a46"
|
||||
},
|
||||
"source": [
|
||||
"### Best practices\n",
|
||||
@@ -1105,7 +1102,7 @@
|
||||
"cell_type": "markdown",
|
||||
"id": "ml8fgoIQWSbY",
|
||||
"metadata": {
|
||||
"id": "ml8fgoIQWSbY"
|
||||
"id": "565cbdc3a06b"
|
||||
},
|
||||
"source": [
|
||||
"## Speculative Decoding <a name=\"spec-decoding\"></a>\n",
|
||||
@@ -1147,7 +1144,7 @@
|
||||
"cell_type": "markdown",
|
||||
"id": "NmWRro8Q-Td6",
|
||||
"metadata": {
|
||||
"id": "NmWRro8Q-Td6"
|
||||
"id": "94eaa9050abb"
|
||||
},
|
||||
"source": [
|
||||
"### Try out Speculative Decoding with vLLM"
|
||||
@@ -1159,7 +1156,7 @@
|
||||
"id": "72d1GlrYifKU",
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "72d1GlrYifKU"
|
||||
"id": "5f358cc230a6"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -1476,7 +1473,7 @@
|
||||
"id": "CNiItf5hdVFU",
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "CNiItf5hdVFU"
|
||||
"id": "be3170e0e05a"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -1502,9 +1499,7 @@
|
||||
" DEDICATED_ENDPOINT_DNS = endpoints[\n",
|
||||
" \"vllm_gpu_spec\"\n",
|
||||
" ].gca_resource.dedicated_endpoint_dns\n",
|
||||
"ENDPOINT_RESOURCE_NAME = \"projects/{}/locations/{}/endpoints/{}\".format(\n",
|
||||
" PROJECT_ID, REGION, endpoints[\"vllm_gpu_spec\"].name\n",
|
||||
")\n",
|
||||
"ENDPOINT_RESOURCE_NAME = endpoints[\"vllm_gpu_spec\"].resource_name\n",
|
||||
"\n",
|
||||
"BASE_URL = (\n",
|
||||
" f\"https://{REGION}-aiplatform.googleapis.com/v1beta1/{ENDPOINT_RESOURCE_NAME}\"\n",
|
||||
@@ -1553,7 +1548,7 @@
|
||||
"cell_type": "markdown",
|
||||
"id": "WahYGAZyq6Gl",
|
||||
"metadata": {
|
||||
"id": "WahYGAZyq6Gl"
|
||||
"id": "30c5d2535df3"
|
||||
},
|
||||
"source": [
|
||||
"## Clean up resources"
|
||||
@@ -1563,7 +1558,7 @@
|
||||
"cell_type": "markdown",
|
||||
"id": "bV5Yjkgav9BZ",
|
||||
"metadata": {
|
||||
"id": "bV5Yjkgav9BZ"
|
||||
"id": "63c10917ff95"
|
||||
},
|
||||
"source": [
|
||||
"### Delete the models and endpoints"
|
||||
@@ -1575,7 +1570,7 @@
|
||||
"id": "qsks36cOH9rb",
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "qsks36cOH9rb"
|
||||
"id": "92892e1b1730"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -221,7 +221,7 @@
|
||||
"# @markdown *--- Or ---*\n",
|
||||
"\n",
|
||||
"# @markdown #### Access CodeGemma models on HuggingFace\n",
|
||||
"# @markdown You must provide a Hugging Face User Access Token (read) to access the CodeGemma models. You can follow the [Hugging Face documentation](https://huggingface.co/docs/hub/en/security-tokens) to create a **read** access token and put it in the `HF_TOKEN` field below.\n",
|
||||
"# @markdown You must provide a Hugging Face User Access Token (with read access) to access the CodeGemma models. You can follow the [Hugging Face documentation](https://huggingface.co/docs/hub/en/security-tokens) to create a **read** access token and put it in the `HF_TOKEN` field below.\n",
|
||||
"HF_TOKEN = \"\" # @param {type:\"string\", isTemplate:true}\n",
|
||||
"if LOAD_MODEL_FROM == \"Hugging Face\":\n",
|
||||
" assert (\n",
|
||||
@@ -339,6 +339,7 @@
|
||||
" disagg_topology: str = None,\n",
|
||||
" hbm_utilization_factor: float = 0.6,\n",
|
||||
" max_running_seqs: int = 256,\n",
|
||||
" decode_seqs_padding: int = None,\n",
|
||||
" max_model_len: int = 4096,\n",
|
||||
" enable_prefix_cache_hbm: bool = False,\n",
|
||||
" endpoint_id: str = \"\",\n",
|
||||
@@ -379,6 +380,10 @@
|
||||
" f\"--max_running_seqs={max_running_seqs}\",\n",
|
||||
" f\"--max_model_len={max_model_len}\",\n",
|
||||
" ]\n",
|
||||
"\n",
|
||||
" if decode_seqs_padding is not None:\n",
|
||||
" hexllm_args.append(f\"--decode_seqs_padding={decode_seqs_padding}\")\n",
|
||||
"\n",
|
||||
" if disagg_topology:\n",
|
||||
" hexllm_args.append(f\"--disagg_topo={disagg_topology}\")\n",
|
||||
" if enable_prefix_cache_hbm and not disagg_topology:\n",
|
||||
|
||||
@@ -312,9 +312,7 @@
|
||||
" DEDICATED_ENDPOINT_DNS = endpoints[\n",
|
||||
" \"my-endpoint\"\n",
|
||||
" ].gca_resource.dedicated_endpoint_dns\n",
|
||||
"ENDPOINT_RESOURCE_NAME = \"projects/{}/locations/{}/endpoints/{}\".format(\n",
|
||||
" PROJECT_ID, REGION, endpoints[\"my-endpoint\"].name\n",
|
||||
")\n",
|
||||
"ENDPOINT_RESOURCE_NAME = endpoints[\"my-endpoint\"].resource_name\n",
|
||||
"\n",
|
||||
"# @title Chat Completions Inference\n",
|
||||
"\n",
|
||||
|
||||
@@ -241,8 +241,7 @@
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "kg5MwMIfB9Uj"
|
||||
"id": "I1u2FLa9XgVD"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -252,9 +251,13 @@
|
||||
"prebuilt_model_id = \"intfloat/e5-small-v2\" # @param [\"intfloat/multilingual-e5-large-instruct\", \"intfloat/multilingual-e5-large\", \"intfloat/e5-large-v2\", \"intfloat/multilingual-e5-small\", \"intfloat/e5-base-v2\", \"intfloat/e5-small-v2\"]\n",
|
||||
"\n",
|
||||
"# @markdown Specify a processor for the TEI docker image. E5 models can be run on either GPU or CPU.\n",
|
||||
"processor = \"NVIDIA_L4\" # @param[\"NVIDIA_TESLA_V100\", \"NVIDIA_L4\", \"NVIDIA_TESLA_A100\", \"CPU\"]\n",
|
||||
"processor = \"NVIDIA_L4\" # @param[\"NVIDIA_TESLA_T4\", \"NVIDIA_TESLA_V100\", \"NVIDIA_L4\", \"NVIDIA_TESLA_A100\", \"CPU\"]\n",
|
||||
"\n",
|
||||
"if processor == \"NVIDIA_TESLA_V100\":\n",
|
||||
"if processor == \"NVIDIA_TESLA_T4\":\n",
|
||||
" accelerator_type = \"NVIDIA_TESLA_T4\"\n",
|
||||
" machine_type = \"n1-highmem-16\"\n",
|
||||
" accelerator_count = 1\n",
|
||||
"elif processor == \"NVIDIA_TESLA_V100\":\n",
|
||||
" accelerator_type = \"NVIDIA_TESLA_V100\"\n",
|
||||
" machine_type = \"n1-highmem-16\"\n",
|
||||
" accelerator_count = 2\n",
|
||||
|
||||
@@ -32,18 +32,18 @@
|
||||
"source": [
|
||||
"# Vertex AI Model Garden - Finetuning Tutorial\n",
|
||||
"\n",
|
||||
"<table><tbody><tr>\n",
|
||||
" <td style=\"text-align: center\">\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/colab/import/https:%2F%2Fraw.githubusercontent.com%2FGoogleCloudPlatform%2Fvertex-ai-samples%2Fmain%2Fnotebooks%2Fcommunity%2Fmodel_garden%2Fmodel_garden_finetuning_tutorial.ipynb\">\n",
|
||||
" <img alt=\"Google Cloud Colab Enterprise logo\" src=\"https://lh3.googleusercontent.com/JmcxdQi-qOpctIvWKgPtrzZdJJK-J3sWE1RsfjZNwshCFgE_9fULcNpuXYTilIR2hjwN\" width=\"32px\"><br> Run in Colab Enterprise\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td style=\"text-align: center\">\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_finetuning_tutorial.ipynb\">\n",
|
||||
" <img alt=\"GitHub logo\" src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" width=\"32px\"><br> View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
"</tr></tbody></table>"
|
||||
"\u003ctable\u003e\u003ctbody\u003e\u003ctr\u003e\n",
|
||||
" \u003ctd style=\"text-align: center\"\u003e\n",
|
||||
" \u003ca href=\"https://console.cloud.google.com/vertex-ai/colab/import/https:%2F%2Fraw.githubusercontent.com%2FGoogleCloudPlatform%2Fvertex-ai-samples%2Fmain%2Fnotebooks%2Fcommunity%2Fmodel_garden%2Fmodel_garden_finetuning_tutorial.ipynb\"\u003e\n",
|
||||
" \u003cimg alt=\"Google Cloud Colab Enterprise logo\" src=\"https://lh3.googleusercontent.com/JmcxdQi-qOpctIvWKgPtrzZdJJK-J3sWE1RsfjZNwshCFgE_9fULcNpuXYTilIR2hjwN\" width=\"32px\"\u003e\u003cbr\u003e Run in Colab Enterprise\n",
|
||||
" \u003c/a\u003e\n",
|
||||
" \u003c/td\u003e\n",
|
||||
" \u003ctd style=\"text-align: center\"\u003e\n",
|
||||
" \u003ca href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_finetuning_tutorial.ipynb\"\u003e\n",
|
||||
" \u003cimg alt=\"GitHub logo\" src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" width=\"32px\"\u003e\u003cbr\u003e View on GitHub\n",
|
||||
" \u003c/a\u003e\n",
|
||||
" \u003c/td\u003e\n",
|
||||
"\u003c/tr\u003e\u003c/tbody\u003e\u003c/table\u003e"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -168,11 +168,11 @@
|
||||
"\n",
|
||||
"# @markdown 1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
|
||||
"\n",
|
||||
"# @markdown 2. For finetuning, **[click here](https://console.cloud.google.com/iam-admin/quotas?location=us-central1&metric=aiplatform.googleapis.com%2Frestricted_image_training_nvidia_a100_80gb_gpus)** to check if your project already has the required 8 Nvidia A100 80 GB GPUs in the us-central1 region. If yes, then run this notebook in the us-central1 region. If you do not have 8 Nvidia A100 80 GPUs or have more GPU requirements than this, then schedule your job with Nvidia H100 GPUs via Dynamic Workload Scheduler using [these instructions](https://cloud.google.com/vertex-ai/docs/training/schedule-jobs-dws). For Dynamic Workload Scheduler, check the [us-central1](https://console.cloud.google.com/iam-admin/quotas?location=us-central1&metric=aiplatform.googleapis.com%2Fcustom_model_training_preemptible_nvidia_h100_gpus) or [europe-west4](https://console.cloud.google.com/iam-admin/quotas?location=europe-west4&metric=aiplatform.googleapis.com%2Fcustom_model_training_preemptible_nvidia_h100_gpus) quota for Nvidia H100 GPUs. If you do not have enough GPUs, then you can follow [these instructions](https://cloud.google.com/docs/quotas/view-manage#viewing_your_quota_console) to request quota.\n",
|
||||
"# @markdown 2. For finetuning, **[click here](https://console.cloud.google.com/iam-admin/quotas?location=us-central1\u0026metric=aiplatform.googleapis.com%2Frestricted_image_training_nvidia_a100_80gb_gpus)** to check if your project already has the required 8 Nvidia A100 80 GB GPUs in the us-central1 region. If yes, then run this notebook in the us-central1 region. If you do not have 8 Nvidia A100 80 GPUs or have more GPU requirements than this, then schedule your job with Nvidia H100 GPUs via Dynamic Workload Scheduler using [these instructions](https://cloud.google.com/vertex-ai/docs/training/schedule-jobs-dws). For Dynamic Workload Scheduler, check the [us-central1](https://console.cloud.google.com/iam-admin/quotas?location=us-central1\u0026metric=aiplatform.googleapis.com%2Fcustom_model_training_preemptible_nvidia_h100_gpus) or [europe-west4](https://console.cloud.google.com/iam-admin/quotas?location=europe-west4\u0026metric=aiplatform.googleapis.com%2Fcustom_model_training_preemptible_nvidia_h100_gpus) quota for Nvidia H100 GPUs. If you do not have enough GPUs, then you can follow [these instructions](https://cloud.google.com/docs/quotas/view-manage#viewing_your_quota_console) to request quota.\n",
|
||||
"\n",
|
||||
"# @markdown 3. For evaluation and deployment, **[click here](https://console.cloud.google.com/iam-admin/quotas?location=us-central1&metric=aiplatform.googleapis.com%2Fcustom_model_serving_nvidia_l4_gpus)** to check if your project already has the required 1 L4 GPU in the us-central1 region. If yes, then run this notebook in the us-central1 region. If you need more L4 GPUs for your project, then you can follow [these instructions](https://cloud.google.com/docs/quotas/view-manage#viewing_your_quota_console) to request more. Alternatively, if you want to run predictions with A100 80GB or H100 GPUs, we recommend using the regions listed below. **NOTE:** Make sure you have associated quota in selected regions. Click the links to see your current quota for each GPU type: [Nvidia A100 80GB](https://console.cloud.google.com/iam-admin/quotas?metric=aiplatform.googleapis.com%2Fcustom_model_serving_nvidia_a100_80gb_gpus), [Nvidia H100 80GB](https://console.cloud.google.com/iam-admin/quotas?metric=aiplatform.googleapis.com%2Fcustom_model_serving_nvidia_h100_gpus).\n",
|
||||
"# @markdown 3. For evaluation and deployment, **[click here](https://console.cloud.google.com/iam-admin/quotas?location=us-central1\u0026metric=aiplatform.googleapis.com%2Fcustom_model_serving_nvidia_l4_gpus)** to check if your project already has the required 1 L4 GPU in the us-central1 region. If yes, then run this notebook in the us-central1 region. If you need more L4 GPUs for your project, then you can follow [these instructions](https://cloud.google.com/docs/quotas/view-manage#viewing_your_quota_console) to request more. Alternatively, if you want to run predictions with A100 80GB or H100 GPUs, we recommend using the regions listed below. **NOTE:** Make sure you have associated quota in selected regions. Click the links to see your current quota for each GPU type: [Nvidia A100 80GB](https://console.cloud.google.com/iam-admin/quotas?metric=aiplatform.googleapis.com%2Fcustom_model_serving_nvidia_a100_80gb_gpus), [Nvidia H100 80GB](https://console.cloud.google.com/iam-admin/quotas?metric=aiplatform.googleapis.com%2Fcustom_model_serving_nvidia_h100_gpus).\n",
|
||||
"\n",
|
||||
"# @markdown > | Machine Type | Accelerator Type | Recommended Regions |\n",
|
||||
"# @markdown \u003e | Machine Type | Accelerator Type | Recommended Regions |\n",
|
||||
"# @markdown | ----------- | ----------- | ----------- |\n",
|
||||
"# @markdown | a2-ultragpu-1g | 1 NVIDIA_A100_80GB | us-central1, us-east4, europe-west4, asia-southeast1, us-east4 |\n",
|
||||
"# @markdown | a3-highgpu-2g | 2 NVIDIA_H100_80GB | us-west1, asia-southeast1, europe-west4 |\n",
|
||||
@@ -188,8 +188,8 @@
|
||||
"REGION = \"\" # @param {type:\"string\"}\n",
|
||||
"\n",
|
||||
"# Import the necessary packages.\n",
|
||||
"! rm -rf vertex-ai-samples && git clone https://github.com/GoogleCloudPlatform/vertex-ai-samples.git\n",
|
||||
"! cd vertex-ai-samples && git reset --hard 0727e19520cf7957bceb701c248221bd3dbe4f1f\n",
|
||||
"! rm -rf vertex-ai-samples \u0026\u0026 git clone https://github.com/GoogleCloudPlatform/vertex-ai-samples.git\n",
|
||||
"! cd vertex-ai-samples \u0026\u0026 git reset --hard 0727e19520cf7957bceb701c248221bd3dbe4f1f\n",
|
||||
"\n",
|
||||
"import datetime\n",
|
||||
"import importlib\n",
|
||||
@@ -269,13 +269,10 @@
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "36c21f10355f"
|
||||
"id": "VUSi9jUcvdBC"
|
||||
},
|
||||
"outputs": [],
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"# @title Access Llama 3.1 models\n",
|
||||
"\n",
|
||||
@@ -288,7 +285,7 @@
|
||||
"base_model_id = \"meta-llama/Llama-3.1-8B-Instruct\" # @param {type: \"string\"}\n",
|
||||
"pretrained_model_id = base_model_id\n",
|
||||
"\n",
|
||||
"# @markdown Additionally, you must provide a Hugging Face User Access Token (read) to access the Llama 3.1 models. You can follow the [Hugging Face documentation](https://huggingface.co/docs/hub/en/security-tokens) to create a **read** access token and put it in the `HF_TOKEN` field below.\n",
|
||||
"# @markdown Additionally, you must provide a Hugging Face User Access Token (with read access) to access the Llama 3.1 models. You can follow the [Hugging Face documentation](https://huggingface.co/docs/hub/en/security-tokens) to create a **read** access token and put it in the `HF_TOKEN` field below.\n",
|
||||
"\n",
|
||||
"HF_TOKEN = \"\" # @param {type:\"string\", isTemplate:true}\n",
|
||||
"\n",
|
||||
@@ -296,7 +293,9 @@
|
||||
" assert (\n",
|
||||
" HF_TOKEN\n",
|
||||
" ), \"Provide a read access HF_TOKEN to load models from Hugging Face, or select a different model source. You can comment out this assert statement to skip this check.\""
|
||||
]
|
||||
],
|
||||
"outputs": [],
|
||||
"execution_count": null
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
@@ -323,7 +322,7 @@
|
||||
"| coqa | 0.1158 | 0.0137 | 0.1872 | 0.0150 |\n"
|
||||
],
|
||||
"text/plain": [
|
||||
"<IPython.core.display.Markdown object>"
|
||||
"\u003cIPython.core.display.Markdown object\u003e"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
@@ -359,7 +358,7 @@
|
||||
" lora_path: str = None,\n",
|
||||
" max_num_seqs: int = 64,\n",
|
||||
" eval_task: str = \"coqa\",\n",
|
||||
") -> str:\n",
|
||||
") -\u003e str:\n",
|
||||
" \"\"\"Run lm-evaluation-harness to evaluate the model, and returns .\"\"\"\n",
|
||||
"\n",
|
||||
" if accelerator_type == \"NVIDIA_L4\":\n",
|
||||
@@ -607,7 +606,7 @@
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Downloading readme: 0%| | 0.00/8.20k [00:00<?, ?B/s]"
|
||||
"Downloading readme: 0%| | 0.00/8.20k [00:00\u003c?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
@@ -621,7 +620,7 @@
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Downloading data: 0%| | 0.00/13.1M [00:00<?, ?B/s]"
|
||||
"Downloading data: 0%| | 0.00/13.1M [00:00\u003c?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
@@ -635,7 +634,7 @@
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Generating train split: 0%| | 0/15011 [00:00<?, ? examples/s]"
|
||||
"Generating train split: 0%| | 0/15011 [00:00\u003c?, ? examples/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
@@ -653,7 +652,7 @@
|
||||
"| When was Tomoaki Komorida born? | Komorida was born in Kumamoto Prefecture on July 10, 1981. After graduating from high school, he joined the J1 League club Avispa Fukuoka in 2000. Although he debuted as a midfielder in 2001, he did not play much and the club was relegated to the J2 League at the end of the 2001 season. In 2002, he moved to the J2 club Oita Trinita. He became a regular player as a defensive midfielder and the club won the championship in 2002 and was promoted in 2003. He played many matches until 2005. In September 2005, he moved to the J2 club Montedio Yamagata. In 2006, he moved to the J2 club Vissel Kobe. Although he became a regular player as a defensive midfielder, his gradually was played less during the summer. In 2007, he moved to the Japan Football League club Rosso Kumamoto (later Roasso Kumamoto) based in his local region. He played as a regular player and the club was promoted to J2 in 2008. Although he did not play as much, he still played in many matches. In 2010, he moved to Indonesia and joined Persela Lamongan. In July 2010, he returned to Japan and joined the J2 club Giravanz Kitakyushu. He played often as a defensive midfielder and center back until 2012 when he retired. | Tomoaki Komorida was born on July 10,1981. | closed_qa |"
|
||||
],
|
||||
"text/plain": [
|
||||
"<IPython.core.display.Markdown object>"
|
||||
"\u003cIPython.core.display.Markdown object\u003e"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
@@ -768,14 +767,14 @@
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"('<|begin_of_text|><|start_header_id|>system<|end_header_id|>\\n'\n",
|
||||
"('\u003c|begin_of_text|\u003e\u003c|start_header_id|\u003esystem\u003c|end_header_id|\u003e\\n'\n",
|
||||
" '\\n'\n",
|
||||
" 'You are a helpful '\n",
|
||||
" 'assistant.<|eot_id|><|start_header_id|>user<|end_header_id|>\\n'\n",
|
||||
" 'assistant.\u003c|eot_id|\u003e\u003c|start_header_id|\u003euser\u003c|end_header_id|\u003e\\n'\n",
|
||||
" '\\n'\n",
|
||||
" 'Hello, how are you?<|eot_id|><|start_header_id|>assistant<|end_header_id|>\\n'\n",
|
||||
" 'Hello, how are you?\u003c|eot_id|\u003e\u003c|start_header_id|\u003eassistant\u003c|end_header_id|\u003e\\n'\n",
|
||||
" '\\n'\n",
|
||||
" 'I am doing well, thank you.<|eot_id|>')\n"
|
||||
" 'I am doing well, thank you.\u003c|eot_id|\u003e')\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
@@ -794,7 +793,7 @@
|
||||
"# @markdown to translate the Jinja template for you. For example, you can ask:\n",
|
||||
"\n",
|
||||
"# @markdown ````\n",
|
||||
"# @markdown Translate the following Jinja template to Python, where bos_token is \"<|begin_of_text|>\":\n",
|
||||
"# @markdown Translate the following Jinja template to Python, where bos_token is \"\u003c|begin_of_text|\u003e\":\n",
|
||||
"# @markdown ```\n",
|
||||
"# @markdown {{- bos_token }}\n",
|
||||
"# @markdown {#- This block extracts the system message, so we can slot it into the right place. #}\n",
|
||||
@@ -805,14 +804,14 @@
|
||||
"# @markdown {%- set system_message = \"\" %}\n",
|
||||
"# @markdown {%- endif %}\n",
|
||||
"# @markdown {#- System message #}\n",
|
||||
"# @markdown {{- \"<|start_header_id|>system<|end_header_id|>\\n\\n\" }}\n",
|
||||
"# @markdown {{- \"\u003c|start_header_id|\u003esystem\u003c|end_header_id|\u003e\\n\\n\" }}\n",
|
||||
"# @markdown {{- system_message }}\n",
|
||||
"# @markdown {{- \"<|eot_id|>\" }}\n",
|
||||
"# @markdown {{- \"\u003c|eot_id|\u003e\" }}\n",
|
||||
"# @markdown {%- for message in messages %}\n",
|
||||
"# @markdown {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\\n\\n'+ message['content'] | trim + '<|eot_id|>' }}\n",
|
||||
"# @markdown {{- '\u003c|start_header_id|\u003e' + message['role'] + '\u003c|end_header_id|\u003e\\n\\n'+ message['content'] | trim + '\u003c|eot_id|\u003e' }}\n",
|
||||
"# @markdown {%- endfor %}\n",
|
||||
"# @markdown {%- if add_generation_prompt %}\n",
|
||||
"# @markdown {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }}\n",
|
||||
"# @markdown {{- '\u003c|start_header_id|\u003eassistant\u003c|end_header_id|\u003e\\n\\n' }}\n",
|
||||
"# @markdown {%- endif %}\n",
|
||||
"# @markdown ```\n",
|
||||
"# @markdown ````\n",
|
||||
@@ -821,7 +820,7 @@
|
||||
"# @markdown `bos_token` is specified in the [tokenizer_config.json](https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct/blob/main/tokenizer_config.json#L2052).\n",
|
||||
"# @markdown ```\n",
|
||||
"# @markdown def render_template(messages, add_generation_prompt=False):\n",
|
||||
"# @markdown bos_token = \"<|begin_of_text|>\"\n",
|
||||
"# @markdown bos_token = \"\u003c|begin_of_text|\u003e\"\n",
|
||||
"# @markdown output = bos_token\n",
|
||||
"# @markdown\n",
|
||||
"# @markdown system_message = \"\"\n",
|
||||
@@ -829,22 +828,22 @@
|
||||
"# @markdown system_message = messages[0]['content'].strip()\n",
|
||||
"# @markdown messages = messages[1:]\n",
|
||||
"# @markdown\n",
|
||||
"# @markdown output += \"<|start_header_id|>system<|end_header_id|>\\n\\n\"\n",
|
||||
"# @markdown output += \"\u003c|start_header_id|\u003esystem\u003c|end_header_id|\u003e\\n\\n\"\n",
|
||||
"# @markdown output += system_message\n",
|
||||
"# @markdown output += \"<|eot_id|>\"\n",
|
||||
"# @markdown output += \"\u003c|eot_id|\u003e\"\n",
|
||||
"# @markdown\n",
|
||||
"# @markdown for message in messages:\n",
|
||||
"# @markdown output += f\"<|start_header_id|>{message['role']}<|end_header_id|>\\n\\n{message['content'].strip()}<|eot_id|>\"\n",
|
||||
"# @markdown output += f\"\u003c|start_header_id|\u003e{message['role']}\u003c|end_header_id|\u003e\\n\\n{message['content'].strip()}\u003c|eot_id|\u003e\"\n",
|
||||
"# @markdown\n",
|
||||
"# @markdown if add_generation_prompt:\n",
|
||||
"# @markdown output += \"<|start_header_id|>assistant<|end_header_id|>\\n\\n\"\n",
|
||||
"# @markdown output += \"\u003c|start_header_id|\u003eassistant\u003c|end_header_id|\u003e\\n\\n\"\n",
|
||||
"# @markdown\n",
|
||||
"# @markdown return output\n",
|
||||
"# @markdown ```\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def render_template(messages, add_generation_prompt=False):\n",
|
||||
" bos_token = \"<|begin_of_text|>\"\n",
|
||||
" bos_token = \"\u003c|begin_of_text|\u003e\"\n",
|
||||
" output = bos_token\n",
|
||||
"\n",
|
||||
" system_message = \"\"\n",
|
||||
@@ -852,26 +851,26 @@
|
||||
" system_message = messages[0][\"content\"].strip()\n",
|
||||
" messages = messages[1:]\n",
|
||||
"\n",
|
||||
" output += \"<|start_header_id|>system<|end_header_id|>\\n\\n\"\n",
|
||||
" output += \"\u003c|start_header_id|\u003esystem\u003c|end_header_id|\u003e\\n\\n\"\n",
|
||||
" output += system_message\n",
|
||||
" output += \"<|eot_id|>\"\n",
|
||||
" output += \"\u003c|eot_id|\u003e\"\n",
|
||||
"\n",
|
||||
" for message in messages:\n",
|
||||
" output += (\n",
|
||||
" \"<|start_header_id|>\"\n",
|
||||
" \"\u003c|start_header_id|\u003e\"\n",
|
||||
" + message[\"role\"]\n",
|
||||
" + \"<|end_header_id|>\\n\\n\"\n",
|
||||
" + \"\u003c|end_header_id|\u003e\\n\\n\"\n",
|
||||
" + message[\"content\"].strip()\n",
|
||||
" + \"<|eot_id|>\"\n",
|
||||
" + \"\u003c|eot_id|\u003e\"\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" if add_generation_prompt:\n",
|
||||
" output += \"<|start_header_id|>assistant<|end_header_id|>\\n\\n\"\n",
|
||||
" output += \"\u003c|start_header_id|\u003eassistant\u003c|end_header_id|\u003e\\n\\n\"\n",
|
||||
"\n",
|
||||
" return output\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# @markdown The `<|begin_of_text|>`, `<|start_header_id|>`, `<|end_header_id|>`, and `<|eot_id|>` tokens serve specific purposes in the context of text generation models.\n",
|
||||
"# @markdown The `\u003c|begin_of_text|\u003e`, `\u003c|start_header_id|\u003e`, `\u003c|end_header_id|\u003e`, and `\u003c|eot_id|\u003e` tokens serve specific purposes in the context of text generation models.\n",
|
||||
"# @markdown These tokens help the model delineate the boundaries of a text generation task. They provide clear markers for the start and end points, enabling the model to function effectively and produce coherent text.\n",
|
||||
"\n",
|
||||
"# @markdown Run this cell to show an example output of the template given the\n",
|
||||
@@ -1030,15 +1029,15 @@
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"('<|begin_of_text|><|start_header_id|>user<|end_header_id|>\\n'\n",
|
||||
"('\u003c|begin_of_text|\u003e\u003c|start_header_id|\u003euser\u003c|end_header_id|\u003e\\n'\n",
|
||||
" '\\n'\n",
|
||||
" 'Hello, how are you? Context: This is a test '\n",
|
||||
" 'context.<|eot_id|><|start_header_id|>assistant<|end_header_id|>\\n'\n",
|
||||
" 'context.\u003c|eot_id|\u003e\u003c|start_header_id|\u003eassistant\u003c|end_header_id|\u003e\\n'\n",
|
||||
" '\\n'\n",
|
||||
" \"I'm doing well, thank \"\n",
|
||||
" 'you!<|eot_id|><|start_header_id|>user<|end_header_id|>\\n'\n",
|
||||
" 'you!\u003c|eot_id|\u003e\u003c|start_header_id|\u003euser\u003c|end_header_id|\u003e\\n'\n",
|
||||
" '\\n'\n",
|
||||
" 'Another question without context.<|eot_id|>')\n"
|
||||
" 'Another question without context.\u003c|eot_id|\u003e')\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
@@ -1052,15 +1051,15 @@
|
||||
"# @markdown chat_template_string = r\"\"\"{{- bos_token }}\n",
|
||||
"# @markdown\n",
|
||||
"# @markdown {% for message in messages %}\n",
|
||||
"# @markdown {{- '<|start_header_id|>' + message.role + '<|end_header_id|>\\n\\n' + message.content | trim }}\n",
|
||||
"# @markdown {% if message.context and message.context | length > 0 %}\n",
|
||||
"# @markdown {{- '\u003c|start_header_id|\u003e' + message.role + '\u003c|end_header_id|\u003e\\n\\n' + message.content | trim }}\n",
|
||||
"# @markdown {% if message.context and message.context | length \u003e 0 %}\n",
|
||||
"# @markdown {{- ' Context: ' + message.context }}\n",
|
||||
"# @markdown {% endif %}\n",
|
||||
"# @markdown {{- '<|eot_id|>' }}\n",
|
||||
"# @markdown {{- '\u003c|eot_id|\u003e' }}\n",
|
||||
"# @markdown {% endfor %}\n",
|
||||
"# @markdown\n",
|
||||
"# @markdown {% if add_generation_prompt %}\n",
|
||||
"# @markdown {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }}\n",
|
||||
"# @markdown {{- '\u003c|start_header_id|\u003eassistant\u003c|end_header_id|\u003e\\n\\n' }}\n",
|
||||
"# @markdown {% endif %}\n",
|
||||
"# @markdown \"\"\"\n",
|
||||
"# @markdown ```\n",
|
||||
@@ -1071,15 +1070,15 @@
|
||||
"chat_template_string = r\"\"\"{{- bos_token }}\n",
|
||||
"\n",
|
||||
"{% for message in messages %}\n",
|
||||
" {{- '<|start_header_id|>' + message.role + '<|end_header_id|>\\n\\n' + message.content | trim }}\n",
|
||||
" {% if message.context and message.context | length > 0 %}\n",
|
||||
" {{- '\u003c|start_header_id|\u003e' + message.role + '\u003c|end_header_id|\u003e\\n\\n' + message.content | trim }}\n",
|
||||
" {% if message.context and message.context | length \u003e 0 %}\n",
|
||||
" {{- ' Context: ' + message.context }}\n",
|
||||
" {% endif %}\n",
|
||||
" {{- '<|eot_id|>' }}\n",
|
||||
" {{- '\u003c|eot_id|\u003e' }}\n",
|
||||
"{% endfor %}\n",
|
||||
"\n",
|
||||
"{% if add_generation_prompt %}\n",
|
||||
" {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }}\n",
|
||||
" {{- '\u003c|start_header_id|\u003eassistant\u003c|end_header_id|\u003e\\n\\n' }}\n",
|
||||
"{% endif %}\n",
|
||||
"\"\"\"\n",
|
||||
"\n",
|
||||
@@ -1089,19 +1088,19 @@
|
||||
"# @markdown output = bos_token\n",
|
||||
"# @markdown\n",
|
||||
"# @markdown for message in messages:\n",
|
||||
"# @markdown output += f\"<|start_header_id|>{message['role']}<|end_header_id|>\\n\\n{message['content'].strip()}\"\n",
|
||||
"# @markdown output += f\"\u003c|start_header_id|\u003e{message['role']}\u003c|end_header_id|\u003e\\n\\n{message['content'].strip()}\"\n",
|
||||
"# @markdown if 'context' in message and message['context']:\n",
|
||||
"# @markdown output += f\" Context: {message['context']}\"\n",
|
||||
"# @markdown output += \"<|eot_id|>\"\n",
|
||||
"# @markdown output += \"\u003c|eot_id|\u003e\"\n",
|
||||
"# @markdown\n",
|
||||
"# @markdown if add_generation_prompt:\n",
|
||||
"# @markdown output += \"<|start_header_id|>assistant<|end_header_id|>\\n\\n\"\n",
|
||||
"# @markdown output += \"\u003c|start_header_id|\u003eassistant\u003c|end_header_id|\u003e\\n\\n\"\n",
|
||||
"# @markdown\n",
|
||||
"# @markdown return output\n",
|
||||
"# @markdown ```\n",
|
||||
"\n",
|
||||
"# @markdown Run this cell to show an example output of the template given the\n",
|
||||
"# @markdown below `messages` and `bos_token=\"<|begin_of_text|>\"`.\n",
|
||||
"# @markdown below `messages` and `bos_token=\"\u003c|begin_of_text|\u003e\"`.\n",
|
||||
"# @markdown ```\n",
|
||||
"# @markdown messages = [\n",
|
||||
"# @markdown {\"role\": \"user\", \"content\": \"Hello, how are you?\", \"context\": \"This is a test context.\"},\n",
|
||||
@@ -1117,13 +1116,13 @@
|
||||
" output = bos_token\n",
|
||||
"\n",
|
||||
" for message in messages:\n",
|
||||
" output += f\"<|start_header_id|>{message['role']}<|end_header_id|>\\n\\n{message['content'].strip()}\"\n",
|
||||
" output += f\"\u003c|start_header_id|\u003e{message['role']}\u003c|end_header_id|\u003e\\n\\n{message['content'].strip()}\"\n",
|
||||
" if \"context\" in message and message[\"context\"]:\n",
|
||||
" output += f\" Context: {message['context']}\"\n",
|
||||
" output += \"<|eot_id|>\"\n",
|
||||
" output += \"\u003c|eot_id|\u003e\"\n",
|
||||
"\n",
|
||||
" if add_generation_prompt:\n",
|
||||
" output += \"<|start_header_id|>assistant<|end_header_id|>\\n\\n\"\n",
|
||||
" output += \"\u003c|start_header_id|\u003eassistant\u003c|end_header_id|\u003e\\n\\n\"\n",
|
||||
"\n",
|
||||
" return output\n",
|
||||
"\n",
|
||||
@@ -1138,7 +1137,7 @@
|
||||
" {\"role\": \"user\", \"content\": \"Another question without context.\", \"context\": \"\"},\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"rendered_text = render_template(messages, bos_token=\"<|begin_of_text|>\")\n",
|
||||
"rendered_text = render_template(messages, bos_token=\"\u003c|begin_of_text|\u003e\")\n",
|
||||
"pprint.pprint(rendered_text, width=80)"
|
||||
]
|
||||
},
|
||||
@@ -1173,16 +1172,16 @@
|
||||
"# @markdown template = {\n",
|
||||
"# @markdown \"description\": \"Template used by Llama 3.1, accepting databricks dolly dataset.\",\n",
|
||||
"# @markdown \"chat_template\": chat_template_string,\n",
|
||||
"# @markdown \"instruction_separator\": \"<|start_header_id|>user<|end_header_id|>\\n\\n\",\n",
|
||||
"# @markdown \"response_separator\": \"<|start_header_id|>assistant<|end_header_id|>\\n\\n\"\n",
|
||||
"# @markdown \"instruction_separator\": \"\u003c|start_header_id|\u003euser\u003c|end_header_id|\u003e\\n\\n\",\n",
|
||||
"# @markdown \"response_separator\": \"\u003c|start_header_id|\u003eassistant\u003c|end_header_id|\u003e\\n\\n\"\n",
|
||||
"# @markdown }\n",
|
||||
"# @markdown ```\n",
|
||||
"\n",
|
||||
"template_data = {\n",
|
||||
" \"description\": \"Template used by Llama 3.1, accepting databricks dolly dataset.\",\n",
|
||||
" \"chat_template\": chat_template_string,\n",
|
||||
" \"instruction_separator\": \"<|start_header_id|>user<|end_header_id|>\\n\\n\",\n",
|
||||
" \"response_separator\": \"<|start_header_id|>assistant<|end_header_id|>\\n\\n\",\n",
|
||||
" \"instruction_separator\": \"\u003c|start_header_id|\u003euser\u003c|end_header_id|\u003e\\n\\n\",\n",
|
||||
" \"response_separator\": \"\u003c|start_header_id|\u003eassistant\u003c|end_header_id|\u003e\\n\\n\",\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"template_filename = \"template.json\"\n",
|
||||
@@ -1346,7 +1345,7 @@
|
||||
"\n",
|
||||
"# @markdown **Note**:\n",
|
||||
"# @markdown 1. We recommend setting `finetuning_precision_mode` to `float16`.\n",
|
||||
"# @markdown 1. If `max_steps>0`, it takes precedence over `epochs`. One can set a small `max_steps`\n",
|
||||
"# @markdown 1. If `max_steps\u003e0`, it takes precedence over `epochs`. One can set a small `max_steps`\n",
|
||||
"# @markdown value to quickly check the pipeline.\n",
|
||||
"\n",
|
||||
"# @markdown Acceletor type to use for training.\n",
|
||||
@@ -1393,7 +1392,7 @@
|
||||
"# Set config file.\n",
|
||||
"if replica_count == 1:\n",
|
||||
" config_file = \"vertex_vision_model_garden_peft/llama_fsdp_8gpu.yaml\"\n",
|
||||
"elif replica_count <= 4:\n",
|
||||
"elif replica_count \u003c= 4:\n",
|
||||
" config_file = (\n",
|
||||
" \"vertex_vision_model_garden_peft/\"\n",
|
||||
" f\"llama_hsdp_{replica_count * per_node_accelerator_count}gpu.yaml\"\n",
|
||||
@@ -1654,7 +1653,7 @@
|
||||
"\n",
|
||||
"\n",
|
||||
"# @markdown Expected evaluation results:\n",
|
||||
"# @markdown > | alias | exact_match | exact_match_stderr | f1 | f1_stderr |\n",
|
||||
"# @markdown \u003e | alias | exact_match | exact_match_stderr | f1 | f1_stderr |\n",
|
||||
"# @markdown | --- | --- | --- | --- | --- |\n",
|
||||
"# @markdown | coqa | 0.3213 | 0.0197 | 0.4660 | 0.0187 |\n",
|
||||
"\n",
|
||||
@@ -1710,14 +1709,14 @@
|
||||
" is_for_training=False,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# Dedicated endpoint not supported yet.\n",
|
||||
"use_dedicated_endpoint = False\n",
|
||||
"# @markdown Set `use_dedicated_endpoint` to False if you don't want to use [dedicated endpoint](https://cloud.google.com/vertex-ai/docs/general/deployment#create-dedicated-endpoint).\n",
|
||||
"use_dedicated_endpoint = True # @param {type:\"boolean\"}\n",
|
||||
"\n",
|
||||
"gpu_memory_utilization = 0.85\n",
|
||||
"max_model_len = 8192 # Maximum context length.\n",
|
||||
"\n",
|
||||
"# Ensure max_model_len does not exceed the limit.\n",
|
||||
"if max_model_len > 8192:\n",
|
||||
"if max_model_len \u003e 8192:\n",
|
||||
" raise ValueError(\"max_model_len cannot exceed 8192\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
@@ -1743,7 +1742,7 @@
|
||||
" use_dedicated_endpoint: bool = False,\n",
|
||||
" max_num_seqs: int = 256,\n",
|
||||
" model_type: str = None,\n",
|
||||
") -> Tuple[aiplatform.Model, aiplatform.Endpoint]:\n",
|
||||
") -\u003e Tuple[aiplatform.Model, aiplatform.Endpoint]:\n",
|
||||
" \"\"\"Deploys trained models with vLLM into Vertex AI.\"\"\"\n",
|
||||
" endpoint = aiplatform.Endpoint.create(\n",
|
||||
" display_name=f\"{model_name}-endpoint\",\n",
|
||||
@@ -1787,7 +1786,7 @@
|
||||
" if enable_prefix_cache:\n",
|
||||
" vllm_args.append(\"--enable-prefix-caching\")\n",
|
||||
"\n",
|
||||
" if 0 < host_prefix_kv_cache_utilization_target < 1:\n",
|
||||
" if 0 \u003c host_prefix_kv_cache_utilization_target \u003c 1:\n",
|
||||
" vllm_args.append(\n",
|
||||
" f\"--host-prefix-kv-cache-utilization-target={host_prefix_kv_cache_utilization_target}\"\n",
|
||||
" )\n",
|
||||
@@ -1846,6 +1845,7 @@
|
||||
" top_k: int,\n",
|
||||
" raw_response: bool,\n",
|
||||
" lora_weight: str = \"\",\n",
|
||||
" use_dedicated_endpoint: bool = False,\n",
|
||||
"):\n",
|
||||
" # Parameters for inference.\n",
|
||||
" instance = {\n",
|
||||
@@ -1901,7 +1901,7 @@
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Prompt:\n",
|
||||
"<|start_header_id|>user<|end_header_id|>\\n\\nWhat was Anya looking for? Context: Anya clutched the worn teddy bear, its button eye dangling precariously. She'd lost it in the park yesterday, and the thought of never seeing Mr. Snuggles again made her tummy ache. She retraced her steps, her eyes scanning the colorful playground equipment and the sprawling green lawn.<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n",
|
||||
"\u003c|start_header_id|\u003euser\u003c|end_header_id|\u003e\\n\\nWhat was Anya looking for? Context: Anya clutched the worn teddy bear, its button eye dangling precariously. She'd lost it in the park yesterday, and the thought of never seeing Mr. Snuggles again made her tummy ache. She retraced her steps, her eyes scanning the colorful playground equipment and the sprawling green lawn.\u003c|eot_id|\u003e\u003c|start_header_id|\u003eassistant\u003c|end_header_id|\u003e\n",
|
||||
"Output:\n",
|
||||
"Anya was looking for her teddy bear, Mr. Snuggles.\n"
|
||||
]
|
||||
@@ -1925,8 +1925,8 @@
|
||||
"\n",
|
||||
"prompt = \"What was Anya looking for? Context: Anya clutched the worn teddy bear, its button eye dangling precariously. She'd lost it in the park yesterday, and the thought of never seeing Mr. Snuggles again made her tummy ache. She retraced her steps, her eyes scanning the colorful playground equipment and the sprawling green lawn.\" # @param {type: \"string\"}\n",
|
||||
"prompt_with_headers = (\n",
|
||||
" f\"<|start_header_id|>user<|end_header_id|>\\\\n\\\\n{prompt}<|eot_id|>\"\n",
|
||||
" \"<|start_header_id|>assistant<|end_header_id|>\\n\\n\"\n",
|
||||
" f\"\u003c|start_header_id|\u003euser\u003c|end_header_id|\u003e\\\\n\\\\n{prompt}\u003c|eot_id|\u003e\"\n",
|
||||
" \"\u003c|start_header_id|\u003eassistant\u003c|end_header_id|\u003e\\n\\n\"\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# @markdown If you encounter the issue like `ServiceUnavailable: 503 Took too long to respond when processing`, you can reduce the maximum number of output tokens, such as set `max_tokens` as 20.\n",
|
||||
@@ -1945,6 +1945,7 @@
|
||||
" top_k=top_k,\n",
|
||||
" raw_response=raw_response,\n",
|
||||
" lora_weight=lora_output_dir,\n",
|
||||
" use_dedicated_endpoint=use_dedicated_endpoint,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# @markdown Click \"Show Code\" to see more details."
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"language": "python",
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "7d9bbf86da5e"
|
||||
@@ -35,6 +34,11 @@
|
||||
"\n",
|
||||
"<table><tbody><tr>\n",
|
||||
" <td style=\"text-align: center\">\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/workbench/instances\">\n",
|
||||
" <img alt=\"Workbench logo\" src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" width=\"32px\"><br> Run in Workbench\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td style=\"text-align: center\">\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/colab/import/https:%2F%2Fraw.githubusercontent.com%2FGoogleCloudPlatform%2Fvertex-ai-samples%2Fmain%2Fnotebooks%2Fcommunity%2Fmodel_garden%2Fmodel_garden_gemma2_deployment_on_vertex.ipynb\">\n",
|
||||
" <img alt=\"Google Cloud Colab Enterprise logo\" src=\"https://lh3.googleusercontent.com/JmcxdQi-qOpctIvWKgPtrzZdJJK-J3sWE1RsfjZNwshCFgE_9fULcNpuXYTilIR2hjwN\" width=\"32px\"><br> Run in Colab Enterprise\n",
|
||||
" </a>\n",
|
||||
@@ -65,6 +69,10 @@
|
||||
"- Deploy Gemma 2 with Hex-LLM on TPU\n",
|
||||
"- Deploy Gemma with [TGI](https://github.com/huggingface/text-generation-inference) on GPU\n",
|
||||
"\n",
|
||||
"### File a bug\n",
|
||||
"\n",
|
||||
"File a bug on [GitHub](https://github.com/GoogleCloudPlatform/vertex-ai-samples/issues/new) if you encounter any issue with the notebook.\n",
|
||||
"\n",
|
||||
"### Costs\n",
|
||||
"\n",
|
||||
"This tutorial uses billable components of Google Cloud:\n",
|
||||
@@ -111,32 +119,38 @@
|
||||
"\n",
|
||||
"# @markdown 1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
|
||||
"\n",
|
||||
"# @markdown 2. **[Optional]** [Create a Cloud Storage bucket](https://cloud.google.com/storage/docs/creating-buckets) for storing experiment outputs. Set the BUCKET_URI for the experiment environment. The specified Cloud Storage bucket (`BUCKET_URI`) should be located in the same region as where the notebook was launched. Note that a multi-region bucket (eg. \"us\") is not considered a match for a single region covered by the multi-region range (eg. \"us-central1\"). If not set, a unique GCS bucket will be created instead.\n",
|
||||
"\n",
|
||||
"BUCKET_URI = \"gs://\" # @param {type:\"string\"}\n",
|
||||
"\n",
|
||||
"# @markdown 3. **[Optional]** Set region. If not set, the region will be set automatically according to Colab Enterprise environment.\n",
|
||||
"# @markdown 2. **[Optional]** Set region. If not set, the region will be set automatically according to Colab Enterprise environment.\n",
|
||||
"\n",
|
||||
"REGION = \"\" # @param {type:\"string\"}\n",
|
||||
"\n",
|
||||
"# Import the necessary packages\n",
|
||||
"# @markdown 3. If you want to run predictions with A100 80GB or H100 GPUs, we recommend using the regions listed below. **NOTE:** Make sure you have associated quota in selected regions. Click the links to see your current quota for each GPU type: [Nvidia A100 80GB](https://console.cloud.google.com/iam-admin/quotas?metric=aiplatform.googleapis.com%2Fcustom_model_serving_nvidia_a100_80gb_gpus), [Nvidia H100 80GB](https://console.cloud.google.com/iam-admin/quotas?metric=aiplatform.googleapis.com%2Fcustom_model_serving_nvidia_h100_gpus). You can request for quota following the instructions at [\"Request a higher quota\"](https://cloud.google.com/docs/quota/view-manage#requesting_higher_quota).\n",
|
||||
"\n",
|
||||
"# @markdown > | Machine Type | Accelerator Type | Recommended Regions |\n",
|
||||
"# @markdown | ----------- | ----------- | ----------- |\n",
|
||||
"# @markdown | a2-ultragpu-1g | 1 NVIDIA_A100_80GB | us-central1, us-east4, europe-west4, asia-southeast1, us-east4 |\n",
|
||||
"# @markdown | a3-highgpu-2g | 2 NVIDIA_H100_80GB | us-west1, asia-southeast1, europe-west4 |\n",
|
||||
"# @markdown | a3-highgpu-4g | 4 NVIDIA_H100_80GB | us-west1, asia-southeast1, europe-west4 |\n",
|
||||
"# @markdown | a3-highgpu-8g | 8 NVIDIA_H100_80GB | us-central1, europe-west4, us-west1, asia-southeast1 |\n",
|
||||
"\n",
|
||||
"# Upgrade Vertex AI SDK.\n",
|
||||
"! pip3 install --upgrade --quiet 'google-cloud-aiplatform>=1.64.0'\n",
|
||||
"! git clone https://github.com/GoogleCloudPlatform/vertex-ai-samples.git\n",
|
||||
"! pip3 install --upgrade --quiet 'google-cloud-aiplatform>=1.84.0'\n",
|
||||
"\n",
|
||||
"import datetime\n",
|
||||
"import importlib\n",
|
||||
"import os\n",
|
||||
"import uuid\n",
|
||||
"from typing import Tuple\n",
|
||||
"\n",
|
||||
"from google.cloud import aiplatform\n",
|
||||
"\n",
|
||||
"# Upgrade Vertex AI SDK.\n",
|
||||
"if os.environ.get(\"VERTEX_PRODUCT\") != \"COLAB_ENTERPRISE\":\n",
|
||||
" ! pip install --upgrade tensorflow\n",
|
||||
"! git clone https://github.com/GoogleCloudPlatform/vertex-ai-samples.git\n",
|
||||
"\n",
|
||||
"common_util = importlib.import_module(\n",
|
||||
" \"vertex-ai-samples.community-content.vertex_model_garden.model_oss.notebook_util.common_util\"\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"LABEL = \"tgi\"\n",
|
||||
"models, endpoints = {}, {}\n",
|
||||
"\n",
|
||||
"# Get the default cloud project id.\n",
|
||||
@@ -144,64 +158,26 @@
|
||||
"\n",
|
||||
"# Get the default region for launching jobs.\n",
|
||||
"if not REGION:\n",
|
||||
" if not os.environ.get(\"GOOGLE_CLOUD_REGION\"):\n",
|
||||
" raise ValueError(\n",
|
||||
" \"REGION must be set. See\"\n",
|
||||
" \" https://cloud.google.com/vertex-ai/docs/general/locations for\"\n",
|
||||
" \" available cloud locations.\"\n",
|
||||
" )\n",
|
||||
" REGION = os.environ[\"GOOGLE_CLOUD_REGION\"]\n",
|
||||
"\n",
|
||||
"# Enable the Vertex AI API and Compute Engine API, if not already.\n",
|
||||
"print(\"Enabling Vertex AI API and Compute Engine API.\")\n",
|
||||
"! gcloud services enable aiplatform.googleapis.com compute.googleapis.com\n",
|
||||
"\n",
|
||||
"# Cloud Storage bucket for storing the experiment artifacts.\n",
|
||||
"# A unique GCS bucket will be created for the purpose of this notebook. If you\n",
|
||||
"# prefer using your own GCS bucket, change the value yourself below.\n",
|
||||
"now = datetime.datetime.now().strftime(\"%Y%m%d%H%M%S\")\n",
|
||||
"BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n",
|
||||
"\n",
|
||||
"if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n",
|
||||
" BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n",
|
||||
" BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n",
|
||||
" ! gsutil mb -l {REGION} {BUCKET_URI}\n",
|
||||
"else:\n",
|
||||
" assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n",
|
||||
" shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n",
|
||||
" bucket_region = shell_output[0].strip().lower()\n",
|
||||
" if bucket_region != REGION:\n",
|
||||
" raise ValueError(\n",
|
||||
" \"Bucket region %s is different from notebook region %s\"\n",
|
||||
" % (bucket_region, REGION)\n",
|
||||
" )\n",
|
||||
"print(f\"Using this GCS Bucket: {BUCKET_URI}\")\n",
|
||||
"\n",
|
||||
"STAGING_BUCKET = os.path.join(BUCKET_URI, \"temporal\")\n",
|
||||
"MODEL_BUCKET = os.path.join(BUCKET_URI, \"gemma2\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Initialize Vertex AI API.\n",
|
||||
"print(\"Initializing Vertex AI API.\")\n",
|
||||
"aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=STAGING_BUCKET)\n",
|
||||
"\n",
|
||||
"# Gets the default SERVICE_ACCOUNT.\n",
|
||||
"shell_output = ! gcloud projects describe $PROJECT_ID\n",
|
||||
"project_number = shell_output[-1].split(\":\")[1].strip().replace(\"'\", \"\")\n",
|
||||
"SERVICE_ACCOUNT = f\"{project_number}-compute@developer.gserviceaccount.com\"\n",
|
||||
"print(\"Using this default Service Account:\", SERVICE_ACCOUNT)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n",
|
||||
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n",
|
||||
"aiplatform.init(project=PROJECT_ID, location=REGION)\n",
|
||||
"\n",
|
||||
"! gcloud config set project $PROJECT_ID\n",
|
||||
"! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n",
|
||||
"! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\"\n",
|
||||
"\n",
|
||||
"import vertexai\n",
|
||||
"\n",
|
||||
"vertexai.init(\n",
|
||||
" project=PROJECT_ID,\n",
|
||||
" location=REGION,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# @markdown Set use_dedicated_endpoint to False if you don't want to use [dedicated endpoint](https://cloud.google.com/vertex-ai/docs/general/deployment#create-dedicated-endpoint). Note that [dedicated endpoint does not support VPC Service Controls](https://cloud.google.com/vertex-ai/docs/predictions/choose-endpoint-type), uncheck the box if you are using VPC-SC.\n",
|
||||
"use_dedicated_endpoint = True # @param {type:\"boolean\"}\n",
|
||||
"# @markdown ## Access Gemma 2 Models\n",
|
||||
"\n",
|
||||
"# @markdown You must provide a Hugging Face User Access Token (read) to access the Gemma 2 models. You can follow the [Hugging Face documentation](https://huggingface.co/docs/hub/en/security-tokens) to create a **read** access token and put it in the `HF_TOKEN` field below.\n",
|
||||
"# @markdown You must provide a Hugging Face User Access Token (with read access) to access the Gemma 2 models. You can follow the [Hugging Face documentation](https://huggingface.co/docs/hub/en/security-tokens) to create a **read** access token and put it in the `HF_TOKEN` field below.\n",
|
||||
"\n",
|
||||
"HF_TOKEN = \"\" # @param {type:\"string\", isTemplate:true}\n",
|
||||
"assert (\n",
|
||||
@@ -229,21 +205,22 @@
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "E8OiHHNNE_wj"
|
||||
"id": "B7bg9nM0S0Mp"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Deploy\n",
|
||||
"# @markdown Set the model ID. Model weights can be loaded from HuggingFace or from a GCS bucket.\n",
|
||||
"\n",
|
||||
"# The pre-built serving docker images.\n",
|
||||
"HEXLLM_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai-restricted/vertex-vision-model-garden-dockers/hex-llm-serve:20241210_2323_RC00\"\n",
|
||||
"# @title Select the model variants\n",
|
||||
"\n",
|
||||
"# @markdown Select one of the four model variations.\n",
|
||||
"MODEL_ID = \"gemma-2-2b-it\" # @param [\"gemma-2-2b\", \"gemma-2-2b-it\", \"gemma-2-9b\", \"gemma-2-9b-it\", \"gemma-2-27b\", \"gemma-2-27b-it\"] {allow-input: true, isTemplate: true}\n",
|
||||
"version_id = f\"publishers/google/models/gemma2/@{MODEL_ID}\"\n",
|
||||
"\n",
|
||||
"TPU_DEPLOYMENT_REGION = \"us-west1\" # @param [\"us-west1\"] {isTemplate:true}\n",
|
||||
"model_id = os.path.join(model_path_prefix, MODEL_ID)\n",
|
||||
"\n",
|
||||
"# @markdown Set use_dedicated_endpoint to False if you don't want to use [dedicated endpoint](https://cloud.google.com/vertex-ai/docs/general/deployment#create-dedicated-endpoint). Note that [dedicated endpoint does not support VPC Service Controls](https://cloud.google.com/vertex-ai/docs/predictions/choose-endpoint-type), uncheck the box if you are using VPC-SC.\n",
|
||||
"use_dedicated_endpoint = True # @param {type:\"boolean\"}\n",
|
||||
"\n",
|
||||
"# @markdown Find Vertex AI prediction TPUv5e machine types in\n",
|
||||
"# @markdown https://cloud.google.com/vertex-ai/docs/predictions/use-tpu#deploy_a_model.\n",
|
||||
"if \"2b\" in model_id:\n",
|
||||
@@ -271,16 +248,29 @@
|
||||
" accelerator_type=accelerator_type,\n",
|
||||
" accelerator_count=accelerator_count,\n",
|
||||
" is_for_training=False,\n",
|
||||
")\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "E8OiHHNNE_wj"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Deploy Gemma2 models with Hex-LLM on TPU\n",
|
||||
"# @markdown Set the model ID. Model weights can be loaded from HuggingFace or from a GCS bucket.\n",
|
||||
"\n",
|
||||
"# The pre-built serving docker images.\n",
|
||||
"HEXLLM_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai-restricted/vertex-vision-model-garden-dockers/hex-llm-serve:20241210_2323_RC00\"\n",
|
||||
"\n",
|
||||
"# Server parameters.\n",
|
||||
"tensor_parallel_size = accelerator_count\n",
|
||||
"hbm_utilization_factor = 0.6 # Fraction of HBM memory allocated for KV cache after model loading. A larger value improves throughput but gives higher risk of TPU out-of-memory errors with long prompts.\n",
|
||||
"max_running_seqs = 256 # Maximum number of running sequences in a continuous batch.\n",
|
||||
"\n",
|
||||
"# @markdown Set `use_dedicated_endpoint` to False if you don't want to use [dedicated endpoint](https://cloud.google.com/vertex-ai/docs/general/deployment#create-dedicated-endpoint).\n",
|
||||
"use_dedicated_endpoint = True # @param {type:\"boolean\"}\n",
|
||||
"\n",
|
||||
"# Endpoint configurations.\n",
|
||||
"min_replica_count = 1\n",
|
||||
"max_replica_count = 1\n",
|
||||
@@ -291,7 +281,6 @@
|
||||
" model_id: str,\n",
|
||||
" publisher: str,\n",
|
||||
" publisher_model_id: str,\n",
|
||||
" service_account: str = None,\n",
|
||||
" base_model_id: str = None,\n",
|
||||
" data_parallel_size: int = 1,\n",
|
||||
" tensor_parallel_size: int = 1,\n",
|
||||
@@ -380,7 +369,6 @@
|
||||
" machine_type=machine_type,\n",
|
||||
" tpu_topology=tpu_topology if num_hosts > 1 else None,\n",
|
||||
" deploy_request_timeout=1800,\n",
|
||||
" service_account=service_account,\n",
|
||||
" min_replica_count=min_replica_count,\n",
|
||||
" max_replica_count=max_replica_count,\n",
|
||||
" system_labels={\n",
|
||||
@@ -396,7 +384,6 @@
|
||||
" model_id=model_id,\n",
|
||||
" publisher=\"google\",\n",
|
||||
" publisher_model_id=\"gemma2\",\n",
|
||||
" service_account=SERVICE_ACCOUNT,\n",
|
||||
" machine_type=machine_type,\n",
|
||||
" tensor_parallel_size=tensor_parallel_size,\n",
|
||||
" hbm_utilization_factor=hbm_utilization_factor,\n",
|
||||
@@ -489,23 +476,28 @@
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "TBNJYZMlBNwZ"
|
||||
"id": "eYst8GHqcGco"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Deploy\n",
|
||||
"# @title Select the model variants\n",
|
||||
"\n",
|
||||
"# The pre-built serving docker image.\n",
|
||||
"TGI_DOCKER_URI = \"us-docker.pkg.dev/deeplearning-platform-release/gcr.io/huggingface-text-generation-inference-cu121.2-1.ubuntu2204.py310\"\n",
|
||||
"\n",
|
||||
"MODEL_ID = \"gemma-2-2b\" # @param [\"gemma-2-2b\", \"gemma-2-2b-it\", \"gemma-2-9b\", \"gemma-2-9b-it\", \"gemma-2-27b\", \"gemma-2-27b-it\"] {allow-input: true, isTemplate: true}\n",
|
||||
"model_id = os.path.join(model_path_prefix, MODEL_ID)\n",
|
||||
"PUBLISHER_MODEL_NAME = f\"publishers/google/models/gemma2@{MODEL_ID}\"\n",
|
||||
"\n",
|
||||
"# @markdown Finds Vertex AI prediction supported accelerators and regions in\n",
|
||||
"# @markdown https://cloud.google.com/vertex-ai/docs/predictions/configure-compute.\n",
|
||||
"\n",
|
||||
"accelerator_type = \"NVIDIA_L4\" # @param [\"NVIDIA_L4\"] {isTemplate: true}\n",
|
||||
"\n",
|
||||
"# @markdown Set use_dedicated_endpoint to False if you don't want to use [dedicated endpoint](https://cloud.google.com/vertex-ai/docs/general/deployment#create-dedicated-endpoint). Note that [dedicated endpoint does not support VPC Service Controls](https://cloud.google.com/vertex-ai/docs/predictions/choose-endpoint-type), uncheck the box if you are using VPC-SC.\n",
|
||||
"use_dedicated_endpoint = True # @param {type:\"boolean\"}\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"if \"2b\" in MODEL_ID:\n",
|
||||
" if accelerator_type == \"NVIDIA_L4\":\n",
|
||||
" # Sets 1 L4 (24G) to deploy Gemma 2 2B models.\n",
|
||||
@@ -547,6 +539,44 @@
|
||||
" is_for_training=False,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# @markdown Click \"Show Code\" to see more details."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "nlRmOQmZhjvp"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title [Option 1] Deploy with Model Garden SDK\n",
|
||||
"\n",
|
||||
"# @markdown Deploy with Gen AI model-centric SDK. This section uploads the prebuilt model to Model Registry and deploys it to a Vertex AI Endpoint. It takes 15 minutes to 1 hour to finish depending on the size of the model. See [use open models with Vertex AI](https://cloud.google.com/vertex-ai/generative-ai/docs/open-models/use-open-models) for documentation on other use cases.\n",
|
||||
"from vertexai.preview import model_garden\n",
|
||||
"\n",
|
||||
"model = model_garden.OpenModel(PUBLISHER_MODEL_NAME)\n",
|
||||
"endpoints[LABEL] = model.deploy(\n",
|
||||
" machine_type=machine_type,\n",
|
||||
" accelerator_type=accelerator_type,\n",
|
||||
" accelerator_count=accelerator_count,\n",
|
||||
" use_dedicated_endpoint=use_dedicated_endpoint,\n",
|
||||
" accept_eula=True, # Accept the End User License Agreement (EULA) on the model card before deploy. Otherwise, the deployment will be forbidden.\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "TBNJYZMlBNwZ"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title [Option 2] Deploy Gemma models with TGI on GPU\n",
|
||||
"\n",
|
||||
"# Note that larger token counts will require more GPU memory. For example, if you'd\n",
|
||||
"# like to increase the `max_total_tokens` and `max_batch_prefill_tokens` to 8192,\n",
|
||||
"# you may need 1 L4 for 2b model, 4 L4s for the 9b model, and 8 L4s for the 27b model.\n",
|
||||
@@ -560,7 +590,7 @@
|
||||
" model_id: str,\n",
|
||||
" publisher: str,\n",
|
||||
" publisher_model_id: str,\n",
|
||||
" service_account: str,\n",
|
||||
" service_account: str = None,\n",
|
||||
" machine_type: str = \"g2-standard-8\",\n",
|
||||
" accelerator_type: str = \"NVIDIA_L4\",\n",
|
||||
" accelerator_count: int = 1,\n",
|
||||
@@ -591,6 +621,9 @@
|
||||
" except NameError:\n",
|
||||
" pass\n",
|
||||
"\n",
|
||||
" if service_account:\n",
|
||||
" env_vars[\"SERVICE_ACCOUNT\"] = service_account\n",
|
||||
"\n",
|
||||
" model = aiplatform.Model.upload(\n",
|
||||
" display_name=model_name,\n",
|
||||
" serving_container_image_uri=TGI_DOCKER_URI,\n",
|
||||
@@ -617,12 +650,11 @@
|
||||
" return model, endpoint\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"models[\"tgi\"], endpoints[\"tgi\"] = deploy_model_tgi(\n",
|
||||
"models[LABEL], endpoints[LABEL] = deploy_model_tgi(\n",
|
||||
" model_name=common_util.get_job_name_with_datetime(prefix=MODEL_ID),\n",
|
||||
" model_id=model_id,\n",
|
||||
" publisher=\"google\",\n",
|
||||
" publisher_model_id=\"gemma2\",\n",
|
||||
" service_account=SERVICE_ACCOUNT,\n",
|
||||
" machine_type=machine_type,\n",
|
||||
" accelerator_type=accelerator_type,\n",
|
||||
" accelerator_count=accelerator_count,\n",
|
||||
@@ -630,7 +662,9 @@
|
||||
" max_total_tokens=max_total_tokens,\n",
|
||||
" max_batch_prefill_tokens=max_batch_prefill_tokens,\n",
|
||||
" use_dedicated_endpoint=use_dedicated_endpoint,\n",
|
||||
")"
|
||||
")\n",
|
||||
"\n",
|
||||
"# @markdown Click \"Show Code\" to see more details."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -713,6 +747,8 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Delete the models and endpoints\n",
|
||||
"\n",
|
||||
"# @markdown Delete the experiment models and endpoints to recycle the resources\n",
|
||||
"# @markdown and avoid unnecessary continuous charges that may incur.\n",
|
||||
"\n",
|
||||
@@ -722,11 +758,7 @@
|
||||
"\n",
|
||||
"# Delete models.\n",
|
||||
"for model in models.values():\n",
|
||||
" model.delete()\n",
|
||||
"\n",
|
||||
"delete_bucket = False # @param {type:\"boolean\"}\n",
|
||||
"if delete_bucket:\n",
|
||||
" ! gsutil -m rm -r $BUCKET_NAME"
|
||||
" model.delete()"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
@@ -36,6 +36,11 @@
|
||||
"\n",
|
||||
"<table><tbody><tr>\n",
|
||||
" <td style=\"text-align: center\">\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/workbench/instances\">\n",
|
||||
" <img alt=\"Workbench logo\" src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" width=\"32px\"><br> Run in Workbench\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td style=\"text-align: center\">\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/colab/import/https:%2F%2Fraw.githubusercontent.com%2FGoogleCloudPlatform%2Fvertex-ai-samples%2Fmain%2Fnotebooks%2Fcommunity%2Fmodel_garden%2Fmodel_garden_gemma2_finetuning_on_vertex.ipynb\">\n",
|
||||
" <img alt=\"Google Cloud Colab Enterprise logo\" src=\"https://lh3.googleusercontent.com/JmcxdQi-qOpctIvWKgPtrzZdJJK-J3sWE1RsfjZNwshCFgE_9fULcNpuXYTilIR2hjwN\" width=\"32px\"><br> Run in Colab Enterprise\n",
|
||||
" </a>\n",
|
||||
@@ -156,6 +161,9 @@
|
||||
"from google.cloud.aiplatform.compat.types import \\\n",
|
||||
" custom_job as gca_custom_job_compat\n",
|
||||
"\n",
|
||||
"if os.environ.get(\"VERTEX_PRODUCT\") != \"COLAB_ENTERPRISE\":\n",
|
||||
" ! pip install --upgrade tensorflow\n",
|
||||
"\n",
|
||||
"common_util = importlib.import_module(\n",
|
||||
" \"vertex-ai-samples.community-content.vertex_model_garden.model_oss.notebook_util.common_util\"\n",
|
||||
")\n",
|
||||
@@ -224,7 +232,7 @@
|
||||
"\n",
|
||||
"# @markdown ## Access Gemma 2 Models\n",
|
||||
"\n",
|
||||
"# @markdown You must provide a Hugging Face User Access Token (read) to access the Gemma 2 models. You can follow the [Hugging Face documentation](https://huggingface.co/docs/hub/en/security-tokens) to create a **read** access token and put it in the `HF_TOKEN` field below.\n",
|
||||
"# @markdown You must provide a Hugging Face User Access Token (with read access) to access the Gemma 2 models. You can follow the [Hugging Face documentation](https://huggingface.co/docs/hub/en/security-tokens) to create a **read** access token and put it in the `HF_TOKEN` field below.\n",
|
||||
"\n",
|
||||
"HF_TOKEN = \"\" # @param {type:\"string\", isTemplate:true}\n",
|
||||
"assert HF_TOKEN, \"Provide a read HF_TOKEN to load models from Hugging Face.\"\n",
|
||||
@@ -502,8 +510,9 @@
|
||||
"merged_model_output_dir = os.path.join(base_output_dir, \"merged-model\")\n",
|
||||
"\n",
|
||||
"# Add labels for the finetuning job.\n",
|
||||
"\n",
|
||||
"labels = {\n",
|
||||
" \"mg-source\": \"notebook\",\n",
|
||||
" \"mg-source\": common_util.get_deploy_source(),\n",
|
||||
" \"mg-notebook-name\": \"model_garden_gemma2_finetuning_on_vertex.ipynb\".split(\".\")[0],\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
|
||||
@@ -66,6 +66,10 @@
|
||||
"\n",
|
||||
"- Deploy Gemma 3 with vLLM on GPU\n",
|
||||
"\n",
|
||||
"### File a bug\n",
|
||||
"\n",
|
||||
"File a bug on [GitHub](https://github.com/GoogleCloudPlatform/vertex-ai-samples/issues/new) if you encounter any issue with the notebook.\n",
|
||||
"\n",
|
||||
"### Costs\n",
|
||||
"\n",
|
||||
"This tutorial uses billable components of Google Cloud:\n",
|
||||
@@ -467,9 +471,7 @@
|
||||
"\n",
|
||||
"if use_dedicated_endpoint:\n",
|
||||
" DEDICATED_ENDPOINT_DNS = endpoints[\"vllm_gpu\"].gca_resource.dedicated_endpoint_dns\n",
|
||||
"ENDPOINT_RESOURCE_NAME = \"projects/{}/locations/{}/endpoints/{}\".format(\n",
|
||||
" PROJECT_ID, REGION, endpoints[\"vllm_gpu\"].name\n",
|
||||
")\n",
|
||||
"ENDPOINT_RESOURCE_NAME = endpoints[\"vllm_gpu\"].resource_name\n",
|
||||
"\n",
|
||||
"# @title Chat Completions Inference\n",
|
||||
"\n",
|
||||
@@ -779,9 +781,7 @@
|
||||
"\n",
|
||||
"if use_dedicated_endpoint:\n",
|
||||
" DEDICATED_ENDPOINT_DNS = endpoints[\"vllm_gpu\"].gca_resource.dedicated_endpoint_dns\n",
|
||||
"ENDPOINT_RESOURCE_NAME = \"projects/{}/locations/{}/endpoints/{}\".format(\n",
|
||||
" PROJECT_ID, REGION, endpoints[\"vllm_gpu\"].name\n",
|
||||
")\n",
|
||||
"ENDPOINT_RESOURCE_NAME = endpoints[\"vllm_gpu\"].resource_name\n",
|
||||
"\n",
|
||||
"# @title Chat Completions Inference\n",
|
||||
"\n",
|
||||
@@ -856,9 +856,7 @@
|
||||
"\n",
|
||||
"if use_dedicated_endpoint:\n",
|
||||
" DEDICATED_ENDPOINT_DNS = endpoints[\"vllm_gpu\"].gca_resource.dedicated_endpoint_dns\n",
|
||||
"ENDPOINT_RESOURCE_NAME = \"projects/{}/locations/{}/endpoints/{}\".format(\n",
|
||||
" PROJECT_ID, REGION, endpoints[\"vllm_gpu\"].name\n",
|
||||
")\n",
|
||||
"ENDPOINT_RESOURCE_NAME = endpoints[\"vllm_gpu\"].resource_name\n",
|
||||
"\n",
|
||||
"# @title Chat Completions Inference\n",
|
||||
"\n",
|
||||
|
||||
@@ -224,7 +224,7 @@
|
||||
"\n",
|
||||
"# @markdown ## Access Gemma 3 Models\n",
|
||||
"\n",
|
||||
"# @markdown You must provide a Hugging Face User Access Token (read) to access the Gemma 3 models. You can follow the [Hugging Face documentation](https://huggingface.co/docs/hub/en/security-tokens) to create a **read** access token and put it in the `HF_TOKEN` field below.\n",
|
||||
"# @markdown You must provide a Hugging Face User Access Token (with read access) to access the Gemma 3 models. You can follow the [Hugging Face documentation](https://huggingface.co/docs/hub/en/security-tokens) to create a **read** access token and put it in the `HF_TOKEN` field below.\n",
|
||||
"\n",
|
||||
"HF_TOKEN = \"\" # @param {type:\"string\", isTemplate:true}\n",
|
||||
"assert HF_TOKEN, \"Provide a read HF_TOKEN to load models from Hugging Face.\"\n",
|
||||
|
||||
@@ -199,7 +199,7 @@
|
||||
"# @markdown ---\n",
|
||||
"\n",
|
||||
"# @markdown ### Access Gemma models on Hugging Face\n",
|
||||
"# @markdown You must provide a Hugging Face User Access Token (read) to access the Gemma models. You can follow the [Hugging Face documentation](https://huggingface.co/docs/hub/en/security-tokens) to create a **read** access token and put it in the `HF_TOKEN` field below.\n",
|
||||
"# @markdown You must provide a Hugging Face User Access Token (with read access) to access the Gemma models. You can follow the [Hugging Face documentation](https://huggingface.co/docs/hub/en/security-tokens) to create a **read** access token and put it in the `HF_TOKEN` field below.\n",
|
||||
"\n",
|
||||
"HF_TOKEN = \"\" # @param {type:\"string\", isTemplate:true}\n",
|
||||
"if LOAD_MODEL_FROM == \"Hugging Face\":\n",
|
||||
@@ -325,6 +325,7 @@
|
||||
" disagg_topology: str = None,\n",
|
||||
" hbm_utilization_factor: float = 0.6,\n",
|
||||
" max_running_seqs: int = 256,\n",
|
||||
" decode_seqs_padding: int = None,\n",
|
||||
" max_model_len: int = 4096,\n",
|
||||
" enable_prefix_cache_hbm: bool = False,\n",
|
||||
" endpoint_id: str = \"\",\n",
|
||||
@@ -365,6 +366,10 @@
|
||||
" f\"--max_running_seqs={max_running_seqs}\",\n",
|
||||
" f\"--max_model_len={max_model_len}\",\n",
|
||||
" ]\n",
|
||||
"\n",
|
||||
" if decode_seqs_padding is not None:\n",
|
||||
" hexllm_args.append(f\"--decode_seqs_padding={decode_seqs_padding}\")\n",
|
||||
"\n",
|
||||
" if disagg_topology:\n",
|
||||
" hexllm_args.append(f\"--disagg_topo={disagg_topology}\")\n",
|
||||
" if enable_prefix_cache_hbm and not disagg_topology:\n",
|
||||
@@ -513,9 +518,7 @@
|
||||
"\n",
|
||||
"if use_dedicated_endpoint:\n",
|
||||
" DEDICATED_ENDPOINT_DNS = endpoints[\"hexllm_tpu\"].gca_resource.dedicated_endpoint_dns\n",
|
||||
"ENDPOINT_RESOURCE_NAME = \"projects/{}/locations/{}/endpoints/{}\".format(\n",
|
||||
" PROJECT_ID, REGION, endpoints[\"hexllm_tpu\"].name\n",
|
||||
")\n",
|
||||
"ENDPOINT_RESOURCE_NAME = endpoints[\"hexllm_tpu\"].resource_name\n",
|
||||
"\n",
|
||||
"# @title Chat Completions Inference\n",
|
||||
"\n",
|
||||
@@ -909,9 +912,7 @@
|
||||
"\n",
|
||||
"if use_dedicated_endpoint:\n",
|
||||
" DEDICATED_ENDPOINT_DNS = endpoints[\"vllm_gpu\"].gca_resource.dedicated_endpoint_dns\n",
|
||||
"ENDPOINT_RESOURCE_NAME = \"projects/{}/locations/{}/endpoints/{}\".format(\n",
|
||||
" PROJECT_ID, REGION, endpoints[\"vllm_gpu\"].name\n",
|
||||
")\n",
|
||||
"ENDPOINT_RESOURCE_NAME = endpoints[\"vllm_gpu\"].resource_name\n",
|
||||
"\n",
|
||||
"# @title Chat Completions Inference\n",
|
||||
"\n",
|
||||
|
||||
@@ -203,7 +203,7 @@
|
||||
"\n",
|
||||
"# @markdown This section demonstrates how to evaluate the Gemma models with and without finetuned LoRA adapters using EleutherAI's [Language Model Evaluation Harness (lm-evaluation-harness)](https://github.com/EleutherAI/lm-evaluation-harness) with Vertex CustomJob. Refer the peak GPU memory usage for serving and adjust the machine type, accelerator type and accelerator count accordingly.\n",
|
||||
"\n",
|
||||
"# @markdown You must provide a Hugging Face User Access Token (read) to access the Gemma models. You can follow the [Hugging Face documentation](https://huggingface.co/docs/hub/en/security-tokens) to create a **read** access token and put it in the `HF_TOKEN` field below.\n",
|
||||
"# @markdown You must provide a Hugging Face User Access Token (with read access) to access the Gemma models. You can follow the [Hugging Face documentation](https://huggingface.co/docs/hub/en/security-tokens) to create a **read** access token and put it in the `HF_TOKEN` field below.\n",
|
||||
"HF_TOKEN = \"\" # @param {type:\"string\", isTemplate:true}\n",
|
||||
"\n",
|
||||
"# @markdown This example uses the dataset [HellaSwag](https://arxiv.org/abs/1905.07830). All supported tasks are listed in [this task table](https://github.com/EleutherAI/lm-evaluation-harness/blob/master/docs/task_table.md).\n",
|
||||
|
||||
@@ -113,10 +113,9 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"language": "python",
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "855d6b96f291"
|
||||
"id": "8CQcnBfWvc-f"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -238,7 +237,7 @@
|
||||
"# @markdown ---\n",
|
||||
"\n",
|
||||
"# @markdown ### Access Gemma models on Hugging Face for GPU based finetuning and serving\n",
|
||||
"# @markdown You must provide a Hugging Face User Access Token (read) to access the Gemma models. You can follow the [Hugging Face documentation](https://huggingface.co/docs/hub/en/security-tokens) to create a **read** access token and put it in the `HF_TOKEN` field below.\n",
|
||||
"# @markdown You must provide a Hugging Face User Access Token (with read access) to access the Gemma models. You can follow the [Hugging Face documentation](https://huggingface.co/docs/hub/en/security-tokens) to create a **read** access token and put it in the `HF_TOKEN` field below.\n",
|
||||
"\n",
|
||||
"HF_TOKEN = \"\" # @param {type:\"string\", isTemplate:true}\n",
|
||||
"if LOAD_MODEL_FROM == \"Hugging Face\":\n",
|
||||
|
||||
+4
-1
@@ -316,7 +316,7 @@
|
||||
" model_id: str,\n",
|
||||
" publisher: str,\n",
|
||||
" publisher_model_id: str,\n",
|
||||
" service_account: str,\n",
|
||||
" service_account: str = None,\n",
|
||||
" machine_type: str = \"g2-standard-8\",\n",
|
||||
" accelerator_type: str = \"NVIDIA_L4\",\n",
|
||||
" accelerator_count: int = 1,\n",
|
||||
@@ -347,6 +347,9 @@
|
||||
" except NameError:\n",
|
||||
" pass\n",
|
||||
"\n",
|
||||
" if service_account:\n",
|
||||
" env_vars[\"SERVICE_ACCOUNT\"] = service_account\n",
|
||||
"\n",
|
||||
" model = aiplatform.Model.upload(\n",
|
||||
" display_name=model_name,\n",
|
||||
" serving_container_image_uri=TGI_DOCKER_URI,\n",
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -34,6 +34,11 @@
|
||||
"\n",
|
||||
"<table><tbody><tr>\n",
|
||||
" <td style=\"text-align: center\">\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/workbench/instances\">\n",
|
||||
" <img alt=\"Workbench logo\" src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" width=\"32px\"><br> Run in Workbench\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td style=\"text-align: center\">\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/colab/import/https:%2F%2Fraw.githubusercontent.com%2FGoogleCloudPlatform%2Fvertex-ai-samples%2Fmain%2Fnotebooks%2Fcommunity%2Fmodel_garden%2Fmodel_garden_hf_paligemma2_deployment.ipynb\">\n",
|
||||
" <img alt=\"Google Cloud Colab Enterprise logo\" src=\"https://lh3.googleusercontent.com/JmcxdQi-qOpctIvWKgPtrzZdJJK-J3sWE1RsfjZNwshCFgE_9fULcNpuXYTilIR2hjwN\" width=\"32px\"><br> Run in Colab Enterprise\n",
|
||||
" </a>\n",
|
||||
@@ -63,6 +68,10 @@
|
||||
"- Make predictions to the endpoint including:\n",
|
||||
" - Answering questions about a given image.\n",
|
||||
"\n",
|
||||
"### File a bug\n",
|
||||
"\n",
|
||||
"File a bug on [GitHub](https://github.com/GoogleCloudPlatform/vertex-ai-samples/issues/new) if you encounter any issue with the notebook.\n",
|
||||
"\n",
|
||||
"### Costs\n",
|
||||
"\n",
|
||||
"This tutorial uses billable components of Google Cloud:\n",
|
||||
@@ -93,16 +102,6 @@
|
||||
"source": [
|
||||
"# @title Setup Google Cloud project\n",
|
||||
"\n",
|
||||
"# Used for common utilities.\n",
|
||||
"! git clone https://github.com/GoogleCloudPlatform/vertex-ai-samples.git\n",
|
||||
"\n",
|
||||
"import importlib\n",
|
||||
"# Import the necessary packages\n",
|
||||
"import os\n",
|
||||
"from typing import Any, Dict, Tuple\n",
|
||||
"\n",
|
||||
"from google.cloud import aiplatform\n",
|
||||
"\n",
|
||||
"# @markdown 1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
|
||||
"\n",
|
||||
"# @markdown 2. **[Optional]** Set region. If not set, the region will be set automatically according to Colab Enterprise environment.\n",
|
||||
@@ -118,6 +117,27 @@
|
||||
"# @markdown | a3-highgpu-4g | 4 NVIDIA_H100_80GB | us-west1, asia-southeast1, europe-west4 |\n",
|
||||
"# @markdown | a3-highgpu-8g | 8 NVIDIA_H100_80GB | us-central1, europe-west4, us-west1, asia-southeast1 |\n",
|
||||
"\n",
|
||||
"# Upgrade Vertex AI SDK.\n",
|
||||
"! pip3 install --upgrade --quiet 'google-cloud-aiplatform>=1.84.0'\n",
|
||||
"\n",
|
||||
"import importlib\n",
|
||||
"# Import the necessary packages\n",
|
||||
"import os\n",
|
||||
"from typing import Any, Dict, Tuple\n",
|
||||
"\n",
|
||||
"from google.cloud import aiplatform\n",
|
||||
"\n",
|
||||
"if os.environ.get(\"VERTEX_PRODUCT\") != \"COLAB_ENTERPRISE\":\n",
|
||||
" ! pip install --upgrade tensorflow\n",
|
||||
"! git clone https://github.com/GoogleCloudPlatform/vertex-ai-samples.git\n",
|
||||
"\n",
|
||||
"common_util = importlib.import_module(\n",
|
||||
" \"vertex-ai-samples.community-content.vertex_model_garden.model_oss.notebook_util.common_util\"\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"models, endpoints = {}, {}\n",
|
||||
"LABEL = \"paligemma2\"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Get the default cloud project id.\n",
|
||||
"PROJECT_ID = os.environ[\"GOOGLE_CLOUD_PROJECT\"]\n",
|
||||
@@ -126,27 +146,116 @@
|
||||
"if not REGION:\n",
|
||||
" REGION = os.environ[\"GOOGLE_CLOUD_REGION\"]\n",
|
||||
"\n",
|
||||
"# Enable the Vertex AI API and Compute Engine API, if not already.\n",
|
||||
"print(\"Enabling Vertex AI API and Compute Engine API.\")\n",
|
||||
"! gcloud services enable aiplatform.googleapis.com compute.googleapis.com\n",
|
||||
"\n",
|
||||
"# Initialize Vertex AI API.\n",
|
||||
"print(\"Initializing Vertex AI API.\")\n",
|
||||
"aiplatform.init(project=PROJECT_ID, location=REGION)\n",
|
||||
"\n",
|
||||
"# Gets the default SERVICE_ACCOUNT.\n",
|
||||
"shell_output = ! gcloud projects describe $PROJECT_ID\n",
|
||||
"project_number = shell_output[-1].split(\":\")[1].strip().replace(\"'\", \"\")\n",
|
||||
"SERVICE_ACCOUNT = f\"{project_number}-compute@developer.gserviceaccount.com\"\n",
|
||||
"print(\"Using this default Service Account:\", SERVICE_ACCOUNT)\n",
|
||||
"\n",
|
||||
"! gcloud config set project $PROJECT_ID\n",
|
||||
"import vertexai\n",
|
||||
"\n",
|
||||
"common_util = importlib.import_module(\n",
|
||||
" \"vertex-ai-samples.community-content.vertex_model_garden.model_oss.notebook_util.common_util\"\n",
|
||||
"vertexai.init(\n",
|
||||
" project=PROJECT_ID,\n",
|
||||
" location=REGION,\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "kyMJXkfviWgl"
|
||||
},
|
||||
"source": [
|
||||
"## Deploy Model to a Vertex AI Endpoint"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "toY-WPKDFesF"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Deploy\n",
|
||||
"\n",
|
||||
"MODEL_NAME = \"paligemma2-3b-pt-224\" # @param [\"paligemma2-3b-pt-224\", \"paligemma2-3b-mix-224\", \"paligemma2-3b-ft-docci-448\", \"paligemma2-3b-mix-448\", \"paligemma2-3b-pt-448\", \"paligemma2-3b-pt-896\", \"paligemma2-10b-mix-224\", \"paligemma2-10b-pt-224\", \"paligemma2-10b-ft-docci-448\", \"paligemma2-10b-mix-448\", \"paligemma2-10b-pt-448\", \"paligemma2-10b-pt-896\", \"paligemma2-28b-mix-224\", \"paligemma2-28b-pt-224\", \"paligemma2-28b-mix-448\", \"paligemma2-28b-pt-448\", \"paligemma2-28b-pt-896\"]\n",
|
||||
"GCS_PREFIX = \"gs://vertex-model-garden-restricted-us/paligemma2\"\n",
|
||||
"\n",
|
||||
"MODEL_ID = os.path.join(GCS_PREFIX, MODEL_NAME)\n",
|
||||
"\n",
|
||||
"PUBLISHER_MODEL_NAME = f\"publishers/google/models/paligemma@{MODEL_NAME}\"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# @markdown If you want to use other accelerator types not listed above, then check other Vertex AI prediction supported accelerators and regions at https://cloud.google.com/vertex-ai/docs/predictions/configure-compute. You may need to manually set the `machine_type`, `accelerator_type`, and `accelerator_count` in the code by clicking `Show code` first.\n",
|
||||
"\n",
|
||||
"if \"3b\" in MODEL_NAME:\n",
|
||||
" accelerator_type = \"NVIDIA_L4\"\n",
|
||||
" machine_type = \"g2-standard-16\"\n",
|
||||
" accelerator_count = 1\n",
|
||||
"elif \"10b\" in MODEL_NAME:\n",
|
||||
" accelerator_type = \"NVIDIA_TESLA_A100\"\n",
|
||||
" machine_type = \"a2-highgpu-1g\"\n",
|
||||
" accelerator_count = 1\n",
|
||||
"elif \"28b\" in MODEL_NAME:\n",
|
||||
" accelerator_type = \"NVIDIA_H100_80GB\"\n",
|
||||
" machine_type = \"a3-highgpu-8g\"\n",
|
||||
" accelerator_count = 8\n",
|
||||
"else:\n",
|
||||
" raise ValueError(f\"Recommended GPU setting not found for: {MODEL_NAME}.\")\n",
|
||||
"\n",
|
||||
"common_util.check_quota(\n",
|
||||
" project_id=PROJECT_ID,\n",
|
||||
" region=REGION,\n",
|
||||
" accelerator_type=accelerator_type,\n",
|
||||
" accelerator_count=accelerator_count,\n",
|
||||
" is_for_training=False,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"models, endpoints = {}, {}\n",
|
||||
"# @markdown Set use_dedicated_endpoint to False if you don't want to use [dedicated endpoint](https://cloud.google.com/vertex-ai/docs/general/deployment#create-dedicated-endpoint). Note that [dedicated endpoint does not support VPC Service Controls](https://cloud.google.com/vertex-ai/docs/predictions/choose-endpoint-type), uncheck the box if you are using VPC-SC.\n",
|
||||
"use_dedicated_endpoint = True # @param {type:\"boolean\"}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "pe_qbTCA6nKf"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title [Option 1] Deploy with Model Garden SDK\n",
|
||||
"\n",
|
||||
"# @markdown Deploy with Gen AI model-centric SDK. This section uploads the prebuilt model to Model Registry and deploys it to a Vertex AI Endpoint. It takes 15 minutes to 1 hour to finish depending on the size of the model. See [use open models with Vertex AI](https://cloud.google.com/vertex-ai/generative-ai/docs/open-models/use-open-models) for documentation on other use cases.\n",
|
||||
"from vertexai.preview import model_garden\n",
|
||||
"\n",
|
||||
"model = model_garden.OpenModel(PUBLISHER_MODEL_NAME)\n",
|
||||
"endpoints[LABEL] = model.deploy(\n",
|
||||
" machine_type=machine_type,\n",
|
||||
" accelerator_type=accelerator_type,\n",
|
||||
" accelerator_count=accelerator_count,\n",
|
||||
" use_dedicated_endpoint=use_dedicated_endpoint,\n",
|
||||
" accept_eula=True, # Accept the End User License Agreement (EULA) on the model card before deploy. Otherwise, the deployment will be forbidden.\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "jbeLl-9C6nKf"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title [Option 2] Deploy with customized configs\n",
|
||||
"\n",
|
||||
"# @markdown This section uploads the prebuilt PaliGemma 2 models to Model Registry and deploys it to a Vertex AI Endpoint. It takes approximately 15 minutes to finish.\n",
|
||||
"\n",
|
||||
"# @markdown Select the desired resolution and precision of prebuilt model to deploy, leaving the optional `custom_paligemma_model_uri` as is. Higher resolution and precision_type can result in better inference results, but may require additional GPU.\n",
|
||||
"\n",
|
||||
"TASK = \"paligemma_VQA\"\n",
|
||||
"\n",
|
||||
"# The pre-built serving docker images.\n",
|
||||
"SERVE_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-one-serve:20250205_0822_RC00\"\n",
|
||||
@@ -159,7 +268,6 @@
|
||||
" machine_type: str = \"g2-standard-8\",\n",
|
||||
" accelerator_type: str = \"NVIDIA_L4\",\n",
|
||||
" accelerator_count: int = 1,\n",
|
||||
" service_account: str = None,\n",
|
||||
" serving_port: int = 7080,\n",
|
||||
" serving_route: str = \"/predict\",\n",
|
||||
" serving_docker_uri: str = SERVE_DOCKER_URI,\n",
|
||||
@@ -173,7 +281,6 @@
|
||||
" machine_type: The machine type.\n",
|
||||
" accelerator_type: The accelerator type.\n",
|
||||
" accelerator_count: The accelerator count.\n",
|
||||
" service_account: The service account.\n",
|
||||
" serving_port: The serving port.\n",
|
||||
" serving_route: The serving route.\n",
|
||||
" hf_token: HuggingFace token for model access.\n",
|
||||
@@ -204,111 +311,15 @@
|
||||
" machine_type=machine_type,\n",
|
||||
" accelerator_type=accelerator_type,\n",
|
||||
" accelerator_count=accelerator_count,\n",
|
||||
" service_account=service_account,\n",
|
||||
" sync=False,\n",
|
||||
" system_labels={\"NOTEBOOK_NAME\": \"model_garden_hf_paligemma2_deployment.ipynb\"},\n",
|
||||
" system_labels={\n",
|
||||
" \"NOTEBOOK_NAME\": \"model_garden_hf_paligemma2_deployment.ipynb\",\n",
|
||||
" \"NOTEBOOK_ENVIRONMENT\": common_util.get_deploy_source(),\n",
|
||||
" },\n",
|
||||
" )\n",
|
||||
" return endpoint, model\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def vqa_predict(\n",
|
||||
" endpoint: aiplatform.Endpoint,\n",
|
||||
" image_url: str,\n",
|
||||
" text_prompt: str,\n",
|
||||
" parameters: Dict[str, Any] = None,\n",
|
||||
") -> str:\n",
|
||||
" \"\"\"Predicts the answer to a question about an image using an Endpoint,\n",
|
||||
"\n",
|
||||
" and passes parameters in the payload.\n",
|
||||
"\n",
|
||||
" Args:\n",
|
||||
" endpoint: The deployed Vertex AI endpoint.\n",
|
||||
" image_url: URL of the image to ask about.\n",
|
||||
" text_prompt: The text prompt question.\n",
|
||||
" parameters: Additional parameters for the prediction request.\n",
|
||||
"\n",
|
||||
" Returns:\n",
|
||||
" The predicted answer string or None if no prediction.\n",
|
||||
" \"\"\"\n",
|
||||
"\n",
|
||||
" instances = []\n",
|
||||
" if text_prompt:\n",
|
||||
" instances.append(\n",
|
||||
" {\n",
|
||||
" \"text_prompt\": text_prompt,\n",
|
||||
" \"image_url\": image_url,\n",
|
||||
" }\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" # Construct the prediction payload\n",
|
||||
" payload = {\"instances\": instances}\n",
|
||||
" if parameters:\n",
|
||||
" payload[\"parameters\"] = parameters\n",
|
||||
"\n",
|
||||
" response = endpoint.predict(instances=instances, parameters=parameters)\n",
|
||||
" answer = None\n",
|
||||
" if response.predictions:\n",
|
||||
" answer = response.predictions[0][\"text\"].split(\"\\n\")[1]\n",
|
||||
" return answer"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "kyMJXkfviWgl"
|
||||
},
|
||||
"source": [
|
||||
"## Deploy Model to a Vertex AI Endpoint"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "toY-WPKDFesF"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Deploy\n",
|
||||
"\n",
|
||||
"# @markdown This section uploads the prebuilt PaliGemma 2 models to Model Registry and deploys it to a Vertex AI Endpoint. It takes approximately 15 minutes to finish.\n",
|
||||
"\n",
|
||||
"# @markdown Select the desired resolution and precision of prebuilt model to deploy, leaving the optional `custom_paligemma_model_uri` as is. Higher resolution and precision_type can result in better inference results, but may require additional GPU.\n",
|
||||
"\n",
|
||||
"MODEL_NAME = \"paligemma2-3b-pt-224\" # @param [\"paligemma2-3b-pt-224\", \"paligemma2-3b-mix-224\", \"paligemma2-3b-ft-docci-448\", \"paligemma2-3b-mix-448\", \"paligemma2-3b-pt-448\", \"paligemma2-3b-pt-896\", \"paligemma2-10b-mix-224\", \"paligemma2-10b-pt-224\", \"paligemma2-10b-ft-docci-448\", \"paligemma2-10b-mix-448\", \"paligemma2-10b-pt-448\", \"paligemma2-10b-pt-896\", \"paligemma2-28b-mix-224\", \"paligemma2-28b-pt-224\", \"paligemma2-28b-mix-448\", \"paligemma2-28b-pt-448\", \"paligemma2-28b-pt-896\"]\n",
|
||||
"GCS_PREFIX = \"gs://vertex-model-garden-restricted-us/paligemma2\"\n",
|
||||
"\n",
|
||||
"MODEL_ID = os.path.join(GCS_PREFIX, MODEL_NAME)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# @markdown If you want to use other accelerator types not listed above, then check other Vertex AI prediction supported accelerators and regions at https://cloud.google.com/vertex-ai/docs/predictions/configure-compute. You may need to manually set the `machine_type`, `accelerator_type`, and `accelerator_count` in the code by clicking `Show code` first.\n",
|
||||
"\n",
|
||||
"if \"3b\" in MODEL_NAME:\n",
|
||||
" accelerator_type = \"NVIDIA_L4\"\n",
|
||||
" machine_type = \"g2-standard-8\"\n",
|
||||
" accelerator_count = 1\n",
|
||||
"elif \"10b\" in MODEL_NAME:\n",
|
||||
" accelerator_type = \"NVIDIA_TESLA_A100\"\n",
|
||||
" machine_type = \"a2-highgpu-1g\"\n",
|
||||
" accelerator_count = 1\n",
|
||||
"elif \"28b\" in MODEL_NAME:\n",
|
||||
" accelerator_type = \"NVIDIA_H100_80GB\"\n",
|
||||
" machine_type = \"a3-highgpu-8g\"\n",
|
||||
" accelerator_count = 8\n",
|
||||
"else:\n",
|
||||
" raise ValueError(f\"Recommended GPU setting not found for: {MODEL_NAME}.\")\n",
|
||||
"\n",
|
||||
"common_util.check_quota(\n",
|
||||
" project_id=PROJECT_ID,\n",
|
||||
" region=REGION,\n",
|
||||
" accelerator_type=accelerator_type,\n",
|
||||
" accelerator_count=accelerator_count,\n",
|
||||
" is_for_training=False,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"TASK = \"paligemma_VQA\"\n",
|
||||
"\n",
|
||||
"endpoints[\"paligemma2\"], models[\"paligemma2\"] = deploy_model(\n",
|
||||
" model_name=MODEL_NAME,\n",
|
||||
" model_id=MODEL_ID,\n",
|
||||
@@ -316,7 +327,6 @@
|
||||
" machine_type=machine_type,\n",
|
||||
" accelerator_type=accelerator_type,\n",
|
||||
" accelerator_count=accelerator_count,\n",
|
||||
" service_account=SERVICE_ACCOUNT,\n",
|
||||
" serving_port=7080,\n",
|
||||
" serving_route=\"/predict\",\n",
|
||||
" serving_docker_uri=SERVE_DOCKER_URI,\n",
|
||||
@@ -379,6 +389,48 @@
|
||||
"\n",
|
||||
"# @markdown The question prompt can be non-English languages.\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def vqa_predict(\n",
|
||||
" endpoint: aiplatform.Endpoint,\n",
|
||||
" image_url: str,\n",
|
||||
" text_prompt: str,\n",
|
||||
" parameters: Dict[str, Any] = None,\n",
|
||||
") -> str:\n",
|
||||
" \"\"\"Predicts the answer to a question about an image using an Endpoint,\n",
|
||||
"\n",
|
||||
" and passes parameters in the payload.\n",
|
||||
"\n",
|
||||
" Args:\n",
|
||||
" endpoint: The deployed Vertex AI endpoint.\n",
|
||||
" image_url: URL of the image to ask about.\n",
|
||||
" text_prompt: The text prompt question.\n",
|
||||
" parameters: Additional parameters for the prediction request.\n",
|
||||
"\n",
|
||||
" Returns:\n",
|
||||
" The predicted answer string or None if no prediction.\n",
|
||||
" \"\"\"\n",
|
||||
"\n",
|
||||
" instances = []\n",
|
||||
" if text_prompt:\n",
|
||||
" instances.append(\n",
|
||||
" {\n",
|
||||
" \"text_prompt\": text_prompt,\n",
|
||||
" \"image_url\": image_url,\n",
|
||||
" }\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" # Construct the prediction payload\n",
|
||||
" payload = {\"instances\": instances}\n",
|
||||
" if parameters:\n",
|
||||
" payload[\"parameters\"] = parameters\n",
|
||||
"\n",
|
||||
" response = endpoint.predict(instances=instances, parameters=parameters)\n",
|
||||
" answer = None\n",
|
||||
" if response.predictions:\n",
|
||||
" answer = response.predictions[0][\"text\"].split(\"\\n\")[1]\n",
|
||||
" return answer\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Using max_new_tokens along with other parameters\n",
|
||||
"parameters_with_tokens = {\"max_new_tokens\": 50}\n",
|
||||
"predictions_with_tokens = vqa_predict(\n",
|
||||
|
||||
@@ -34,6 +34,11 @@
|
||||
"\n",
|
||||
"<table><tbody><tr>\n",
|
||||
" <td style=\"text-align: center\">\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/workbench/instances\">\n",
|
||||
" <img alt=\"Workbench logo\" src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" width=\"32px\"><br> Run in Workbench\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td style=\"text-align: center\">\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/colab/import/https:%2F%2Fraw.githubusercontent.com%2FGoogleCloudPlatform%2Fvertex-ai-samples%2Fmain%2Fnotebooks%2Fcommunity%2Fmodel_garden%2Fmodel_garden_huggingface_tei_deployment.ipynb\">\n",
|
||||
" <img alt=\"Google Cloud Colab Enterprise logo\" src=\"https://lh3.googleusercontent.com/JmcxdQi-qOpctIvWKgPtrzZdJJK-J3sWE1RsfjZNwshCFgE_9fULcNpuXYTilIR2hjwN\" width=\"32px\"><br> Run in Colab Enterprise\n",
|
||||
" </a>\n",
|
||||
@@ -62,6 +67,10 @@
|
||||
"- Download and deploy the `nomic-ai/nomic-embed-text-v1` model with TEI\n",
|
||||
"- Send prediction request to the deployed endpoint\n",
|
||||
"\n",
|
||||
"### File a bug\n",
|
||||
"\n",
|
||||
"File a bug on [GitHub](https://github.com/GoogleCloudPlatform/vertex-ai-samples/issues/new) if you encounter any issue with the notebook.\n",
|
||||
"\n",
|
||||
"### Costs\n",
|
||||
"\n",
|
||||
"This tutorial uses billable components of Google Cloud:\n",
|
||||
@@ -97,8 +106,7 @@
|
||||
"\n",
|
||||
"REGION = \"\" # @param {type:\"string\"}\n",
|
||||
"\n",
|
||||
"# Upgrade Vertex AI SDK.\n",
|
||||
"! pip3 install --upgrade --quiet 'google-cloud-aiplatform>=1.64.0'\n",
|
||||
"! pip3 install --upgrade --quiet 'google-cloud-aiplatform>=1.84.0'\n",
|
||||
"! git clone https://github.com/GoogleCloudPlatform/vertex-ai-samples.git\n",
|
||||
"\n",
|
||||
"import importlib\n",
|
||||
@@ -135,24 +143,16 @@
|
||||
"\n",
|
||||
"models, endpoints = {}, {}\n",
|
||||
"\n",
|
||||
"HF_TOKEN = \"\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "USB7dvYqvNdu"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Deploy with TEI from Hugging Face\n",
|
||||
"import vertexai\n",
|
||||
"\n",
|
||||
"# @markdown This section downloads the `nomic-ai/nomic-embed-text-v1` model from Hugging Face and deploys it to a Vertex AI Endpoint.\n",
|
||||
"# @markdown It takes ~20 minutes to complete the deployment.\n",
|
||||
"vertexai.init(\n",
|
||||
" project=PROJECT_ID,\n",
|
||||
" location=REGION,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"MODEL_ID = \"nomic-ai/nomic-embed-text-v1\" # @param {type: \"string\", isTemplate: true}\n",
|
||||
"HF_TOKEN = \"\"\n",
|
||||
"\n",
|
||||
"HUGGING_FACE_MODEL_ID = \"nomic-ai/nomic-embed-text-v1\" # @param {type: \"string\", isTemplate: true}\n",
|
||||
"\n",
|
||||
"# The pre-built serving docker images for TEI.\n",
|
||||
"TEI_CPU_DOCKER_URI = \"us-docker.pkg.dev/deeplearning-platform-release/gcr.io/huggingface-text-embeddings-inference-cpu.1-4\"\n",
|
||||
@@ -173,6 +173,53 @@
|
||||
" is_for_training=False,\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
"LABEL = \"tei\"\n",
|
||||
"\n",
|
||||
"# @markdown Set `use_dedicated_endpoint` to False if you don't want to use [dedicated endpoint](https://cloud.google.com/vertex-ai/docs/general/deployment#create-dedicated-endpoint).\n",
|
||||
"use_dedicated_endpoint = True # @param {type:\"boolean\"}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "obbeTtMJ5C8j"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title [Option 1] Deploy with Model Garden SDK\n",
|
||||
"\n",
|
||||
"accelerator_count = 1\n",
|
||||
"\n",
|
||||
"# @markdown Deploy with Gen AI model-centric SDK. This section uploads the prebuilt model to Model Registry and deploys it to a Vertex AI Endpoint. It takes 15 minutes to 1 hour to finish depending on the size of the model. See [use open models with Vertex AI](https://cloud.google.com/vertex-ai/generative-ai/docs/open-models/use-open-models) for documentation on other use cases.\n",
|
||||
"from vertexai.preview import model_garden\n",
|
||||
"\n",
|
||||
"model = model_garden.OpenModel(HUGGING_FACE_MODEL_ID)\n",
|
||||
"endpoints[LABEL] = model.deploy(\n",
|
||||
" machine_type=machine_type,\n",
|
||||
" accelerator_type=accelerator_type,\n",
|
||||
" accelerator_count=accelerator_count,\n",
|
||||
" hugging_face_access_token=HF_TOKEN,\n",
|
||||
" use_dedicated_endpoint=use_dedicated_endpoint,\n",
|
||||
" accept_eula=True, # Accept the End User License Agreement (EULA) on the model card before deploy. Otherwise, the deployment will be forbidden.\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "USB7dvYqvNdu"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title [Option 2] Deploy with customized configs\n",
|
||||
"\n",
|
||||
"# @markdown This section downloads the `nomic-ai/nomic-embed-text-v1` model from Hugging Face and deploys it to a Vertex AI Endpoint.\n",
|
||||
"# @markdown It takes ~20 minutes to complete the deployment.\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def deploy_model_tei(\n",
|
||||
" model_name: str,\n",
|
||||
@@ -229,13 +276,9 @@
|
||||
" return model, endpoint\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# @markdown Set `use_dedicated_endpoint` to False if you don't want to use [dedicated endpoint](https://cloud.google.com/vertex-ai/docs/general/deployment#create-dedicated-endpoint).\n",
|
||||
"use_dedicated_endpoint = True # @param {type:\"boolean\"}\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"models[\"tei\"], endpoints[\"tei\"] = deploy_model_tei(\n",
|
||||
" model_name=common_util.get_job_name_with_datetime(prefix=MODEL_ID),\n",
|
||||
" model_id=MODEL_ID,\n",
|
||||
" model_name=common_util.get_job_name_with_datetime(prefix=HUGGING_FACE_MODEL_ID),\n",
|
||||
" model_id=HUGGING_FACE_MODEL_ID,\n",
|
||||
" publisher=\"hf-nomic-ai\",\n",
|
||||
" publisher_model_id=\"nomic-embed-text-v1\",\n",
|
||||
" service_account=\"\",\n",
|
||||
|
||||
@@ -236,7 +236,7 @@
|
||||
" model_id: str,\n",
|
||||
" publisher: str,\n",
|
||||
" publisher_model_id: str,\n",
|
||||
" service_account: str,\n",
|
||||
" service_account: str = None,\n",
|
||||
" machine_type: str = \"g2-standard-8\",\n",
|
||||
" accelerator_type: str = \"NVIDIA_L4\",\n",
|
||||
" accelerator_count: int = 1,\n",
|
||||
@@ -267,6 +267,9 @@
|
||||
" except NameError:\n",
|
||||
" pass\n",
|
||||
"\n",
|
||||
" if service_account:\n",
|
||||
" env_vars[\"SERVICE_ACCOUNT\"] = service_account\n",
|
||||
"\n",
|
||||
" model = aiplatform.Model.upload(\n",
|
||||
" display_name=model_name,\n",
|
||||
" serving_container_image_uri=TGI_DOCKER_URI,\n",
|
||||
|
||||
@@ -164,7 +164,7 @@
|
||||
"HF_TOKEN = \"\" # @param {type:\"string\", isTemplate: true}\n",
|
||||
"\n",
|
||||
"# The pre-built vLLM serving docker image.\n",
|
||||
"VLLM_DOCKER_URI = \"us-docker.pkg.dev/deeplearning-platform-release/vertex-model-garden/vllm-inference.cu121.0-6.ubuntu2204.py310\"\n",
|
||||
"VLLM_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-vllm-serve:20250506_0916_RC01\"\n",
|
||||
"SERVING_CONTAINER_IMAGE_URI = VLLM_DOCKER_URI\n",
|
||||
"LABEL = \"vllm\"\n",
|
||||
"\n",
|
||||
@@ -403,6 +403,8 @@
|
||||
" accelerator_type=accelerator_type,\n",
|
||||
" accelerator_count=accelerator_count,\n",
|
||||
" use_dedicated_endpoint=use_dedicated_endpoint,\n",
|
||||
" enforce_eager=True,\n",
|
||||
" max_num_seqs=5,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# @markdown Click \"Show Code\" to see more details."
|
||||
|
||||
@@ -0,0 +1,588 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "20qcPG1PmFUM"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Copyright 2025 Google LLC\n",
|
||||
"#\n",
|
||||
"# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
|
||||
"# you may not use this file except in compliance with the License.\n",
|
||||
"# You may obtain a copy of the License at\n",
|
||||
"#\n",
|
||||
"# https://www.apache.org/licenses/LICENSE-2.0\n",
|
||||
"#\n",
|
||||
"# Unless required by applicable law or agreed to in writing, software\n",
|
||||
"# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
|
||||
"# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
|
||||
"# See the License for the specific language governing permissions and\n",
|
||||
"# limitations under the License."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "QXYOa1odnikj"
|
||||
},
|
||||
"source": [
|
||||
"# Vertex AI Model Garden Integration With ADK\n",
|
||||
"\n",
|
||||
"\u003ctable\u003e\u003ctbody\u003e\u003ctr\u003e\n",
|
||||
" \u003ctd style=\"text-align: center\"\u003e\n",
|
||||
" \u003ca href=\"https://console.cloud.google.com/vertex-ai/colab/import/https:%2F%2Fraw.githubusercontent.com%2FGoogleCloudPlatform%2Fvertex-ai-samples%2Fmain%2Fnotebooks%2Fcommunity%2Fmodel_garden%2Fmodel_garden_integration_with_adk.ipynb\"\u003e\n",
|
||||
" \u003cimg alt=\"Google Cloud Colab Enterprise logo\" src=\"https://lh3.googleusercontent.com/JmcxdQi-qOpctIvWKgPtrzZdJJK-J3sWE1RsfjZNwshCFgE_9fULcNpuXYTilIR2hjwN\" width=\"32px\"\u003e\u003cbr\u003e Run in Colab Enterprise\n",
|
||||
" \u003c/a\u003e\n",
|
||||
" \u003c/td\u003e\n",
|
||||
" \u003ctd style=\"text-align: center\"\u003e\n",
|
||||
" \u003ca href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_integration_with_adk.ipynb\"\u003e\n",
|
||||
" \u003cimg alt=\"GitHub logo\" src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" width=\"32px\"\u003e\u003cbr\u003e View on GitHub\n",
|
||||
" \u003c/a\u003e\n",
|
||||
" \u003c/td\u003e\n",
|
||||
"\u003c/tr\u003e\u003c/tbody\u003e\u003c/table\u003e"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "cbDI9ag4oR4C"
|
||||
},
|
||||
"source": [
|
||||
"## Overview\n",
|
||||
"\n",
|
||||
"This notebook demonstrates how to build an agent with a Model Garden open model deployed through Vertex endpoint and [Google Agent Development Kit](https://google.github.io/adk-docs/) (ADK). The agent can automatically call function tools like `get_weather` and `get_current_time`.\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"### Objective\n",
|
||||
"\n",
|
||||
"- Deploy Vertex AI Model Garden OSS LLMs properly for ADK integration\n",
|
||||
"- Test deployed endpoints\n",
|
||||
"- Build agent web apps with deployed endpoints and ADK\n",
|
||||
"- Deploy agent web apps to Cloud Run\n",
|
||||
"\n",
|
||||
"### Costs\n",
|
||||
"\n",
|
||||
"This tutorial uses billable components of Google Cloud:\n",
|
||||
"\n",
|
||||
"* Vertex AI\n",
|
||||
"* Cloud Storage\n",
|
||||
"\n",
|
||||
"Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing), [Cloud Storage pricing](https://cloud.google.com/storage/pricing), and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "hQJWRopioSKT"
|
||||
},
|
||||
"source": [
|
||||
"## Before you begin"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "J_jmxcIZoSxU",
|
||||
"cellView": "form"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Setup Google Cloud project\n",
|
||||
"\n",
|
||||
"# @markdown 1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
|
||||
"\n",
|
||||
"# @markdown 2. **[Optional]** Set region. If not set, the region will be set automatically according to Colab Enterprise environment.\n",
|
||||
"\n",
|
||||
"REGION = \"\" # @param {type:\"string\"}\n",
|
||||
"\n",
|
||||
"# @markdown 3. If you want to run predictions with A100 80GB or H100 GPUs, we recommend using the regions listed below. **NOTE:** Make sure you have associated quota in selected regions. Click the links to see your current quota for each GPU type: [Nvidia A100 80GB](https://console.cloud.google.com/iam-admin/quotas?metric=aiplatform.googleapis.com%2Fcustom_model_serving_nvidia_a100_80gb_gpus), [Nvidia H100 80GB](https://console.cloud.google.com/iam-admin/quotas?metric=aiplatform.googleapis.com%2Fcustom_model_serving_nvidia_h100_gpus). You can request for quota following the instructions at [\"Request a higher quota\"](https://cloud.google.com/docs/quota/view-manage#requesting_higher_quota).\n",
|
||||
"\n",
|
||||
"# @markdown \u003e | Machine Type | Accelerator Type | Recommended Regions |\n",
|
||||
"# @markdown | ----------- | ----------- | ----------- |\n",
|
||||
"# @markdown | a2-ultragpu-1g | 1 NVIDIA_A100_80GB | us-central1, us-east4, europe-west4, asia-southeast1, us-east4 |\n",
|
||||
"# @markdown | a3-highgpu-2g | 2 NVIDIA_H100_80GB | us-west1, asia-southeast1, europe-west4 |\n",
|
||||
"# @markdown | a3-highgpu-4g | 4 NVIDIA_H100_80GB | us-west1, asia-southeast1, europe-west4 |\n",
|
||||
"# @markdown | a3-highgpu-8g | 8 NVIDIA_H100_80GB | us-central1, europe-west4, us-west1, asia-southeast1 |\n",
|
||||
"\n",
|
||||
"# Upgrade Vertex AI SDK.\n",
|
||||
"! pip3 install --upgrade --quiet 'google-cloud-aiplatform\u003e=1.84.0'\n",
|
||||
"! pip install -qU openai google-auth requests\n",
|
||||
"\n",
|
||||
"# Import the necessary packages\n",
|
||||
"import datetime\n",
|
||||
"import importlib\n",
|
||||
"import os\n",
|
||||
"import uuid\n",
|
||||
"from typing import Tuple\n",
|
||||
"\n",
|
||||
"from google.cloud import aiplatform\n",
|
||||
"\n",
|
||||
"if not os.path.exists(\"./vertex-ai-samples\"):\n",
|
||||
" ! git clone https://github.com/GoogleCloudPlatform/vertex-ai-samples.git\n",
|
||||
"\n",
|
||||
"models, endpoints = {}, {}\n",
|
||||
"\n",
|
||||
"common_util = importlib.import_module(\n",
|
||||
" \"vertex-ai-samples.community-content.vertex_model_garden.model_oss.notebook_util.common_util\"\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# Get the default cloud project id.\n",
|
||||
"PROJECT_ID = os.environ[\"GOOGLE_CLOUD_PROJECT\"]\n",
|
||||
"\n",
|
||||
"# Get the default region for launching jobs.\n",
|
||||
"if not REGION:\n",
|
||||
" if not os.environ.get(\"GOOGLE_CLOUD_REGION\"):\n",
|
||||
" raise ValueError(\n",
|
||||
" \"REGION must be set. See\"\n",
|
||||
" \" https://cloud.google.com/vertex-ai/docs/general/locations for\"\n",
|
||||
" \" available cloud locations.\"\n",
|
||||
" )\n",
|
||||
" REGION = os.environ[\"GOOGLE_CLOUD_REGION\"]\n",
|
||||
"\n",
|
||||
"# Enable the Vertex AI API and Compute Engine API, if not already.\n",
|
||||
"print(\"Enabling Vertex AI API and Compute Engine API.\")\n",
|
||||
"! gcloud services enable aiplatform.googleapis.com compute.googleapis.com\n",
|
||||
"\n",
|
||||
"# Initialize Vertex AI API.\n",
|
||||
"print(\"Initializing Vertex AI API.\")\n",
|
||||
"aiplatform.init(project=PROJECT_ID, location=REGION)\n",
|
||||
"\n",
|
||||
"# Gets the default SERVICE_ACCOUNT.\n",
|
||||
"shell_output = ! gcloud projects describe $PROJECT_ID\n",
|
||||
"project_number = shell_output[-1].split(\":\")[1].strip().replace(\"'\", \"\")\n",
|
||||
"SERVICE_ACCOUNT = f\"{project_number}-compute@developer.gserviceaccount.com\"\n",
|
||||
"print(\"Using this default Service Account:\", SERVICE_ACCOUNT)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n",
|
||||
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n",
|
||||
"\n",
|
||||
"! gcloud config set project $PROJECT_ID\n",
|
||||
"! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n",
|
||||
"! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\"\n",
|
||||
"\n",
|
||||
"from etils import epath\n",
|
||||
"TUTORIAL_DIR = epath.Path(\"vmg_adk_agent_tutorial\")\n",
|
||||
"BUILD_DIR = TUTORIAL_DIR / \"build\"\n",
|
||||
"BUILD_DIR.mkdir(exist_ok=True, parents=True)\n",
|
||||
"\n",
|
||||
"REPOSITORY_NAME = \"vertex-vision-model-garden-dockers\"\n",
|
||||
"\n",
|
||||
"!gcloud artifacts repositories create $REPOSITORY_NAME \\\n",
|
||||
" --repository-format=docker \\\n",
|
||||
" --location=$REGION \\\n",
|
||||
" --project=$PROJECT_ID\n",
|
||||
"\n",
|
||||
"! gcloud auth configure-docker $REGION-docker.pkg.dev --quiet"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"## Integrate OSS LLMs With ADK"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "T9UiFwbiowOV"
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"# @title Deploy OSS LLMs For Agents\n",
|
||||
"# @markdown In order to use OSS LLM endpoints smoothly with ADK,\n",
|
||||
"# @markdown these models should be deployed with:\n",
|
||||
"\n",
|
||||
"# @markdown - **enabling tool calls**. e.g.:\n",
|
||||
"# @markdown If the models are deployed with *vllm*,\n",
|
||||
"# @markdown the deployment should specify settings like `--enable-auto-tool-choice`\n",
|
||||
"# @markdown and `--tool-call-parser=hermes`.\n",
|
||||
"# @markdown If the models are deployed with *sglang*, the deployment should specify\n",
|
||||
"# @markdown setting like `--tool-call-parser=qwen25`.\n",
|
||||
"# @markdown Refer to tool calls in\n",
|
||||
"# @markdown [vllm](https://docs.vllm.ai/en/stable/features/tool_calling.html)\n",
|
||||
"# @markdown and [sglang](https://docs.sglang.ai/backend/function_calling.html) for more details.\n",
|
||||
"\n",
|
||||
"# @markdown - **disable dedicated endpoints**. The dedicated endpoints are not\n",
|
||||
"# @markdown supported in ADK yet.\n",
|
||||
"\n",
|
||||
"# @markdown You can deploy models below with proper deployment settings for ADK integration.\n",
|
||||
"\n",
|
||||
"MODEL_ID = \"Qwen3-32B\" # @param [\"Qwen3-32B\"] {isTemplate: true}\n",
|
||||
"accelerator_type = \"NVIDIA_H100_80GB\" # @param [\"NVIDIA_L4\", \"NVIDIA_A100_80GB\", \"NVIDIA_H100_80GB\"] {isTemplate: true}\n",
|
||||
"\n",
|
||||
"if accelerator_type == \"NVIDIA_L4\":\n",
|
||||
" accelerator_count = 4\n",
|
||||
" # Sets machine type to g2-standard-48 for 4 L4's\n",
|
||||
" machine_type = \"g2-standard-48\"\n",
|
||||
"elif accelerator_type == \"NVIDIA_A100_80GB\":\n",
|
||||
" accelerator_count = 1\n",
|
||||
" # Sets machine type to a2-ultragpu-1g for 1 Nvidia A100 80 GB.\n",
|
||||
" machine_type = \"a2-ultragpu-1g\"\n",
|
||||
"elif accelerator_type == \"NVIDIA_H100_80GB\":\n",
|
||||
" accelerator_count = 2\n",
|
||||
" machine_type = \"a3-highgpu-2g\"\n",
|
||||
"\n",
|
||||
"else:\n",
|
||||
" raise ValueError(\n",
|
||||
" \"Recommended machine settings not found for accelerator type: %s\"\n",
|
||||
" % accelerator_type\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
"deploy_request_timeout = 1800 # 30 minutes\n",
|
||||
"\n",
|
||||
"from vertexai.preview import model_garden\n",
|
||||
"publisher_model_name = f\"publishers/qwen/models/qwen3@{MODEL_ID.lower()}\"\n",
|
||||
"model = model_garden.OpenModel(publisher_model_name)\n",
|
||||
"\n",
|
||||
"container_spec = model.list_deploy_options()[0].container_spec\n",
|
||||
"updated_args = container_spec.args[:-2] + [f\"--tp={accelerator_count}\", \"--tool-call-parser=qwen25\"]\n",
|
||||
"container_spec.args = updated_args\n",
|
||||
"\n",
|
||||
"print(\"The container spec are:\")\n",
|
||||
"print(container_spec)\n",
|
||||
"\n",
|
||||
"print(\"Start to check quota for the deployment.\")\n",
|
||||
"common_util.check_quota(\n",
|
||||
" project_id=PROJECT_ID,\n",
|
||||
" region=REGION,\n",
|
||||
" accelerator_type=accelerator_type,\n",
|
||||
" accelerator_count=accelerator_count,\n",
|
||||
" is_for_training=False,\n",
|
||||
")\n",
|
||||
"print(\"Finished to check quota for the deployment.\")\n",
|
||||
"\n",
|
||||
"print(\"Start to deploy models to endpoints.\")\n",
|
||||
"endpoint = model.deploy(\n",
|
||||
" serving_container_spec=container_spec,\n",
|
||||
" machine_type=machine_type,\n",
|
||||
" accelerator_type=accelerator_type,\n",
|
||||
" accelerator_count=accelerator_count,\n",
|
||||
" use_dedicated_endpoint=False,\n",
|
||||
" spot=False,\n",
|
||||
" deploy_request_timeout=deploy_request_timeout,\n",
|
||||
" accept_eula=False,\n",
|
||||
")\n",
|
||||
"print(\"Finished to deploy models to endpoints.\")\n",
|
||||
"# @markdown After endpoints are deployed successfully, you get the endpoint\n",
|
||||
"# @markdown resource name with the format as\n",
|
||||
"# @markdown `projects/{PROJECT_ID}/locations/{REGION}/endpoints/{ENDPOINT_ID}`.\n",
|
||||
"# @markdown The endpoint resource name will be used in local predictions and\n",
|
||||
"# @markdown integration with ADK below.\n",
|
||||
"endpoint_resource_name = endpoint.resource_name\n",
|
||||
"print(\"The deployed endpoint resource name is:\")\n",
|
||||
"print(endpoint_resource_name)\n",
|
||||
"# @markdown Click \"Show Code\" to see more details.\n"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "gbfFZLT5KkUV",
|
||||
"cellView": "form"
|
||||
},
|
||||
"execution_count": null,
|
||||
"outputs": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"# @title Test The Endpoint\n",
|
||||
"# endpoint_resource_name = (\n",
|
||||
"# f\"projects/{PROJECT_ID}/locations/{REGION}/endpoints/{ENDPOINT_NAME}\"\n",
|
||||
"# )\n",
|
||||
"endpoint = aiplatform.Endpoint(endpoint_resource_name)\n",
|
||||
"\n",
|
||||
"location = endpoint_resource_name.split('/')[3]\n",
|
||||
"base_url = f\"https://{location}-aiplatform.googleapis.com/v1beta1/{endpoint.resource_name}\"\n",
|
||||
"\n",
|
||||
"# @markdown Predict locally with some requests.\n",
|
||||
"\n",
|
||||
"user_message = \"How is your day going?\" # @param {type: \"string\"}\n",
|
||||
"# @markdown If you encounter the issue like `ServiceUnavailable: 503 Took too long to respond when processing`, you can reduce the maximum number of output tokens, such as set `max_tokens` as 20.\n",
|
||||
"max_tokens = 100 # @param {type: \"integer\"}\n",
|
||||
"temperature = 1.0 # @param {type: \"number\"}\n",
|
||||
"stream = True # @param {type: \"boolean\"}\n",
|
||||
"\n",
|
||||
"import google.auth\n",
|
||||
"import openai\n",
|
||||
"\n",
|
||||
"creds, project = google.auth.default()\n",
|
||||
"auth_req = google.auth.transport.requests.Request()\n",
|
||||
"creds.refresh(auth_req)\n",
|
||||
"\n",
|
||||
"client = openai.OpenAI(base_url=base_url, api_key=creds.token)\n",
|
||||
"\n",
|
||||
"model_response = client.chat.completions.create(\n",
|
||||
" model=\"\",\n",
|
||||
" messages=[{\"role\": \"user\", \"content\": user_message}],\n",
|
||||
" temperature=temperature,\n",
|
||||
" max_tokens=max_tokens,\n",
|
||||
" stream=stream,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"if stream:\n",
|
||||
" usage = None\n",
|
||||
" contents = []\n",
|
||||
" for chunk in model_response:\n",
|
||||
" if chunk.usage is not None:\n",
|
||||
" usage = chunk.usage\n",
|
||||
" continue\n",
|
||||
" print(chunk.choices[0].delta.content, end=\"\")\n",
|
||||
" contents.append(chunk.choices[0].delta.content)\n",
|
||||
" print(f\"\\n\\n{usage}\")\n",
|
||||
"else:\n",
|
||||
" print(model_response.choices[0].message.content)"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "_MNVrfJomCd0",
|
||||
"cellView": "form"
|
||||
},
|
||||
"execution_count": null,
|
||||
"outputs": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "Aa4e1-6FvRAP",
|
||||
"cellView": "form"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Build Agent Web App Dockers With VMG Endpoints\n",
|
||||
"# @markdown The section will create required python and docker files first, and\n",
|
||||
"# @markdown then build the dockers with cloud build.\n",
|
||||
"\n",
|
||||
"# @markdown 1. Create `agent.py` by loading VMG endpoints and example tool functions.\n",
|
||||
"agent_app = '''\n",
|
||||
"\"\"\"This is a sample agent for model garden agents.\"\"\"\n",
|
||||
"\n",
|
||||
"import datetime\n",
|
||||
"import os\n",
|
||||
"import re\n",
|
||||
"import zoneinfo\n",
|
||||
"\n",
|
||||
"from google.adk.agents import LlmAgent\n",
|
||||
"from google.adk.models.lite_llm import LiteLlm\n",
|
||||
"import google.auth\n",
|
||||
"\n",
|
||||
"_MODEL_GARDEN_ENDPOINT_REGEX = r\"projects\\/.+\\/locations\\/.+\\/endpoints\\/.+\"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def get_weather(city: str) -\u003e str:\n",
|
||||
" \"\"\"Simulates a web search. Use it get information on weather.\n",
|
||||
"\n",
|
||||
" Args:\n",
|
||||
" city: A string containing the location to get weather information for.\n",
|
||||
"\n",
|
||||
" Returns:\n",
|
||||
" A string with the simulated weather information for the queried city.\n",
|
||||
" \"\"\"\n",
|
||||
" if \"sf\" in city.lower() or \"san francisco\" in city.lower():\n",
|
||||
" return \"It's 70 degrees and foggy.\"\n",
|
||||
" return \"It's 80 degrees and sunny.\"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def get_current_time(city: str) -\u003e str:\n",
|
||||
" \"\"\"Simulates getting the current time for a city.\n",
|
||||
"\n",
|
||||
" Args:\n",
|
||||
" city: The name of the city to get the current time for.\n",
|
||||
"\n",
|
||||
" Returns:\n",
|
||||
" A string with the current time information.\n",
|
||||
" \"\"\"\n",
|
||||
" if \"sf\" in city.lower() or \"san francisco\" in city.lower():\n",
|
||||
" tz_identifier = \"America/Los_Angeles\"\n",
|
||||
" else:\n",
|
||||
" return f\"Sorry, I don't have timezone information for city: {city}.\"\n",
|
||||
"\n",
|
||||
" tz = zoneinfo.ZoneInfo(tz_identifier)\n",
|
||||
" now = datetime.datetime.now(tz)\n",
|
||||
" return (\n",
|
||||
" f\"The current time for city {city} is\"\n",
|
||||
" f\" {now.strftime('%Y-%m-%d %H:%M:%S %Z%z')}\"\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def _get_auth_headers() -\u003e dict[str, str]:\n",
|
||||
" \"\"\"Gets the auth headers for the model garden endpoint.\"\"\"\n",
|
||||
" creds, _ = google.auth.default(\n",
|
||||
" scopes=[\"https://www.googleapis.com/auth/cloud-platform\"]\n",
|
||||
" )\n",
|
||||
" auth_req = google.auth.transport.requests.Request()\n",
|
||||
" creds.refresh(auth_req)\n",
|
||||
" return {\n",
|
||||
" \"Content-Type\": \"application/json\",\n",
|
||||
" \"Authorization\": f\"Bearer {creds.token}\",\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def _setup_model_garden_endpoint():\n",
|
||||
" \"\"\"Sets up the model garden endpoint.\"\"\"\n",
|
||||
" endpoint = os.environ.get(\"GOOGLE_MODEL_GARDEN_ENDPOINT\", \"\")\n",
|
||||
"\n",
|
||||
" if not re.compile(_MODEL_GARDEN_ENDPOINT_REGEX).fullmatch(endpoint):\n",
|
||||
" raise ValueError(\n",
|
||||
" f\"Invalid model garden endpoint: {endpoint}. Please use the format\"\n",
|
||||
" \" projects/{project}/locations/{location}/endpoints/{endpoint}.\"\n",
|
||||
" )\n",
|
||||
" endpoint_parts = endpoint.split(\"/\")\n",
|
||||
" os.environ.setdefault(\"GOOGLE_GENAI_USE_VERTEXAI\", \"True\")\n",
|
||||
" os.environ[\"VERTEXAI_PROJECT\"] = endpoint_parts[1]\n",
|
||||
" os.environ[\"VERTEXAI_LOCATION\"] = endpoint_parts[3]\n",
|
||||
" os.environ[\"LITELLM_LOG\"] = \"DEBUG\"\n",
|
||||
" return f\"vertex_ai/openai/{endpoint_parts[5]}\"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"auth_headers = _get_auth_headers()\n",
|
||||
"model = _setup_model_garden_endpoint()\n",
|
||||
"\n",
|
||||
"print(\"The current model is: {model}\")\n",
|
||||
"\n",
|
||||
"root_agent = LlmAgent(\n",
|
||||
" name=\"root_agent\",\n",
|
||||
" model=LiteLlm(\n",
|
||||
" model=model,\n",
|
||||
" extra_headers=auth_headers,\n",
|
||||
" ),\n",
|
||||
" instruction=(\n",
|
||||
" \"You are a helpful AI assistant designed to provide accurate and useful\"\n",
|
||||
" \" information. Please output the tool callings with json format if\"\n",
|
||||
" \" exists.\"\n",
|
||||
" ),\n",
|
||||
" description=\"Retrieves the weather and current time using specific tools.\",\n",
|
||||
" tools=[get_weather, get_current_time],\n",
|
||||
")\n",
|
||||
"'''\n",
|
||||
"with BUILD_DIR.joinpath(\"agent.py\").open(\"w\") as f:\n",
|
||||
" f.write(agent_app)\n",
|
||||
"\n",
|
||||
"# @markdown 2. Create `__init__.py` to load agent.py for ADK apps.\n",
|
||||
"initialize = '''\n",
|
||||
"from . import agent\n",
|
||||
"'''\n",
|
||||
"with BUILD_DIR.joinpath(\"__init__.py\").open(\"w\") as f:\n",
|
||||
" f.write(initialize)\n",
|
||||
"\n",
|
||||
"# @markdown 3. Create `Dockerfile` to build agent app dockers.\n",
|
||||
"dockerfile_content = '''\n",
|
||||
"FROM python:3.11-slim\n",
|
||||
"WORKDIR /app\n",
|
||||
"RUN adduser --disabled-password --gecos \"\" myuser\n",
|
||||
"RUN chown -R myuser:myuser /app\n",
|
||||
"USER myuser\n",
|
||||
"ENV PATH=\"/home/myuser/.local/bin:$PATH\"\n",
|
||||
"RUN pip install \\\n",
|
||||
" google-adk~=0.4.0 \\\n",
|
||||
" google-cloud-logging~=3.11.4 \\\n",
|
||||
" opentelemetry-exporter-gcp-trace~=1.9.0 \\\n",
|
||||
" google-cloud-aiplatform[evaluation,agent-engines]~=1.88.0 \\\n",
|
||||
" litellm~=1.66.2\n",
|
||||
"\n",
|
||||
"COPY agent.py \"/app/agents/model_garden_agents/\"\n",
|
||||
"COPY __init__.py \"/app/agents/model_garden_agents/\"\n",
|
||||
"ENV GOOGLE_MODEL_GARDEN_ENDPOINT YOUR_ENDPOINT\n",
|
||||
"EXPOSE 8000\n",
|
||||
"CMD adk web --port=8000 --trace_to_cloud \"/app/agents\"\n",
|
||||
"'''\n",
|
||||
"with BUILD_DIR.joinpath(\"Dockerfile\").open(\"w\") as f:\n",
|
||||
" f.write(dockerfile_content)\n",
|
||||
"\n",
|
||||
"# @markdown 4. Build agent web app dockers.\n",
|
||||
"VMG_AGENT_UI_CONTAINER_IMAGE_URI = (\n",
|
||||
" f\"{REGION}-docker.pkg.dev/{PROJECT_ID}/{REPOSITORY_NAME}/vmg-adk-ui\"\n",
|
||||
")\n",
|
||||
"! gcloud builds submit --tag $VMG_AGENT_UI_CONTAINER_IMAGE_URI --project $PROJECT_ID --machine-type e2-highcpu-32 $BUILD_DIR\n",
|
||||
"print(\"The agent UI docker is :\")\n",
|
||||
"print(VMG_AGENT_UI_CONTAINER_IMAGE_URI)\n",
|
||||
"# @markdown Click \"Show Code\" to see more details."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"# @title Deploy Agent Web App Dockers To Cloud Run\n",
|
||||
"# @markdown After the deployment, there will be a service URL. You can click\n",
|
||||
"# @markdown the service URL and interact with the agent web app. The agent web\n",
|
||||
"# @markdown is a built-in development UI in [ADK](https://github.com/google/adk-python?tab=readme-ov-file)\n",
|
||||
"# @markdown to help you test, evaluate, debug, and showcase your agent(s).\n",
|
||||
"\n",
|
||||
"# @markdown \n",
|
||||
"\n",
|
||||
"! gcloud run deploy vmg-agent-ui-1 \\\n",
|
||||
" --port 8000 \\\n",
|
||||
" --image=\"{VMG_AGENT_UI_CONTAINER_IMAGE_URI}\" \\\n",
|
||||
" --region=\"{REGION}\" \\\n",
|
||||
" --platform=managed \\\n",
|
||||
" --allow-unauthenticated \\\n",
|
||||
" --memory=1024Mi \\\n",
|
||||
" --set-env-vars=\"GOOGLE_MODEL_GARDEN_ENDPOINT={endpoint_resource_name}\"\n",
|
||||
"# @markdown Click \"Show Code\" to see more details."
|
||||
],
|
||||
"metadata": {
|
||||
"id": "L2AGmQZmuVam",
|
||||
"cellView": "form"
|
||||
},
|
||||
"execution_count": null,
|
||||
"outputs": []
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "tAelDidov5AW"
|
||||
},
|
||||
"source": [
|
||||
"## Clean up resources"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "8SeZCFo5v7z-"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @markdown Delete the experiment resources to avoid unnecessary continuous\n",
|
||||
"# @markdown charges that may incur.\n",
|
||||
"\n",
|
||||
"delete_endpoint = False # @param {type:\"boolean\"}\n",
|
||||
"delete_artifact_registry = False # @param {type:\"boolean\"}\n",
|
||||
"delete_tutorial_folder = False # @param {type:\"boolean\"}\n",
|
||||
"\n",
|
||||
"if delete_endpoint:\n",
|
||||
" # Undeploy model and delete endpoint.\n",
|
||||
" endpoint.delete(force=True)\n",
|
||||
"\n",
|
||||
"if delete_artifact_registry:\n",
|
||||
" ! gcloud artifacts repositories delete $REPOSITORY_NAME \\\n",
|
||||
" --repository-format=docker \\\n",
|
||||
" --location=$REGION \\\n",
|
||||
" --project=$PROJECT_ID\n",
|
||||
"\n",
|
||||
"if delete_tutorial_folder:\n",
|
||||
" import shutil\n",
|
||||
" shutil.rmtree(TUTORIAL_DIR)\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"name": "model_garden_integration_with_adk.ipynb",
|
||||
"toc_visible": true,
|
||||
"provenance": [],
|
||||
"collapsed_sections": [
|
||||
"tAelDidov5AW"
|
||||
]
|
||||
},
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"name": "python3"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 0
|
||||
}
|
||||
@@ -26,7 +26,6 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"language": "markdown",
|
||||
"metadata": {
|
||||
"id": "VJWDivOv3OWy"
|
||||
},
|
||||
@@ -35,6 +34,11 @@
|
||||
"\n",
|
||||
"<table><tbody><tr>\n",
|
||||
" <td style=\"text-align: center\">\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/workbench/instances\">\n",
|
||||
" <img alt=\"Workbench logo\" src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" width=\"32px\"><br> Run in Workbench\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td style=\"text-align: center\">\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/colab/import/https:%2F%2Fraw.githubusercontent.com%2FGoogleCloudPlatform%2Fvertex-ai-samples%2Fmain%2Fnotebooks%2Fcommunity%2Fmodel_garden%2Fmodel_garden_jax_paligemma_deployment.ipynb\">\n",
|
||||
" <img alt=\"Google Cloud Colab Enterprise logo\" src=\"https://lh3.googleusercontent.com/JmcxdQi-qOpctIvWKgPtrzZdJJK-J3sWE1RsfjZNwshCFgE_9fULcNpuXYTilIR2hjwN\" width=\"32px\"><br> Run in Colab Enterprise\n",
|
||||
" </a>\n",
|
||||
@@ -68,6 +72,10 @@
|
||||
" - Detecting objects.\n",
|
||||
"- Create a playground website to use with the PaliGemma Vertex AI Endpoint.\n",
|
||||
"\n",
|
||||
"### File a bug\n",
|
||||
"\n",
|
||||
"File a bug on [GitHub](https://github.com/GoogleCloudPlatform/vertex-ai-samples/issues/new) if you encounter any issue with the notebook.\n",
|
||||
"\n",
|
||||
"### Costs\n",
|
||||
"\n",
|
||||
"This tutorial uses billable components of Google Cloud:\n",
|
||||
@@ -90,7 +98,6 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"language": "python",
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "QvQjsmIJ6Y3f"
|
||||
@@ -100,25 +107,29 @@
|
||||
"# @title Setup Google Cloud project\n",
|
||||
"# @markdown 1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
|
||||
"\n",
|
||||
"# @markdown 2. **[Optional]** [Create a Cloud Storage bucket](https://cloud.google.com/storage/docs/creating-buckets) for storing experiment outputs. Set the BUCKET_URI for the experiment environment. The specified Cloud Storage bucket (`BUCKET_URI`) should be located in the same region as where the notebook was launched. Note that a multi-region bucket (eg. \"us\") is not considered a match for a single region covered by the multi-region range (eg. \"us-central1\"). If not set, a unique GCS bucket will be created instead.\n",
|
||||
"\n",
|
||||
"BUCKET_URI = \"gs://\" # @param {type:\"string\"}\n",
|
||||
"\n",
|
||||
"# @markdown 3. **[Optional]** Set region. If not set, the region will be set automatically according to Colab Enterprise environment.\n",
|
||||
"# @markdown 2. **[Optional]** Set region. If not set, the region will be set automatically according to Colab Enterprise environment.\n",
|
||||
"\n",
|
||||
"REGION = \"\" # @param {type:\"string\"}\n",
|
||||
"\n",
|
||||
"! git clone https://github.com/GoogleCloudPlatform/vertex-ai-samples.git\n",
|
||||
"# @markdown 3. If you want to run predictions with A100 80GB or H100 GPUs, we recommend using the regions listed below. **NOTE:** Make sure you have associated quota in selected regions. Click the links to see your current quota for each GPU type: [Nvidia A100 80GB](https://console.cloud.google.com/iam-admin/quotas?metric=aiplatform.googleapis.com%2Fcustom_model_serving_nvidia_a100_80gb_gpus), [Nvidia H100 80GB](https://console.cloud.google.com/iam-admin/quotas?metric=aiplatform.googleapis.com%2Fcustom_model_serving_nvidia_h100_gpus). You can request for quota following the instructions at [\"Request a higher quota\"](https://cloud.google.com/docs/quota/view-manage#requesting_higher_quota).\n",
|
||||
"\n",
|
||||
"# @markdown > | Machine Type | Accelerator Type | Recommended Regions |\n",
|
||||
"# @markdown | ----------- | ----------- | ----------- |\n",
|
||||
"# @markdown | a2-ultragpu-1g | 1 NVIDIA_A100_80GB | us-central1, us-east4, europe-west4, asia-southeast1, us-east4 |\n",
|
||||
"# @markdown | a3-highgpu-2g | 2 NVIDIA_H100_80GB | us-west1, asia-southeast1, europe-west4 |\n",
|
||||
"# @markdown | a3-highgpu-4g | 4 NVIDIA_H100_80GB | us-west1, asia-southeast1, europe-west4 |\n",
|
||||
"# @markdown | a3-highgpu-8g | 8 NVIDIA_H100_80GB | us-central1, europe-west4, us-west1, asia-southeast1 |\n",
|
||||
"\n",
|
||||
"# Upgrade Vertex AI SDK.\n",
|
||||
"! pip3 install --upgrade --quiet 'google-cloud-aiplatform>=1.84.0'\n",
|
||||
"\n",
|
||||
"# Import the necessary packages\n",
|
||||
"! pip install -q gradio==4.21.0\n",
|
||||
"import datetime\n",
|
||||
"import enum\n",
|
||||
"import importlib\n",
|
||||
"import io\n",
|
||||
"import os\n",
|
||||
"import re\n",
|
||||
"import uuid\n",
|
||||
"from typing import Sequence, Tuple\n",
|
||||
"\n",
|
||||
"import gradio as gr\n",
|
||||
@@ -128,71 +139,38 @@
|
||||
"from google.cloud import aiplatform\n",
|
||||
"from PIL import Image\n",
|
||||
"\n",
|
||||
"# Upgrade Vertex AI SDK.\n",
|
||||
"if os.environ.get(\"VERTEX_PRODUCT\") != \"COLAB_ENTERPRISE\":\n",
|
||||
" ! pip install --upgrade tensorflow\n",
|
||||
"! git clone https://github.com/GoogleCloudPlatform/vertex-ai-samples.git\n",
|
||||
"\n",
|
||||
"common_util = importlib.import_module(\n",
|
||||
" \"vertex-ai-samples.community-content.vertex_model_garden.model_oss.notebook_util.common_util\"\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"LABEL = \"endpoint\"\n",
|
||||
"models, endpoints = {}, {}\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Get the default cloud project id.\n",
|
||||
"PROJECT_ID = os.environ[\"GOOGLE_CLOUD_PROJECT\"]\n",
|
||||
"\n",
|
||||
"# Get the default region for launching jobs.\n",
|
||||
"if not REGION:\n",
|
||||
" if not os.environ.get(\"GOOGLE_CLOUD_REGION\"):\n",
|
||||
" raise ValueError(\n",
|
||||
" \"REGION must be set. See\"\n",
|
||||
" \" https://cloud.google.com/vertex-ai/docs/general/locations for\"\n",
|
||||
" \" available cloud locations.\"\n",
|
||||
" )\n",
|
||||
" REGION = os.environ[\"GOOGLE_CLOUD_REGION\"]\n",
|
||||
"\n",
|
||||
"# Enable the Vertex AI API and Compute Engine API, if not already.\n",
|
||||
"print(\"Enabling Vertex AI API and Compute Engine API.\")\n",
|
||||
"! gcloud services enable aiplatform.googleapis.com compute.googleapis.com\n",
|
||||
"\n",
|
||||
"# Cloud Storage bucket for storing the experiment artifacts.\n",
|
||||
"# A unique GCS bucket will be created for the purpose of this notebook. If you\n",
|
||||
"# prefer using your own GCS bucket, change the value yourself below.\n",
|
||||
"now = datetime.datetime.now().strftime(\"%Y%m%d%H%M%S\")\n",
|
||||
"BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n",
|
||||
"\n",
|
||||
"if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n",
|
||||
" BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n",
|
||||
" BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n",
|
||||
" ! gsutil mb -l {REGION} {BUCKET_URI}\n",
|
||||
"else:\n",
|
||||
" assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n",
|
||||
" shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n",
|
||||
" bucket_region = shell_output[0].strip().lower()\n",
|
||||
" if bucket_region != REGION:\n",
|
||||
" raise ValueError(\n",
|
||||
" \"Bucket region %s is different from notebook region %s\"\n",
|
||||
" % (bucket_region, REGION)\n",
|
||||
" )\n",
|
||||
"print(f\"Using this GCS Bucket: {BUCKET_URI}\")\n",
|
||||
"\n",
|
||||
"STAGING_BUCKET = os.path.join(BUCKET_URI, \"temporal\")\n",
|
||||
"MODEL_BUCKET = os.path.join(BUCKET_URI, \"paligemma\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Initialize Vertex AI API.\n",
|
||||
"print(\"Initializing Vertex AI API.\")\n",
|
||||
"aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=STAGING_BUCKET)\n",
|
||||
"\n",
|
||||
"# Gets the default SERVICE_ACCOUNT.\n",
|
||||
"shell_output = ! gcloud projects describe $PROJECT_ID\n",
|
||||
"project_number = shell_output[-1].split(\":\")[1].strip().replace(\"'\", \"\")\n",
|
||||
"SERVICE_ACCOUNT = f\"{project_number}-compute@developer.gserviceaccount.com\"\n",
|
||||
"print(\"Using this default Service Account:\", SERVICE_ACCOUNT)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n",
|
||||
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n",
|
||||
"aiplatform.init(project=PROJECT_ID, location=REGION)\n",
|
||||
"\n",
|
||||
"! gcloud config set project $PROJECT_ID\n",
|
||||
"! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n",
|
||||
"! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\"\n",
|
||||
"\n",
|
||||
"import vertexai\n",
|
||||
"\n",
|
||||
"vertexai.init(\n",
|
||||
" project=PROJECT_ID,\n",
|
||||
" location=REGION,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# @markdown ### Access PaliGemma models on Vertex AI for GPU based serving\n",
|
||||
"# @markdown Accept the model agreement to access the models:\n",
|
||||
@@ -205,17 +183,28 @@
|
||||
"VERTEX_AI_MODEL_GARDEN_PALIGEMMA = \"gs://\" # @param {type:\"string\", isTemplate:true}\n",
|
||||
"assert (\n",
|
||||
" VERTEX_AI_MODEL_GARDEN_PALIGEMMA\n",
|
||||
"), \"Click the agreement of PaliGemma in Vertex AI Model Garden, and get the GCS path of PaliGemma model artifacts.\"\n",
|
||||
"print(\n",
|
||||
" \"Copying PaliGemma model artifacts from\",\n",
|
||||
" VERTEX_AI_MODEL_GARDEN_PALIGEMMA,\n",
|
||||
" \"to \",\n",
|
||||
" MODEL_BUCKET,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"! gsutil -m cp -R $VERTEX_AI_MODEL_GARDEN_PALIGEMMA/* $MODEL_BUCKET\n",
|
||||
"\n",
|
||||
"model_path_prefix = MODEL_BUCKET\n",
|
||||
"), \"Click the agreement of PaliGemma in Vertex AI Model Garden, and get the GCS path of PaliGemma model artifacts.\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "kyMJXkfviWgl"
|
||||
},
|
||||
"source": [
|
||||
"## Deploy PaliGemma to a Vertex AI Endpoint"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "JThvioAxy8-a"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Select the model variants\n",
|
||||
"\n",
|
||||
"pretrained_filename_lookup = {\n",
|
||||
" \"paligemma-224-float32\": \"pt_224.npz\",\n",
|
||||
@@ -235,6 +224,339 @@
|
||||
" \"paligemma-mix-448-bfloat16\": \"mix_448.bf16.npz\",\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"# @markdown Select the desired resolution and precision of prebuilt model to deploy, leaving the optional `custom_paligemma_model_uri` as is. Higher resolution and precision_type can result in better inference results, but may require additional GPU.\n",
|
||||
"\n",
|
||||
"# @markdown You can also serve a finetuned PaliGemma model by setting `resolution` and `precision_type` to the resolution and precision type of the original base model and then setting `custom_paligemma_model_uri` to the GCS URI containing the model.\n",
|
||||
"\n",
|
||||
"# @markdown **Note**: You cannot use accelerator type `NVIDIA_TESLA_V100` to serve prebuilt or finetuned PaliGemma models with resolution `896` and precision_type `float32`.\n",
|
||||
"\n",
|
||||
"model_variant = \"mix\" # @param [\"mix\", \"pt\"]\n",
|
||||
"resolution = 224 # @param [224, 448, 896]\n",
|
||||
"precision_type = \"float32\" # @param [\"float32\", \"float16\", \"bfloat16\"]\n",
|
||||
"custom_paligemma_model_uri = \"gs://\" # @param {type: \"string\"}\n",
|
||||
"\n",
|
||||
"if model_variant == \"mix\":\n",
|
||||
" model_name_prefix = \"paligemma-mix\"\n",
|
||||
"else:\n",
|
||||
" model_name_prefix = \"paligemma\"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"if custom_paligemma_model_uri == \"gs://\" or not custom_paligemma_model_uri:\n",
|
||||
" model_name = f\"{model_name_prefix}-{resolution}-{precision_type}\"\n",
|
||||
" checkpoint_filename = pretrained_filename_lookup[model_name]\n",
|
||||
" checkpoint_path = os.path.join(\n",
|
||||
" VERTEX_AI_MODEL_GARDEN_PALIGEMMA, checkpoint_filename\n",
|
||||
" )\n",
|
||||
" PUBLISHER_MODEL_NAME = f\"publishers/google/models/paligemma@{model_name}\"\n",
|
||||
"else:\n",
|
||||
" model_name = f\"{model_name_prefix}-{resolution}-{precision_type}-custom\"\n",
|
||||
" checkpoint_path = custom_paligemma_model_uri\n",
|
||||
"\n",
|
||||
"# @markdown If you want to use other accelerator types not listed below, then check other Vertex AI prediction supported accelerators and regions at https://cloud.google.com/vertex-ai/docs/predictions/configure-compute. You may need to manually set the `machine_type`, `accelerator_type`, and `accelerator_count` in the code by clicking `Show code` first.\n",
|
||||
"# @markdown Select the accelerator type to use to deploy the model:\n",
|
||||
"accelerator_type = \"NVIDIA_L4\" # @param [\"NVIDIA_L4\", \"NVIDIA_TESLA_V100\"]\n",
|
||||
"if accelerator_type == \"NVIDIA_L4\":\n",
|
||||
" machine_type = \"g2-standard-16\"\n",
|
||||
" accelerator_count = 1\n",
|
||||
"elif accelerator_type == \"NVIDIA_TESLA_V100\":\n",
|
||||
" if resolution == 896 and precision_type == \"float32\":\n",
|
||||
" raise ValueError(\n",
|
||||
" \"NVIDIA_TESLA_V100 is not sufficient. Multi-gpu is not supported for PaLIGemma.\"\n",
|
||||
" )\n",
|
||||
" else:\n",
|
||||
" machine_type = \"n1-highmem-8\"\n",
|
||||
" accelerator_count = 1\n",
|
||||
"else:\n",
|
||||
" raise ValueError(\n",
|
||||
" f\"Recommended machine settings not found for: {accelerator_type}. To use another another accelerator, edit this code block to pass in an appropriate `machine_type`, `accelerator_type`, and `accelerator_count` to the deploy_model function by clicking `Show Code` and then modifying the code.\"\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
"# @markdown Set use_dedicated_endpoint to False if you don't want to use [dedicated endpoint](https://cloud.google.com/vertex-ai/docs/general/deployment#create-dedicated-endpoint). Note that [dedicated endpoint does not support VPC Service Controls](https://cloud.google.com/vertex-ai/docs/predictions/choose-endpoint-type), uncheck the box if you are using VPC-SC.\n",
|
||||
"use_dedicated_endpoint = True # @param {type:\"boolean\"}\n",
|
||||
"\n",
|
||||
"common_util.check_quota(\n",
|
||||
" project_id=PROJECT_ID,\n",
|
||||
" region=REGION,\n",
|
||||
" accelerator_type=accelerator_type,\n",
|
||||
" accelerator_count=accelerator_count,\n",
|
||||
" is_for_training=False,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# @markdown Click \"Show Code\" to see more details."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "144IKkHrzrMs"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title [Option 1] Deploy with Model Garden SDK\n",
|
||||
"\n",
|
||||
"# @markdown Kindly note that the deployment using custom_paligemma_model_uri is not supported.\n",
|
||||
"\n",
|
||||
"# @markdown Deploy with Gen AI model-centric SDK. This section uploads the prebuilt model to Model Registry and deploys it to a Vertex AI Endpoint. It takes 15 minutes to 1 hour to finish depending on the size of the model. See [use open models with Vertex AI](https://cloud.google.com/vertex-ai/generative-ai/docs/open-models/use-open-models) for documentation on other use cases.\n",
|
||||
"from vertexai.preview import model_garden\n",
|
||||
"\n",
|
||||
"model = model_garden.OpenModel(PUBLISHER_MODEL_NAME)\n",
|
||||
"endpoints[LABEL] = model.deploy(\n",
|
||||
" machine_type=machine_type,\n",
|
||||
" accelerator_type=accelerator_type,\n",
|
||||
" accelerator_count=accelerator_count,\n",
|
||||
" use_dedicated_endpoint=use_dedicated_endpoint,\n",
|
||||
" accept_eula=True, # Accept the End User License Agreement (EULA) on the model card before deploy. Otherwise, the deployment will be forbidden.\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "toY-WPKDFesF"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title [Option 2] Deploy with custom configs\n",
|
||||
"\n",
|
||||
"# @markdown This section uploads the prebuilt PaliGemma model to Model Registry and deploys it to a Vertex AI Endpoint. It takes approximately 15 minutes to finish.\n",
|
||||
"\n",
|
||||
"# The pre-built serving docker image.\n",
|
||||
"SERVE_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/jax-paligemma-serve-gpu:20240807_0916_RC00\"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def deploy_model(\n",
|
||||
" model_name: str,\n",
|
||||
" checkpoint_path: str,\n",
|
||||
" machine_type: str = \"g2-standard-32\",\n",
|
||||
" accelerator_type: str = \"NVIDIA_L4\",\n",
|
||||
" accelerator_count: int = 1,\n",
|
||||
" resolution: int = 224,\n",
|
||||
" use_dedicated_endpoint: bool = False,\n",
|
||||
") -> Tuple[aiplatform.Model, aiplatform.Endpoint]:\n",
|
||||
" \"\"\"Create a Vertex AI Endpoint and deploy the specified model to the endpoint.\"\"\"\n",
|
||||
" model_name_with_time = common_util.get_job_name_with_datetime(model_name)\n",
|
||||
" endpoint = aiplatform.Endpoint.create(\n",
|
||||
" display_name=f\"{model_name_with_time}-endpoint\",\n",
|
||||
" dedicated_endpoint_enabled=use_dedicated_endpoint,\n",
|
||||
" )\n",
|
||||
" model = aiplatform.Model.upload(\n",
|
||||
" display_name=model_name_with_time,\n",
|
||||
" serving_container_image_uri=SERVE_DOCKER_URI,\n",
|
||||
" serving_container_ports=[8080],\n",
|
||||
" serving_container_predict_route=\"/predict\",\n",
|
||||
" serving_container_health_route=\"/health\",\n",
|
||||
" serving_container_environment_variables={\n",
|
||||
" \"CKPT_PATH\": checkpoint_path,\n",
|
||||
" \"RESOLUTION\": resolution,\n",
|
||||
" \"MODEL_ID\": \"google/\" + model_name,\n",
|
||||
" \"DEPLOY_SOURCE\": \"notebook\",\n",
|
||||
" },\n",
|
||||
" model_garden_source_model_name=\"publishers/google/models/paligemma\",\n",
|
||||
" )\n",
|
||||
" print(\n",
|
||||
" f\"Deploying {model_name_with_time} on {machine_type} with {accelerator_count} {accelerator_type} GPU(s).\"\n",
|
||||
" )\n",
|
||||
" model.deploy(\n",
|
||||
" endpoint=endpoint,\n",
|
||||
" machine_type=machine_type,\n",
|
||||
" accelerator_type=accelerator_type,\n",
|
||||
" accelerator_count=accelerator_count,\n",
|
||||
" deploy_request_timeout=1800,\n",
|
||||
" enable_access_logging=True,\n",
|
||||
" min_replica_count=1,\n",
|
||||
" sync=True,\n",
|
||||
" system_labels={\n",
|
||||
" \"NOTEBOOK_NAME\": \"model_garden_jax_paligemma_deployment.ipynb\",\n",
|
||||
" \"NOTEBOOK_ENVIRONMENT\": common_util.get_deploy_source(),\n",
|
||||
" },\n",
|
||||
" )\n",
|
||||
" return model, endpoint\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"models[LABEL], endpoints[LABEL] = deploy_model(\n",
|
||||
" model_name=model_name,\n",
|
||||
" checkpoint_path=checkpoint_path,\n",
|
||||
" machine_type=machine_type,\n",
|
||||
" accelerator_type=accelerator_type,\n",
|
||||
" accelerator_count=accelerator_count,\n",
|
||||
" resolution=resolution,\n",
|
||||
" use_dedicated_endpoint=use_dedicated_endpoint,\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "tOtYOhZa3lsx"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title [Optional] Loading an existing Endpoint\n",
|
||||
"# @markdown If you've already deployed an Endpoint, you can load it by filling in the Endpoint's ID below.\n",
|
||||
"# @markdown You can view deployed Endpoints at [Vertex Online Prediction](https://console.cloud.google.com/vertex-ai/online-prediction/endpoints).\n",
|
||||
"endpoint_id = \"\" # @param {type: \"string\"}\n",
|
||||
"\n",
|
||||
"if endpoint_id:\n",
|
||||
" endpoints[LABEL] = aiplatform.Endpoint(\n",
|
||||
" endpoint_name=endpoint_id,\n",
|
||||
" project=PROJECT_ID,\n",
|
||||
" location=REGION,\n",
|
||||
" )"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "MlP2Y7XE4SS5"
|
||||
},
|
||||
"source": [
|
||||
"### Predict\n",
|
||||
"\n",
|
||||
"The following sections will use images from [pexels.com](https://www.pexels.com/) for demoing purposes. All the images have the following license: https://www.pexels.com/license/.\n",
|
||||
"\n",
|
||||
"Images will be resized to a width of 1000 pixels by default since requests made to a Vertex Endpoint are limited to 1.500MB."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "xnZw8wNyQhmN"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Visual Question Answering\n",
|
||||
"\n",
|
||||
"# @markdown This section uses the deployed PaliGemma model to answer questions about a given image.\n",
|
||||
"\n",
|
||||
"# @markdown Once deployment succeeds, you can send requests to the endpoint with images and questions.\n",
|
||||
"# @markdown \n",
|
||||
"image_url = \"https://images.pexels.com/photos/4012966/pexels-photo-4012966.jpeg\" # @param {type:\"string\"}\n",
|
||||
"\n",
|
||||
"image = common_util.download_image(image_url)\n",
|
||||
"display(image)\n",
|
||||
"\n",
|
||||
"# @markdown You may leave question prompts empty and they will be ignored.\n",
|
||||
"question_prompt_1 = \"Which of laptop, book, pencil, clock, flower are in the image?\" # @param {type: \"string\"}\n",
|
||||
"question_prompt_2 = \"Do the book and the cup have the same color?\" # @param {type: \"string\"}\n",
|
||||
"question_prompt_3 = \"Is there a person in the image?\" # @param {type: \"string\"}\n",
|
||||
"question_prompt_4 = \"How many laptop are in the image?\" # @param {type: \"string\"}\n",
|
||||
"question_prompt_5 = \"桌子是什么颜色的?\" # @param {type: \"string\"}\n",
|
||||
"\n",
|
||||
"# @markdown The question prompt can be non-English languages.\n",
|
||||
"questions_list = [\n",
|
||||
" question_prompt_1,\n",
|
||||
" question_prompt_2,\n",
|
||||
" question_prompt_3,\n",
|
||||
" question_prompt_4,\n",
|
||||
" question_prompt_5,\n",
|
||||
"]\n",
|
||||
"questions = [question for question in questions_list if question]\n",
|
||||
"\n",
|
||||
"answers = common_util.vqa_predict(\n",
|
||||
" endpoints[\"endpoint\"],\n",
|
||||
" questions,\n",
|
||||
" image,\n",
|
||||
" use_dedicated_endpoint=use_dedicated_endpoint,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"for question, answer in zip(questions, answers):\n",
|
||||
" print(f\"Question: {question}\")\n",
|
||||
" print(f\"Answer: {answer}\")\n",
|
||||
"# @markdown Click \"Show Code\" to see more details."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "mF1MxC1ouzqj"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Image Captioning\n",
|
||||
"# @markdown This section uses the deployed PaliGemma model to caption and describe an image in a chosen language.\n",
|
||||
"\n",
|
||||
"caption_prompt = True\n",
|
||||
"\n",
|
||||
"# @markdown <img src=\"https://storage.googleapis.com/longcap100/91.jpeg\" width=\"400\" >\n",
|
||||
"\n",
|
||||
"image_url = \"https://storage.googleapis.com/longcap100/91.jpeg\" # @param {type:\"string\"}\n",
|
||||
"\n",
|
||||
"language_code = \"en\" # @param {type: \"string\"}\n",
|
||||
"\n",
|
||||
"image = common_util.download_image(image_url)\n",
|
||||
"display(image)\n",
|
||||
"\n",
|
||||
"# Make a prediction.\n",
|
||||
"image_base64 = common_util.image_to_base64(image)\n",
|
||||
"\n",
|
||||
"caption = common_util.caption_predict(\n",
|
||||
" endpoints[\"endpoint\"],\n",
|
||||
" language_code,\n",
|
||||
" image,\n",
|
||||
" caption_prompt,\n",
|
||||
" use_dedicated_endpoint=use_dedicated_endpoint,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"print(\"Caption: \", caption)\n",
|
||||
"# @markdown Click \"Show Code\" to see more details."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "TtkXMZTIegLq"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title OCR\n",
|
||||
"# @markdown This section uses the deployed PaliGemma model to extract text from an image, starting from the top left.\n",
|
||||
"ocr_prompt = \"ocr\"\n",
|
||||
"\n",
|
||||
"# @markdown \n",
|
||||
"image_url = \"https://images.pexels.com/photos/8919535/pexels-photo-8919535.jpeg?auto=compress&cs=tinysrgb&w=1260&h=750&dpr=2\" # @param {type:\"string\"}\n",
|
||||
"\n",
|
||||
"image = common_util.download_image(image_url)\n",
|
||||
"display(image)\n",
|
||||
"text_found = common_util.ocr_predict(\n",
|
||||
" endpoints[\"endpoint\"],\n",
|
||||
" ocr_prompt,\n",
|
||||
" image,\n",
|
||||
" use_dedicated_endpoint=use_dedicated_endpoint,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"print(f\"Text found: {text_found}\")\n",
|
||||
"# @markdown Click \"Show Code\" to see more details."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "JlLr3nu-YEon"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Object Detection\n",
|
||||
"# @markdown This section uses the deployed PaliGemma model to output bounding boxes for specified object image in a given image.\n",
|
||||
"# @markdown The text output will be parsed into bounding boxes and overlaid on the original image.\n",
|
||||
"\n",
|
||||
"# @markdown Specify what object to detect. To specify multiple objects, enter them as a semicolon separated list as shown below.\n",
|
||||
"objects = \"plant ; pineapple ; glasses\" # @param {type:\"string\"}\n",
|
||||
"detect_promt = f\"detect {objects}\"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def parse_detections(txt):\n",
|
||||
" \"\"\"Parses bounding boxes from a detection string.\"\"\"\n",
|
||||
@@ -277,296 +599,8 @@
|
||||
" buf = io.BytesIO()\n",
|
||||
" fig.savefig(buf)\n",
|
||||
" buf.seek(0)\n",
|
||||
" return Image.open(buf)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "kyMJXkfviWgl"
|
||||
},
|
||||
"source": [
|
||||
"## Deploy PaliGemma to a Vertex AI Endpoint"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "toY-WPKDFesF"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Deploy\n",
|
||||
" return Image.open(buf)\n",
|
||||
"\n",
|
||||
"# @markdown This section uploads the prebuilt PaliGemma model to Model Registry and deploys it to a Vertex AI Endpoint. It takes approximately 15 minutes to finish.\n",
|
||||
"\n",
|
||||
"# @markdown Select the desired resolution and precision of prebuilt model to deploy, leaving the optional `custom_paligemma_model_uri` as is. Higher resolution and precision_type can result in better inference results, but may require additional GPU.\n",
|
||||
"\n",
|
||||
"# @markdown You can also serve a finetuned PaliGemma model by setting `resolution` and `precision_type` to the resolution and precision type of the original base model and then setting `custom_paligemma_model_uri` to the GCS URI containing the model.\n",
|
||||
"\n",
|
||||
"# @markdown **Note**: You cannot use accelerator type `NVIDIA_TESLA_V100` to serve prebuilt or finetuned PaliGemma models with resolution `896` and precision_type `float32`.\n",
|
||||
"\n",
|
||||
"model_variant = \"mix\" # @param [\"mix\", \"pt\"]\n",
|
||||
"resolution = 224 # @param [224, 448, 896]\n",
|
||||
"precision_type = \"float32\" # @param [\"float32\", \"float16\", \"bfloat16\"]\n",
|
||||
"custom_paligemma_model_uri = \"gs://\" # @param {type: \"string\"}\n",
|
||||
"\n",
|
||||
"if model_variant == \"mix\":\n",
|
||||
" model_name_prefix = \"paligemma-mix\"\n",
|
||||
"else:\n",
|
||||
" model_name_prefix = \"paligemma\"\n",
|
||||
"\n",
|
||||
"if custom_paligemma_model_uri == \"gs://\" or not custom_paligemma_model_uri:\n",
|
||||
" print(\"Deploying prebuilt PaliGemma model.\")\n",
|
||||
" model_name = f\"{model_name_prefix}-{resolution}-{precision_type}\"\n",
|
||||
" checkpoint_filename = pretrained_filename_lookup[model_name]\n",
|
||||
" checkpoint_path = os.path.join(model_path_prefix, checkpoint_filename)\n",
|
||||
"else:\n",
|
||||
" print(\"Deploying custom PaliGemma model.\")\n",
|
||||
" model_name = f\"{model_name_prefix}-{resolution}-{precision_type}-custom\"\n",
|
||||
" checkpoint_path = custom_paligemma_model_uri\n",
|
||||
"\n",
|
||||
"# The pre-built serving docker image.\n",
|
||||
"SERVE_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/jax-paligemma-serve-gpu:20240807_0916_RC00\"\n",
|
||||
"\n",
|
||||
"# @markdown If you want to use other accelerator types not listed below, then check other Vertex AI prediction supported accelerators and regions at https://cloud.google.com/vertex-ai/docs/predictions/configure-compute. You may need to manually set the `machine_type`, `accelerator_type`, and `accelerator_count` in the code by clicking `Show code` first.\n",
|
||||
"# @markdown Select the accelerator type to use to deploy the model:\n",
|
||||
"accelerator_type = \"NVIDIA_L4\" # @param [\"NVIDIA_L4\", \"NVIDIA_TESLA_V100\"]\n",
|
||||
"if accelerator_type == \"NVIDIA_L4\":\n",
|
||||
" machine_type = \"g2-standard-16\"\n",
|
||||
" accelerator_count = 1\n",
|
||||
"elif accelerator_type == \"NVIDIA_TESLA_V100\":\n",
|
||||
" if resolution == 896 and precision_type == \"float32\":\n",
|
||||
" raise ValueError(\n",
|
||||
" \"NVIDIA_TESLA_V100 is not sufficient. Multi-gpu is not supported for PaLIGemma.\"\n",
|
||||
" )\n",
|
||||
" else:\n",
|
||||
" machine_type = \"n1-highmem-8\"\n",
|
||||
" accelerator_count = 1\n",
|
||||
"else:\n",
|
||||
" raise ValueError(\n",
|
||||
" f\"Recommended machine settings not found for: {accelerator_type}. To use another another accelerator, edit this code block to pass in an appropriate `machine_type`, `accelerator_type`, and `accelerator_count` to the deploy_model function by clicking `Show Code` and then modifying the code.\"\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
"common_util.check_quota(\n",
|
||||
" project_id=PROJECT_ID,\n",
|
||||
" region=REGION,\n",
|
||||
" accelerator_type=accelerator_type,\n",
|
||||
" accelerator_count=accelerator_count,\n",
|
||||
" is_for_training=False,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def deploy_model(\n",
|
||||
" model_name: str,\n",
|
||||
" checkpoint_path: str,\n",
|
||||
" machine_type: str = \"g2-standard-32\",\n",
|
||||
" accelerator_type: str = \"NVIDIA_L4\",\n",
|
||||
" accelerator_count: int = 1,\n",
|
||||
" resolution: int = 224,\n",
|
||||
") -> Tuple[aiplatform.Model, aiplatform.Endpoint]:\n",
|
||||
" \"\"\"Create a Vertex AI Endpoint and deploy the specified model to the endpoint.\"\"\"\n",
|
||||
" model_name_with_time = common_util.get_job_name_with_datetime(model_name)\n",
|
||||
" endpoint = aiplatform.Endpoint.create(\n",
|
||||
" display_name=f\"{model_name_with_time}-endpoint\"\n",
|
||||
" )\n",
|
||||
" model = aiplatform.Model.upload(\n",
|
||||
" display_name=model_name_with_time,\n",
|
||||
" serving_container_image_uri=SERVE_DOCKER_URI,\n",
|
||||
" serving_container_ports=[8080],\n",
|
||||
" serving_container_predict_route=\"/predict\",\n",
|
||||
" serving_container_health_route=\"/health\",\n",
|
||||
" serving_container_environment_variables={\n",
|
||||
" \"CKPT_PATH\": checkpoint_path,\n",
|
||||
" \"RESOLUTION\": resolution,\n",
|
||||
" \"MODEL_ID\": \"google/\" + model_name,\n",
|
||||
" },\n",
|
||||
" model_garden_source_model_name=\"publishers/google/models/paligemma\",\n",
|
||||
" )\n",
|
||||
" print(\n",
|
||||
" f\"Deploying {model_name_with_time} on {machine_type} with {accelerator_count} {accelerator_type} GPU(s).\"\n",
|
||||
" )\n",
|
||||
" model.deploy(\n",
|
||||
" endpoint=endpoint,\n",
|
||||
" machine_type=machine_type,\n",
|
||||
" accelerator_type=accelerator_type,\n",
|
||||
" accelerator_count=accelerator_count,\n",
|
||||
" deploy_request_timeout=1800,\n",
|
||||
" service_account=SERVICE_ACCOUNT,\n",
|
||||
" enable_access_logging=True,\n",
|
||||
" min_replica_count=1,\n",
|
||||
" sync=True,\n",
|
||||
" system_labels={\"NOTEBOOK_NAME\": \"model_garden_jax_paligemma_deployment.ipynb\"},\n",
|
||||
" )\n",
|
||||
" return model, endpoint\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"models[\"model\"], endpoints[\"endpoint\"] = deploy_model(\n",
|
||||
" model_name=model_name,\n",
|
||||
" checkpoint_path=checkpoint_path,\n",
|
||||
" machine_type=machine_type,\n",
|
||||
" accelerator_type=accelerator_type,\n",
|
||||
" accelerator_count=accelerator_count,\n",
|
||||
" resolution=resolution,\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "tOtYOhZa3lsx"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title [Optional] Loading an existing Endpoint\n",
|
||||
"# @markdown If you've already deployed an Endpoint, you can load it by filling in the Endpoint's ID below.\n",
|
||||
"# @markdown You can view deployed Endpoints at [Vertex Online Prediction](https://console.cloud.google.com/vertex-ai/online-prediction/endpoints).\n",
|
||||
"endpoint_id = \"\" # @param {type: \"string\"}\n",
|
||||
"\n",
|
||||
"if endpoint_id:\n",
|
||||
" endpoint = aiplatform.Endpoint(\n",
|
||||
" endpoint_name=endpoint_id,\n",
|
||||
" project=PROJECT_ID,\n",
|
||||
" location=REGION,\n",
|
||||
" )"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "MlP2Y7XE4SS5"
|
||||
},
|
||||
"source": [
|
||||
"### Predict\n",
|
||||
"\n",
|
||||
"The following sections will use images from [pexels.com](https://www.pexels.com/) for demoing purposes. All the images have the following license: https://www.pexels.com/license/.\n",
|
||||
"\n",
|
||||
"Images will be resized to a width of 1000 pixels by default since requests made to a Vertex Endpoint are limited to 1.500MB."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"language": "python",
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "xnZw8wNyQhmN"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Visual Question Answering\n",
|
||||
"\n",
|
||||
"# @markdown This section uses the deployed PaliGemma model to answer questions about a given image.\n",
|
||||
"\n",
|
||||
"# @markdown Once deployment succeeds, you can send requests to the endpoint with images and questions.\n",
|
||||
"# @markdown \n",
|
||||
"image_url = \"https://images.pexels.com/photos/4012966/pexels-photo-4012966.jpeg\" # @param {type:\"string\"}\n",
|
||||
"\n",
|
||||
"image = common_util.download_image(image_url)\n",
|
||||
"display(image)\n",
|
||||
"\n",
|
||||
"# @markdown You may leave question prompts empty and they will be ignored.\n",
|
||||
"question_prompt_1 = \"Which of laptop, book, pencil, clock, flower are in the image?\" # @param {type: \"string\"}\n",
|
||||
"question_prompt_2 = \"Do the book and the cup have the same color?\" # @param {type: \"string\"}\n",
|
||||
"question_prompt_3 = \"Is there a person in the image?\" # @param {type: \"string\"}\n",
|
||||
"question_prompt_4 = \"How many laptop are in the image?\" # @param {type: \"string\"}\n",
|
||||
"question_prompt_5 = \"桌子是什么颜色的?\" # @param {type: \"string\"}\n",
|
||||
"\n",
|
||||
"# @markdown The question prompt can be non-English languages.\n",
|
||||
"questions_list = [\n",
|
||||
" question_prompt_1,\n",
|
||||
" question_prompt_2,\n",
|
||||
" question_prompt_3,\n",
|
||||
" question_prompt_4,\n",
|
||||
" question_prompt_5,\n",
|
||||
"]\n",
|
||||
"questions = [question for question in questions_list if question]\n",
|
||||
"\n",
|
||||
"answers = common_util.vqa_predict(endpoints[\"endpoint\"], questions, image)\n",
|
||||
"\n",
|
||||
"for question, answer in zip(questions, answers):\n",
|
||||
" print(f\"Question: {question}\")\n",
|
||||
" print(f\"Answer: {answer}\")\n",
|
||||
"# @markdown Click \"Show Code\" to see more details."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "mF1MxC1ouzqj"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Image Captioning\n",
|
||||
"# @markdown This section uses the deployed PaliGemma model to caption and describe an image in a chosen language.\n",
|
||||
"\n",
|
||||
"caption_prompt = True\n",
|
||||
"\n",
|
||||
"# @markdown <img src=\"https://storage.googleapis.com/longcap100/91.jpeg\" width=\"400\" >\n",
|
||||
"\n",
|
||||
"image_url = \"https://storage.googleapis.com/longcap100/91.jpeg\" # @param {type:\"string\"}\n",
|
||||
"language_code = \"en\" # @param {type: \"string\"}\n",
|
||||
"\n",
|
||||
"image = common_util.download_image(image_url)\n",
|
||||
"display(image)\n",
|
||||
"\n",
|
||||
"# Make a prediction.\n",
|
||||
"image_base64 = common_util.image_to_base64(image)\n",
|
||||
"\n",
|
||||
"caption = common_util.caption_predict(\n",
|
||||
" endpoints[\"endpoint\"], language_code, image, caption_prompt\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"print(\"Caption: \", caption)\n",
|
||||
"# @markdown Click \"Show Code\" to see more details."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "TtkXMZTIegLq"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title OCR\n",
|
||||
"# @markdown This section uses the deployed PaliGemma model to extract text from an image, starting from the top left.\n",
|
||||
"ocr_prompt = \"ocr\"\n",
|
||||
"\n",
|
||||
"# @markdown \n",
|
||||
"image_url = \"https://images.pexels.com/photos/8919535/pexels-photo-8919535.jpeg?auto=compress&cs=tinysrgb&w=1260&h=750&dpr=2\" # @param {type:\"string\"}\n",
|
||||
"\n",
|
||||
"image = common_util.download_image(image_url)\n",
|
||||
"display(image)\n",
|
||||
"text_found = common_util.ocr_predict(endpoints[\"endpoint\"], ocr_prompt, image)\n",
|
||||
"\n",
|
||||
"print(f\"Text found: {text_found}\")\n",
|
||||
"# @markdown Click \"Show Code\" to see more details."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "JlLr3nu-YEon"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Object Detection\n",
|
||||
"# @markdown This section uses the deployed PaliGemma model to output bounding boxes for specified object image in a given image.\n",
|
||||
"# @markdown The text output will be parsed into bounding boxes and overlaid on the original image.\n",
|
||||
"\n",
|
||||
"# @markdown Specify what object to detect. To specify multiple objects, enter them as a semicolon separated list as shown below.\n",
|
||||
"objects = \"plant ; pineapple ; glasses\" # @param {type:\"string\"}\n",
|
||||
"detect_promt = f\"detect {objects}\"\n",
|
||||
"\n",
|
||||
"# @markdown \n",
|
||||
"image_url = \"https://images.pexels.com/photos/1006293/pexels-photo-1006293.jpeg?auto=compress&cs=tinysrgb&w=1260&h=750&dpr=2\" # @param {type:\"string\"}\n",
|
||||
@@ -576,10 +610,15 @@
|
||||
"\n",
|
||||
"# Make a prediction.\n",
|
||||
"detection_response = common_util.detect_predict(\n",
|
||||
" endpoints[\"endpoint\"], detect_promt, image\n",
|
||||
" endpoints[\"endpoint\"],\n",
|
||||
" detect_promt,\n",
|
||||
" image,\n",
|
||||
" use_dedicated_endpoint=use_dedicated_endpoint,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"print(\"Output: \", detection_response)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"bboxes = parse_detections(detection_response)\n",
|
||||
"plot_bounding_boxes(image, bboxes)\n",
|
||||
"# @markdown Click \"Show Code\" to see more details."
|
||||
@@ -706,7 +745,9 @@
|
||||
" resolution = int(resolution)\n",
|
||||
" model, endpoint = deploy_model(\n",
|
||||
" model_name=model_choice,\n",
|
||||
" checkpoint_path=os.path.join(model_path_prefix, checkpoint_filename),\n",
|
||||
" checkpoint_path=os.path.join(\n",
|
||||
" VERTEX_AI_MODEL_GARDEN_PALIGEMMA, checkpoint_filename\n",
|
||||
" ),\n",
|
||||
" machine_type=\"g2-standard-16\",\n",
|
||||
" accelerator_type=\"NVIDIA_L4\",\n",
|
||||
" accelerator_count=1,\n",
|
||||
@@ -891,6 +932,8 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Delete the models and endpoints\n",
|
||||
"\n",
|
||||
"# @markdown Delete the experiment models and endpoints to recycle the resources\n",
|
||||
"# @markdown and avoid unnecessary continuous charges that may incur.\n",
|
||||
"\n",
|
||||
@@ -900,11 +943,7 @@
|
||||
"\n",
|
||||
"# Delete models.\n",
|
||||
"for model in models.values():\n",
|
||||
" model.delete()\n",
|
||||
"\n",
|
||||
"delete_bucket = False # @param {type:\"boolean\"}\n",
|
||||
"if delete_bucket:\n",
|
||||
" ! gsutil -m rm -r $BUCKET_NAME"
|
||||
" model.delete()"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
@@ -536,6 +536,10 @@
|
||||
" raise ValueError(\n",
|
||||
" f\"Recommended machine settings not found for: {accelerator_type}. To use another another accelerator, edit this code block to pass in an appropriate `machine_type`, `accelerator_type`, and `accelerator_count` to the deploy_model function by clicking `Show Code` and then modifying the code.\"\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
"# @markdown Set use_dedicated_endpoint to False if you don't want to use [dedicated endpoint](https://cloud.google.com/vertex-ai/docs/general/deployment#create-dedicated-endpoint). Note that [dedicated endpoint does not support VPC Service Controls](https://cloud.google.com/vertex-ai/docs/predictions/choose-endpoint-type), uncheck the box if you are using VPC-SC.\n",
|
||||
"use_dedicated_endpoint = True # @param {type:\"boolean\"}\n",
|
||||
"\n",
|
||||
"common_util.check_quota(\n",
|
||||
" project_id=PROJECT_ID,\n",
|
||||
" region=REGION,\n",
|
||||
@@ -552,11 +556,13 @@
|
||||
" accelerator_type: str = \"NVIDIA_L4\",\n",
|
||||
" accelerator_count: int = 1,\n",
|
||||
" resolution: int = 224,\n",
|
||||
" use_dedicated_endpoint: bool = False,\n",
|
||||
") -> Tuple[aiplatform.Model, aiplatform.Endpoint]:\n",
|
||||
" \"\"\"Create a Vertex AI Endpoint and deploy the specified model to the endpoint.\"\"\"\n",
|
||||
" model_name_with_time = common_util.get_job_name_with_datetime(model_name)\n",
|
||||
" endpoint = aiplatform.Endpoint.create(\n",
|
||||
" display_name=f\"{model_name_with_time}-endpoint\"\n",
|
||||
" display_name=f\"{model_name_with_time}-endpoint\",\n",
|
||||
" dedicated_endpoint_enabled=use_dedicated_endpoint,\n",
|
||||
" )\n",
|
||||
" model = aiplatform.Model.upload(\n",
|
||||
" display_name=model_name_with_time,\n",
|
||||
@@ -598,6 +604,7 @@
|
||||
" accelerator_type=accelerator_type,\n",
|
||||
" accelerator_count=accelerator_count,\n",
|
||||
" resolution=model_resolution,\n",
|
||||
" use_dedicated_endpoint=use_dedicated_endpoint,\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
@@ -626,9 +633,8 @@
|
||||
"\n",
|
||||
"# @markdown <img src=\"https://storage.googleapis.com/longcap100/91.jpeg\" width=\"400\" >\n",
|
||||
"\n",
|
||||
"# fmt: off\n",
|
||||
"image_url = \"https://storage.googleapis.com/longcap100/91.jpeg\" # @param {type:\"string\"}\n",
|
||||
"# fmt: on\n",
|
||||
"\n",
|
||||
"language_code = \"en\" # @param {type: \"string\"}\n",
|
||||
"\n",
|
||||
"image = common_util.download_image(image_url)\n",
|
||||
@@ -638,7 +644,11 @@
|
||||
"image_base64 = common_util.image_to_base64(image)\n",
|
||||
"\n",
|
||||
"caption = common_util.caption_predict(\n",
|
||||
" endpoints[\"endpoint\"], language_code, image, caption_prompt\n",
|
||||
" endpoints[\"endpoint\"],\n",
|
||||
" language_code,\n",
|
||||
" image,\n",
|
||||
" caption_prompt,\n",
|
||||
" use_dedicated_endpoint=use_dedicated_endpoint,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"print(\"Caption: \", caption)\n",
|
||||
|
||||
+1
-1
@@ -427,7 +427,7 @@
|
||||
"id": "L_q9h-SArI0c"
|
||||
},
|
||||
"source": [
|
||||
"You must provide a Hugging Face User Access Token (read) to access the Llama 3.1 models. You can follow the [Hugging Face documentation](https://huggingface.co/docs/hub/en/security-tokens) to create a **read** access token and put it in the `HF_TOKEN` field below."
|
||||
"You must provide a Hugging Face User Access Token (with read access) to access the Llama 3.1 models. You can follow the [Hugging Face documentation](https://huggingface.co/docs/hub/en/security-tokens) to create a **read** access token and put it in the `HF_TOKEN` field below."
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
@@ -59,13 +59,14 @@
|
||||
"source": [
|
||||
"## Overview\n",
|
||||
"\n",
|
||||
"This notebook demonstrates downloading and deploying prebuilt [Llama Guard models](https://huggingface.co/meta-llama) with [vLLM](https://github.com/vllm-project/vllm) on GPU, and demonstrates using the Llama Guard model to safeguard LLM inputs and outputs with the Vertex Llama 3 API service.\n",
|
||||
"This notebook demonstrates downloading and deploying [Llama Guard models](https://huggingface.co/meta-llama) with [vLLM](https://github.com/vllm-project/vllm) on GPU, and demonstrates using the Llama Guard model to safeguard LLM inputs and outputs with the Vertex Llama API service.\n",
|
||||
"\n",
|
||||
"### Objective\n",
|
||||
"\n",
|
||||
"- Download and deploy prebuilt Llama Guard models with [vLLM](https://github.com/vllm-project/vllm) on GPU\n",
|
||||
"- Download and deploy Llama Guard models with [vLLM](https://github.com/vllm-project/vllm) on GPU\n",
|
||||
"- Use the Llama Guard models to safeguard LLM inputs and outputs with the Vertex Llama 3.1 API service\n",
|
||||
"- Use the Llama Guard models to safeguard LLM vision inputs and outputs with the Vertex Llama 3.2 API service\n",
|
||||
"- Use the Llama Guard models to safeguard LLM vision inputs and outputs with the Vertex Llama 4 API service\n",
|
||||
"\n",
|
||||
"### File a bug\n",
|
||||
"\n",
|
||||
@@ -220,14 +221,14 @@
|
||||
"\n",
|
||||
"# @markdown Select one of the three model variations.\n",
|
||||
"\n",
|
||||
"base_model_name = \"Llama-Guard-3-11B-Vision\" # @param [\"Llama-Guard-3-8B\", \"Llama-Guard-3-1B\", \"Llama-Guard-3-11B-Vision\"] {allow-input: true, isTemplate: true}\n",
|
||||
"base_model_name = \"Llama-Guard-4-12B\" # @param [\"Llama-Guard-4-12B\", \"Llama-Guard-3-8B\", \"Llama-Guard-3-1B\", \"Llama-Guard-3-11B-Vision\"] {allow-input: true, isTemplate: true}\n",
|
||||
"model_id = os.path.join(VERTEX_AI_MODEL_GARDEN_LLAMA_GUARD, base_model_name)\n",
|
||||
"hf_model_id = \"meta-llama/\" + base_model_name\n",
|
||||
"version_id = base_model_name.lower()\n",
|
||||
"PUBLISHER_MODEL_NAME = f\"publishers/meta/models/llama-guard@{version_id}\"\n",
|
||||
"\n",
|
||||
"# The pre-built serving docker images.\n",
|
||||
"VLLM_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-vllm-serve:20241202_0916_RC00_maas\"\n",
|
||||
"VLLM_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-vllm-serve:20250429_0916_RC01\"\n",
|
||||
"\n",
|
||||
"# @markdown Set use_dedicated_endpoint to False if you don't want to use [dedicated endpoint](https://cloud.google.com/vertex-ai/docs/general/deployment#create-dedicated-endpoint). Note that [dedicated endpoint does not support VPC Service Controls](https://cloud.google.com/vertex-ai/docs/predictions/choose-endpoint-type), uncheck the box if you are using VPC-SC.\n",
|
||||
"use_dedicated_endpoint = True # @param {type:\"boolean\"}\n",
|
||||
@@ -238,7 +239,7 @@
|
||||
" machine_type = \"g2-standard-12\"\n",
|
||||
" accelerator_count = 1\n",
|
||||
" max_num_seqs = 256\n",
|
||||
"elif \"3-11B\" in base_model_name:\n",
|
||||
"elif \"3-11B\" in base_model_name or \"4-12B\" in base_model_name:\n",
|
||||
" accelerator_type = \"NVIDIA_TESLA_A100\"\n",
|
||||
" machine_type = \"a2-highgpu-1g\"\n",
|
||||
" accelerator_count = 1\n",
|
||||
@@ -290,7 +291,7 @@
|
||||
"source": [
|
||||
"# @title [Option 2] Deploy with customized configurations\n",
|
||||
"\n",
|
||||
"# @markdown This section uploads prebuilt Llama Guard models to Model Registry and deploys it to a Vertex AI Endpoint. It takes 15 minutes to 1 hour to finish depending on the size of the model.\n",
|
||||
"# @markdown This section uploads Llama Guard models to Model Registry and deploys it to a Vertex AI Endpoint. It takes 15 minutes to 1 hour to finish depending on the size of the model.\n",
|
||||
"\n",
|
||||
"gpu_memory_utilization = 0.9\n",
|
||||
"max_model_len = 4096\n",
|
||||
@@ -422,7 +423,7 @@
|
||||
"\n",
|
||||
"\n",
|
||||
"models[LABEL], endpoints[LABEL] = deploy_model_vllm(\n",
|
||||
" model_name=common_util.get_job_name_with_datetime(prefix=\"llama3-guard\"),\n",
|
||||
" model_name=common_util.get_job_name_with_datetime(prefix=\"llama-guard\"),\n",
|
||||
" model_id=model_id,\n",
|
||||
" publisher=\"meta\",\n",
|
||||
" publisher_model_id=\"llama-guard\",\n",
|
||||
@@ -432,10 +433,10 @@
|
||||
" accelerator_count=accelerator_count,\n",
|
||||
" gpu_memory_utilization=gpu_memory_utilization,\n",
|
||||
" max_model_len=max_model_len,\n",
|
||||
" enforce_eager=True,\n",
|
||||
" enforce_eager=False,\n",
|
||||
" use_dedicated_endpoint=use_dedicated_endpoint,\n",
|
||||
" max_num_seqs=max_num_seqs,\n",
|
||||
" enable_llama_tool_parser=True,\n",
|
||||
" enable_llama_tool_parser=False,\n",
|
||||
")\n",
|
||||
"# @markdown Click \"Show Code\" to see more details."
|
||||
]
|
||||
@@ -686,6 +687,131 @@
|
||||
"print(\"Llama Guard prediction:\", prediction)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "h_LgDtVyO13s"
|
||||
},
|
||||
"source": [
|
||||
"## Use the Llama Guard models to safeguard LLM vision inputs and outputs with the Vertex Llama 4 API service\n",
|
||||
"\n",
|
||||
"We use [meta-llama/Llama-Guard-4-12B](https://huggingface.co/meta-llama/Llama-Guard-4-12B) to safeguard input and output conversations with the [Llama 4 model API service on Vertex](https://console.cloud.google.com/vertex-ai/publishers/meta/model-garden/llama-4-maverick-17b-128e-instruct-maas)."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "4hgRrEuqO13s"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"!pip install --upgrade --quiet openai"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "zw5BkBd4O13s"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import google.auth\n",
|
||||
"import openai\n",
|
||||
"\n",
|
||||
"# @markdown Set up the Llama 4 model API service.\n",
|
||||
"\n",
|
||||
"# Programmatically get an access token\n",
|
||||
"creds, _ = google.auth.default(\n",
|
||||
" scopes=[\"https://www.googleapis.com/auth/cloud-platform\"]\n",
|
||||
")\n",
|
||||
"auth_req = google.auth.transport.requests.Request()\n",
|
||||
"creds.refresh(auth_req)\n",
|
||||
"# Note: the credential lives for 1 hour by default (https://cloud.google.com/docs/authentication/token-types#at-lifetime); after expiration, it must be refreshed.\n",
|
||||
"\n",
|
||||
"client = openai.OpenAI(\n",
|
||||
" base_url=f\"https://us-east5-aiplatform.googleapis.com/v1/projects/{PROJECT_ID}/locations/{REGION}/endpoints/openapi\",\n",
|
||||
" api_key=creds.token,\n",
|
||||
")\n",
|
||||
"LLAMA4_MODEL_ID = \"meta/llama-4-scout-17b-16e-instruct-maas\" # @param [\"meta/llama-4-scout-17b-16e-instruct-maas\", \"meta/llama-4-maverick-17b-128e-instruct-maas\"]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "3xX8VqWFO13s"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @markdown Define input message in conversation and get output message from model.\n",
|
||||
"\n",
|
||||
"user_image = \"https://upload.wikimedia.org/wikipedia/commons/thumb/c/cb/The_Blue_Marble_%28remastered%29.jpg/580px-The_Blue_Marble_%28remastered%29.jpg\" # @param {type: \"string\"}\n",
|
||||
"user_message = \"What is in the image?\" # @param {type: \"string\"}\n",
|
||||
"\n",
|
||||
"messages = [\n",
|
||||
" {\n",
|
||||
" \"role\": \"user\",\n",
|
||||
" \"content\": [\n",
|
||||
" {\n",
|
||||
" \"type\": \"image_url\",\n",
|
||||
" \"image_url\": {\"url\": user_image},\n",
|
||||
" },\n",
|
||||
" {\"type\": \"text\", \"text\": user_message},\n",
|
||||
" ],\n",
|
||||
" }\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"print(\"Conversation [turn 1]:\", messages)\n",
|
||||
"\n",
|
||||
"response = client.chat.completions.create(\n",
|
||||
" model=LLAMA4_MODEL_ID,\n",
|
||||
" messages=messages,\n",
|
||||
")\n",
|
||||
"print(\"Response:\", response)\n",
|
||||
"\n",
|
||||
"messages.append(\n",
|
||||
" {\n",
|
||||
" \"role\": response.choices[0].message.role,\n",
|
||||
" \"content\": response.choices[0].message.content,\n",
|
||||
" }\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"print(\"Conversation [turn 2]:\", messages)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "6zhDnfAcO13s"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @markdown Use Llama Guard to classify the conversation: safe versus unsafe.\n",
|
||||
"# @markdown Classification is performed on the last turn of the conversation.\n",
|
||||
"# @markdown If the content is safe, the model will return `safe`. If the content is unsafe, the model will return `unsafe` and additionally the list of offending categories as a comma-separated list in a new line.\n",
|
||||
"# @markdown Set `\"@requestFormat\": \"chatCompletions\"` to use the OpenAI chat completions format.\n",
|
||||
"\n",
|
||||
"instances = [\n",
|
||||
" {\n",
|
||||
" \"messages\": messages,\n",
|
||||
" \"@requestFormat\": \"chatCompletions\",\n",
|
||||
" },\n",
|
||||
"]\n",
|
||||
"response = endpoints[\"vllm_gpu\"].predict(\n",
|
||||
" instances=instances, use_dedicated_endpoint=use_dedicated_endpoint\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"prediction = response.predictions[\"choices\"][0][\"message\"][\"content\"]\n",
|
||||
"print(\"Llama Guard prediction:\", prediction)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
|
||||
@@ -340,9 +340,7 @@
|
||||
"\n",
|
||||
"if use_dedicated_endpoint:\n",
|
||||
" DEDICATED_ENDPOINT_DNS = endpoints[\"ollama\"].gca_resource.dedicated_endpoint_dns\n",
|
||||
"ENDPOINT_RESOURCE_NAME = \"projects/{}/locations/{}/endpoints/{}\".format(\n",
|
||||
" PROJECT_ID, REGION, endpoints[\"ollama\"].name\n",
|
||||
")\n",
|
||||
"ENDPOINT_RESOURCE_NAME = endpoints[\"ollama\"].resource_name\n",
|
||||
"\n",
|
||||
"# @title Chat Completions Inference\n",
|
||||
"\n",
|
||||
|
||||
@@ -674,6 +674,7 @@
|
||||
" disagg_topology: str = None,\n",
|
||||
" hbm_utilization_factor: float = 0.6,\n",
|
||||
" max_running_seqs: int = 256,\n",
|
||||
" decode_seqs_padding: int = None,\n",
|
||||
" max_model_len: int = 4096,\n",
|
||||
" enable_prefix_cache_hbm: bool = False,\n",
|
||||
" endpoint_id: str = \"\",\n",
|
||||
@@ -714,6 +715,10 @@
|
||||
" f\"--max_running_seqs={max_running_seqs}\",\n",
|
||||
" f\"--max_model_len={max_model_len}\",\n",
|
||||
" ]\n",
|
||||
"\n",
|
||||
" if decode_seqs_padding is not None:\n",
|
||||
" hexllm_args.append(f\"--decode_seqs_padding={decode_seqs_padding}\")\n",
|
||||
"\n",
|
||||
" if disagg_topology:\n",
|
||||
" hexllm_args.append(f\"--disagg_topo={disagg_topology}\")\n",
|
||||
" if enable_prefix_cache_hbm and not disagg_topology:\n",
|
||||
|
||||
@@ -218,15 +218,17 @@
|
||||
"# @markdown | Model Version | Default Max Model Length | Available GPU configurations |\n",
|
||||
"# @markdown |----------------------------|------------------|-----------------------------|\n",
|
||||
"# @markdown | Phi-4 | 16384 | 1 NVIDIA_A100 80GB a2-ultragpu-1g, 2 NVIDIA_L4 g2-standard-24 |\n",
|
||||
"# @markdown | Phi-4-reasoning | 32768 | 1 NVIDIA_A100 80GB a2-ultragpu-1g, 1 NVIDIA_H100 80GB a3-highgpu-1g |\n",
|
||||
"# @markdown | Phi-4-reasoning-plus | 32768 | 1 NVIDIA_A100 80GB a2-ultragpu-1g, 1 NVIDIA_H100 80GB a3-highgpu-1g |\n",
|
||||
"\n",
|
||||
"# The pre-built serving docker images.\n",
|
||||
"VLLM_DOCKER_URI = \"us-docker.pkg.dev/deeplearning-platform-release/vertex-model-garden/vllm-inference.cu121.0-6.ubuntu2204.py310\"\n",
|
||||
"VLLM_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-vllm-serve:20250417_0916_RC01\"\n",
|
||||
"\n",
|
||||
"MODEL_ID = \"Phi-4\"\n",
|
||||
"MODEL_ID = \"Phi-4\" # @param [\"Phi-4\", \"Phi-4-reasoning\", \"Phi-4-reasoning-plus\"] {isTemplate:true}\n",
|
||||
"model_path_prefix = \"microsoft\"\n",
|
||||
"model_id = os.path.join(model_path_prefix, MODEL_ID)\n",
|
||||
"\n",
|
||||
"accelerator_type = \"NVIDIA_L4\" # @param [\"NVIDIA_L4\", \"NVIDIA_A100_80GB\"] {isTemplate: true}\n",
|
||||
"accelerator_type = \"NVIDIA_L4\" # @param [\"NVIDIA_L4\", \"NVIDIA_A100_80GB\", \"NVIDIA_H100_80GB\"] {isTemplate: true}\n",
|
||||
"machine_type = None\n",
|
||||
"vllm_dtype = \"bfloat16\"\n",
|
||||
"accelerator_count = None\n",
|
||||
@@ -234,7 +236,22 @@
|
||||
"gpu_memory_utilization = None\n",
|
||||
"enable_trust_remote_code = False\n",
|
||||
"\n",
|
||||
"if \"Phi-4\" == MODEL_ID:\n",
|
||||
"if \"Phi-4-reasoning\" in MODEL_ID:\n",
|
||||
" max_model_len = 32768\n",
|
||||
" if accelerator_type == \"NVIDIA_A100_80GB\":\n",
|
||||
" accelerator_count = 1\n",
|
||||
" machine_type = \"a2-ultragpu-1g\"\n",
|
||||
" gpu_memory_utilization = 0.85\n",
|
||||
" elif accelerator_type == \"NVIDIA_H100_80GB\":\n",
|
||||
" accelerator_count = 1\n",
|
||||
" machine_type = \"a3-highgpu-1g\"\n",
|
||||
" gpu_memory_utilization = 0.85\n",
|
||||
" else:\n",
|
||||
" raise ValueError(\n",
|
||||
" \"Recommended machine settings not found for accelerator type: %s\"\n",
|
||||
" % accelerator_type\n",
|
||||
" )\n",
|
||||
"elif \"Phi-4\" == MODEL_ID:\n",
|
||||
" max_model_len = 16384\n",
|
||||
" if accelerator_type == \"NVIDIA_L4\":\n",
|
||||
" accelerator_count = 2\n",
|
||||
@@ -499,13 +516,13 @@
|
||||
"# @markdown This section uploads prebuilt Phi-4 models to Model Registry and deploys it to a Vertex AI Endpoint. It takes 15 minutes to 1 hour to finish depending on the size of the model.\n",
|
||||
"\n",
|
||||
"# @markdown Select one of the four model variations.\n",
|
||||
"MODEL_ID = \"Phi-4\"\n",
|
||||
"MODEL_ID = \"Phi-4\" # @param [\"Phi-4\", \"Phi-4-reasoning\", \"Phi-4-reasoning-plus\"] {isTemplate:true}\n",
|
||||
"TPU_DEPLOYMENT_REGION = \"us-west1\" # @param [\"us-west1\"] {isTemplate:true}\n",
|
||||
"model_path_prefix = \"microsoft\"\n",
|
||||
"model_id = os.path.join(model_path_prefix, MODEL_ID)\n",
|
||||
"\n",
|
||||
"# The pre-built serving docker images.\n",
|
||||
"HEXLLM_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai-restricted/vertex-vision-model-garden-dockers/hex-llm-serve:20241210_2323_RC00\"\n",
|
||||
"HEXLLM_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai-restricted/vertex-vision-model-garden-dockers/hex-llm-serve:phi4\"\n",
|
||||
"\n",
|
||||
"# @markdown Find Vertex AI prediction TPUv5e machine types in\n",
|
||||
"# @markdown https://cloud.google.com/vertex-ai/docs/predictions/use-tpu#deploy_a_model.\n",
|
||||
@@ -513,14 +530,21 @@
|
||||
"# @markdown | Model Version | Default Max Model Length | Default TPU configuration |\n",
|
||||
"# @markdown |----------------------------|------------------|-----------------------------|\n",
|
||||
"# @markdown | Phi-4 | 16384 | 4 TPU_V5e ct5lp-hightpu-4t |\n",
|
||||
"# @markdown | Phi-4-reasoning | 32768 | 4 TPU_V5e ct5lp-hightpu-4t |\n",
|
||||
"# @markdown | Phi-4-reasoning-plus | 32768 | 4 TPU_V5e ct5lp-hightpu-4t |\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Note: 1 TPU V5 chip has only one core.\n",
|
||||
"tpu_type = \"TPU_V5e\"\n",
|
||||
"\n",
|
||||
"if \"Phi-4\" in MODEL_ID:\n",
|
||||
"if \"Phi-4-reasoning\" in MODEL_ID:\n",
|
||||
" tpu_count = 4\n",
|
||||
" tpu_topo = \"4x4\"\n",
|
||||
" tpu_topo = \"1x4\"\n",
|
||||
" max_model_len = 32768\n",
|
||||
" machine_type = \"ct5lp-hightpu-4t\"\n",
|
||||
"elif \"Phi-4\" in MODEL_ID:\n",
|
||||
" tpu_count = 4\n",
|
||||
" tpu_topo = \"1x4\"\n",
|
||||
" max_model_len = 16384\n",
|
||||
" machine_type = \"ct5lp-hightpu-4t\"\n",
|
||||
"else:\n",
|
||||
@@ -561,6 +585,7 @@
|
||||
" disagg_topology: str = None,\n",
|
||||
" hbm_utilization_factor: float = 0.6,\n",
|
||||
" max_running_seqs: int = 256,\n",
|
||||
" decode_seqs_padding: int = None,\n",
|
||||
" max_model_len: int = 4096,\n",
|
||||
" enable_prefix_cache_hbm: bool = False,\n",
|
||||
" endpoint_id: str = \"\",\n",
|
||||
@@ -601,6 +626,10 @@
|
||||
" f\"--max_running_seqs={max_running_seqs}\",\n",
|
||||
" f\"--max_model_len={max_model_len}\",\n",
|
||||
" ]\n",
|
||||
"\n",
|
||||
" if decode_seqs_padding is not None:\n",
|
||||
" hexllm_args.append(f\"--decode_seqs_padding={decode_seqs_padding}\")\n",
|
||||
"\n",
|
||||
" if disagg_topology:\n",
|
||||
" hexllm_args.append(f\"--disagg_topo={disagg_topology}\")\n",
|
||||
" if enable_prefix_cache_hbm and not disagg_topology:\n",
|
||||
|
||||
+156
-207
@@ -4,11 +4,12 @@
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "7d9bbf86da5e"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Copyright 2023 Google LLC\n",
|
||||
"# Copyright 2025 Google LLC\n",
|
||||
"#\n",
|
||||
"# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
|
||||
"# you may not use this file except in compliance with the License.\n",
|
||||
@@ -31,26 +32,23 @@
|
||||
"source": [
|
||||
"# Vertex AI Model Garden - BLIP Image Captioning\n",
|
||||
"\n",
|
||||
"<table align=\"left\">\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_blip_image_captioning.ipynb\">\n",
|
||||
" <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
|
||||
"<table><tbody><tr>\n",
|
||||
" <td style=\"text-align: center\">\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/workbench/instances\">\n",
|
||||
" <img alt=\"Workbench logo\" src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" width=\"32px\"><br> Run in Workbench\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <td style=\"text-align: center\">\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/colab/import/https:%2F%2Fraw.githubusercontent.com%2FGoogleCloudPlatform%2Fvertex-ai-samples%2Fmain%2Fnotebooks%2Fcommunity%2Fmodel_garden%2Fmodel_garden_pytorch_blip_image_captioning.ipynb\">\n",
|
||||
" <img alt=\"Google Cloud Colab Enterprise logo\" src=\"https://lh3.googleusercontent.com/JmcxdQi-qOpctIvWKgPtrzZdJJK-J3sWE1RsfjZNwshCFgE_9fULcNpuXYTilIR2hjwN\" width=\"32px\"><br> Run in Colab Enterprise\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td style=\"text-align: center\">\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_blip_image_captioning.ipynb\">\n",
|
||||
" <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" <img alt=\"GitHub logo\" src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" width=\"32px\"><br> View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/notebooks/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/model_garden/model_garden_pytorch_blip_image_captioning.ipynb\">\n",
|
||||
" <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
|
||||
"Open in Vertex AI Workbench\n",
|
||||
" </a>\n",
|
||||
" (a Python-3 CPU notebook is recommended)\n",
|
||||
" </td>\n",
|
||||
"</table>"
|
||||
"</tr></tbody></table>"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -69,6 +67,10 @@
|
||||
"- Deploy the model on [Endpoint](https://cloud.google.com/vertex-ai/docs/predictions/using-private-endpoints).\n",
|
||||
"- Run online predictions for image captioning.\n",
|
||||
"\n",
|
||||
"### File a bug\n",
|
||||
"\n",
|
||||
"File a bug on [GitHub](https://github.com/GoogleCloudPlatform/vertex-ai-samples/issues/new) if you encounter any issue with the notebook.\n",
|
||||
"\n",
|
||||
"### Costs\n",
|
||||
"\n",
|
||||
"This tutorial uses billable components of Google Cloud:\n",
|
||||
@@ -76,199 +78,155 @@
|
||||
"* Vertex AI\n",
|
||||
"* Cloud Storage\n",
|
||||
"\n",
|
||||
"Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage pricing](https://cloud.google.com/storage/pricing), and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage."
|
||||
"Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing), [Cloud Storage pricing](https://cloud.google.com/storage/pricing), and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "264c07757582"
|
||||
"id": "G3GYgynjaKG8"
|
||||
},
|
||||
"source": [
|
||||
"## Setup environment\n",
|
||||
"\n",
|
||||
"**NOTE**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "d73ffa0c0b83"
|
||||
},
|
||||
"source": [
|
||||
"### Colab only"
|
||||
"## Before you begin"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "2707b02ef5df"
|
||||
"cellView": "form",
|
||||
"id": "cDtpu7ZVaTFb"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"if \"google.colab\" in str(get_ipython()):\n",
|
||||
" ! pip3 install --upgrade google-cloud-aiplatform\n",
|
||||
" from google.colab import auth as google_auth\n",
|
||||
"# @title Setup Google Cloud project\n",
|
||||
"\n",
|
||||
" google_auth.authenticate_user()\n",
|
||||
"# @markdown 1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
|
||||
"\n",
|
||||
" # Restart the notebook kernel after installs.\n",
|
||||
" import IPython\n",
|
||||
"# @markdown 2. **[Optional]** Set region. If not set, the region will be set automatically according to Colab Enterprise environment.\n",
|
||||
"\n",
|
||||
" app = IPython.Application.instance()\n",
|
||||
" app.kernel.do_shutdown(True)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "0f826ff482a2"
|
||||
},
|
||||
"source": [
|
||||
"### Setup Google Cloud project\n",
|
||||
"REGION = \"\" # @param {type:\"string\"}\n",
|
||||
"\n",
|
||||
"1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
|
||||
"# @markdown 3. If you want to run predictions with A100 80GB or H100 GPUs, we recommend using the regions listed below. **NOTE:** Make sure you have associated quota in selected regions. Click the links to see your current quota for each GPU type: [Nvidia A100 80GB](https://console.cloud.google.com/iam-admin/quotas?metric=aiplatform.googleapis.com%2Fcustom_model_serving_nvidia_a100_80gb_gpus), [Nvidia H100 80GB](https://console.cloud.google.com/iam-admin/quotas?metric=aiplatform.googleapis.com%2Fcustom_model_serving_nvidia_h100_gpus). You can request for quota following the instructions at [\"Request a higher quota\"](https://cloud.google.com/docs/quota/view-manage#requesting_higher_quota).\n",
|
||||
"\n",
|
||||
"1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
|
||||
"# @markdown > | Machine Type | Accelerator Type | Recommended Regions |\n",
|
||||
"# @markdown | ----------- | ----------- | ----------- |\n",
|
||||
"# @markdown | a2-ultragpu-1g | 1 NVIDIA_A100_80GB | us-central1, us-east4, europe-west4, asia-southeast1, us-east4 |\n",
|
||||
"# @markdown | a3-highgpu-2g | 2 NVIDIA_H100_80GB | us-west1, asia-southeast1, europe-west4 |\n",
|
||||
"# @markdown | a3-highgpu-4g | 4 NVIDIA_H100_80GB | us-west1, asia-southeast1, europe-west4 |\n",
|
||||
"# @markdown | a3-highgpu-8g | 8 NVIDIA_H100_80GB | us-central1, europe-west4, us-west1, asia-southeast1 |\n",
|
||||
"\n",
|
||||
"1. [Enable the Vertex AI API and Compute Engine API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component).\n",
|
||||
"# Import the necessary packages\n",
|
||||
"\n",
|
||||
"1. [Create a Cloud Storage bucket](https://cloud.google.com/storage/docs/creating-buckets) for storing experiment outputs."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "8958ebc71868"
|
||||
},
|
||||
"source": [
|
||||
"Fill following variables for experiments environment:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "9db30f827a65"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Cloud project id.\n",
|
||||
"PROJECT_ID = \"\" # @param {type:\"string\"}\n",
|
||||
"# Upgrade Vertex AI SDK.\n",
|
||||
"! pip3 install --upgrade --quiet 'google-cloud-aiplatform>=1.84.0'\n",
|
||||
"\n",
|
||||
"# The region you want to launch jobs in.\n",
|
||||
"REGION = \"us-central1\" # @param {type:\"string\"}\n",
|
||||
"# Import the necessary packages.\n",
|
||||
"import importlib\n",
|
||||
"import os\n",
|
||||
"from typing import Tuple\n",
|
||||
"\n",
|
||||
"# The Cloud Storage bucket for storing experiments output. Fill it without the 'gs://' prefix.\n",
|
||||
"GCS_BUCKET = \"\" # @param {type:\"string\"}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "92f16e22c20b"
|
||||
},
|
||||
"source": [
|
||||
"Initialize Vertex AI API:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "1680c257acfb"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from google.cloud import aiplatform\n",
|
||||
"\n",
|
||||
"aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=GCS_BUCKET)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "6ca48b699d17"
|
||||
},
|
||||
"source": [
|
||||
"### Define constants"
|
||||
"if os.environ.get(\"VERTEX_PRODUCT\") != \"COLAB_ENTERPRISE\":\n",
|
||||
" ! pip install --upgrade tensorflow\n",
|
||||
"! git clone https://github.com/GoogleCloudPlatform/vertex-ai-samples.git\n",
|
||||
"\n",
|
||||
"common_util = importlib.import_module(\n",
|
||||
" \"vertex-ai-samples.community-content.vertex_model_garden.model_oss.notebook_util.common_util\"\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"LABEL = \"transformers_gpu\"\n",
|
||||
"models, endpoints = {}, {}\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Get the default cloud project id.\n",
|
||||
"PROJECT_ID = os.environ[\"GOOGLE_CLOUD_PROJECT\"]\n",
|
||||
"\n",
|
||||
"# Get the default region for launching jobs.\n",
|
||||
"if not REGION:\n",
|
||||
" REGION = os.environ[\"GOOGLE_CLOUD_REGION\"]\n",
|
||||
"\n",
|
||||
"# Initialize Vertex AI API.\n",
|
||||
"print(\"Initializing Vertex AI API.\")\n",
|
||||
"aiplatform.init(project=PROJECT_ID, location=REGION)\n",
|
||||
"\n",
|
||||
"! gcloud config set project $PROJECT_ID\n",
|
||||
"import vertexai\n",
|
||||
"\n",
|
||||
"vertexai.init(\n",
|
||||
" project=PROJECT_ID,\n",
|
||||
" location=REGION,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# @markdown Click \"Show Code\" to see more details."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "de9882ea89ea"
|
||||
"cellView": "form",
|
||||
"id": "RYfNg5r8ezx1"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# The pre-built serving docker image. It contains serving scripts and models.\n",
|
||||
"SERVE_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-transformers-serve\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "10188266a5cd"
|
||||
},
|
||||
"source": [
|
||||
"### Define common functions"
|
||||
"# @title Select the model parameters\n",
|
||||
"\n",
|
||||
"MODEL_ID = \"Salesforce/blip-image-captioning-base\"\n",
|
||||
"TASK = \"image-to-text\"\n",
|
||||
"\n",
|
||||
"# Set the machine specifications.\n",
|
||||
"accelerator_type = \"NVIDIA_L4\" # @param [\"NVIDIA_L4\", \"NVIDIA_TESLA_V100\", \"NVIDIA_TESLA_T4\"]\n",
|
||||
"if accelerator_type == \"NVIDIA_L4\":\n",
|
||||
" accelerator_count = 1\n",
|
||||
" machine_type = \"g2-standard-8\"\n",
|
||||
"elif accelerator_type == \"NVIDIA_TESLA_V100\":\n",
|
||||
" accelerator_count = 1\n",
|
||||
" machine_type = \"n1-standard-4\"\n",
|
||||
"elif accelerator_type == \"NVIDIA_TESLA_T4\":\n",
|
||||
" accelerator_count = 1\n",
|
||||
" machine_type = \"n1-standard-8\"\n",
|
||||
"else:\n",
|
||||
" raise ValueError(\n",
|
||||
" f\"Recommended machine settings not found for: {accelerator_type}. To use another accelerator type, edit this code block to pass in an appropriate `training_machine_type`, `training_accelerator_type`, and `per_node_accelerator_count` by clicking `Show Code` and then modifying the code.\"\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
"# @markdown Set use_dedicated_endpoint to False if you don't want to use [dedicated endpoint](https://cloud.google.com/vertex-ai/docs/general/deployment#create-dedicated-endpoint). Note that [dedicated endpoint does not support VPC Service Controls](https://cloud.google.com/vertex-ai/docs/predictions/choose-endpoint-type), uncheck the box if you are using VPC-SC.\n",
|
||||
"use_dedicated_endpoint = True # @param {type:\"boolean\"}\n",
|
||||
"\n",
|
||||
"# @markdown Click \"Show Code\" to see more details."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "cac4478ae098"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import base64\n",
|
||||
"import os\n",
|
||||
"from datetime import datetime\n",
|
||||
"from io import BytesIO\n",
|
||||
"# @title Deploy with customized configs\n",
|
||||
"\n",
|
||||
"import requests\n",
|
||||
"from google.cloud import aiplatform\n",
|
||||
"from PIL import Image\n",
|
||||
"# The pre-built serving docker image. It contains serving scripts and models.\n",
|
||||
"SERVE_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-transformers-serve\"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def create_job_name(prefix):\n",
|
||||
" user = os.environ.get(\"USER\")\n",
|
||||
" now = datetime.now().strftime(\"%Y%m%d_%H%M%S\")\n",
|
||||
" job_name = f\"{prefix}-{user}-{now}\"\n",
|
||||
" return job_name\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def download_image(url):\n",
|
||||
" response = requests.get(url)\n",
|
||||
" return Image.open(BytesIO(response.content))\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def image_to_base64(image, format=\"JPEG\"):\n",
|
||||
" buffer = BytesIO()\n",
|
||||
" image.save(buffer, format=format)\n",
|
||||
" image_str = base64.b64encode(buffer.getvalue()).decode(\"utf-8\")\n",
|
||||
" return image_str\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def base64_to_image(image_str):\n",
|
||||
" image = Image.open(BytesIO(base64.b64decode(image_str)))\n",
|
||||
" return image\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def image_grid(imgs, rows=2, cols=2):\n",
|
||||
" w, h = imgs[0].size\n",
|
||||
" grid = Image.new(\"RGB\", size=(cols * w, rows * h))\n",
|
||||
" for i, img in enumerate(imgs):\n",
|
||||
" grid.paste(img, box=(i % cols * w, i // cols * h))\n",
|
||||
" return grid\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def deploy_model(model_id, task):\n",
|
||||
"def deploy_model(\n",
|
||||
" model_name: str,\n",
|
||||
" model_id: str,\n",
|
||||
" task: str,\n",
|
||||
" machine_type: str = \"g2-standard-8\",\n",
|
||||
" accelerator_type: str = \"NVIDIA_L4\",\n",
|
||||
" accelerator_count: int = 1,\n",
|
||||
" use_dedicated_endpoint: bool = True,\n",
|
||||
") -> Tuple[aiplatform.Model, aiplatform.Endpoint]:\n",
|
||||
" model_name = \"blip-image-captioning\"\n",
|
||||
" endpoint = aiplatform.Endpoint.create(display_name=f\"{model_name}-endpoint\")\n",
|
||||
" endpoint = aiplatform.Endpoint.create(\n",
|
||||
" display_name=f\"{model_name}-endpoint\",\n",
|
||||
" dedicated_endpoint_enabled=use_dedicated_endpoint,\n",
|
||||
" )\n",
|
||||
" serving_env = {\n",
|
||||
" \"MODEL_ID\": model_id,\n",
|
||||
" \"TASK\": task,\n",
|
||||
@@ -284,7 +242,7 @@
|
||||
" serving_container_health_route=\"/ping\",\n",
|
||||
" serving_container_environment_variables=serving_env,\n",
|
||||
" artifact_uri=artifact_uri,\n",
|
||||
" model_garden_source_model_name=\"publishers/salesforce/models/blip-image-captioning-base\"\n",
|
||||
" model_garden_source_model_name=\"publishers/salesforce/models/blip-image-captioning-base\",\n",
|
||||
" )\n",
|
||||
" model.deploy(\n",
|
||||
" endpoint=endpoint,\n",
|
||||
@@ -296,41 +254,25 @@
|
||||
" \"NOTEBOOK_NAME\": \"model_garden_pytorch_blip_image_captioning.ipynb\"\n",
|
||||
" },\n",
|
||||
" )\n",
|
||||
" return model, endpoint"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "d2d72ecdb8c9"
|
||||
},
|
||||
"source": [
|
||||
"## Upload and deploy models"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "9448c5f545fa"
|
||||
},
|
||||
"source": [
|
||||
"This section uploads the pre-trained model to Model Registry and deploys it on the Endpoint with 1 T4 GPU.\n",
|
||||
" return model, endpoint\n",
|
||||
"\n",
|
||||
"The model deployment step will take ~15 minutes to complete.\n",
|
||||
"\n",
|
||||
"Once deployed, you can send images to get descriptions."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "b4b46c28d8b1"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"model, endpoint = deploy_model(\n",
|
||||
" model_id=\"Salesforce/blip-image-captioning-base\", task=\"image-to-text\"\n",
|
||||
"common_util.check_quota(\n",
|
||||
" project_id=PROJECT_ID,\n",
|
||||
" region=REGION,\n",
|
||||
" accelerator_type=accelerator_type,\n",
|
||||
" accelerator_count=accelerator_count,\n",
|
||||
" is_for_training=False,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"models[LABEL], endpoints[LABEL] = deploy_model(\n",
|
||||
" model_name=common_util.get_job_name_with_datetime(prefix=\"blip-image-captioning\"),\n",
|
||||
" model_id=MODEL_ID,\n",
|
||||
" task=TASK,\n",
|
||||
" machine_type=machine_type,\n",
|
||||
" accelerator_type=accelerator_type,\n",
|
||||
" accelerator_count=accelerator_count,\n",
|
||||
" use_dedicated_endpoint=use_dedicated_endpoint,\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
@@ -347,42 +289,49 @@
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "6be655247cb1"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"image = download_image(\"http://images.cocodataset.org/val2017/000000039769.jpg\")\n",
|
||||
"# @title Predict\n",
|
||||
"\n",
|
||||
"INPUT_IMAGE = \"http://images.cocodataset.org/val2017/000000039769.jpg\" # @param {type: \"string\"}\n",
|
||||
"image = common_util.download_image(INPUT_IMAGE)\n",
|
||||
"display(image)\n",
|
||||
"\n",
|
||||
"instances = [\n",
|
||||
" {\"image\": image_to_base64(image)},\n",
|
||||
" {\"image\": common_util.image_to_base64(image)},\n",
|
||||
"]\n",
|
||||
"preds = endpoint.predict(instances=instances).predictions\n",
|
||||
"\n",
|
||||
"preds = (\n",
|
||||
" endpoints[LABEL]\n",
|
||||
" .predict(instances=instances, use_dedicated_endpoint=use_dedicated_endpoint)\n",
|
||||
" .predictions\n",
|
||||
")\n",
|
||||
"print(preds)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "db7ffebdb4be"
|
||||
},
|
||||
"source": [
|
||||
"### Clean up resources"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "2ccf3714dbe9"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Clean up resources\n",
|
||||
"# @markdown Delete the experiment models and endpoints to recycle the resources\n",
|
||||
"# @markdown and avoid unnecessary continuous charges that may incur.\n",
|
||||
"\n",
|
||||
"# Undeploy model and delete endpoint.\n",
|
||||
"endpoint.delete(force=True)\n",
|
||||
"for endpoint in endpoints.values():\n",
|
||||
" endpoint.delete(force=True)\n",
|
||||
"\n",
|
||||
"# Delete models.\n",
|
||||
"model.delete()"
|
||||
"for model in models.values():\n",
|
||||
" model.delete()"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
@@ -4,11 +4,12 @@
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "7d9bbf86da5e"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Copyright 2023 Google LLC\n",
|
||||
"# Copyright 2025 Google LLC\n",
|
||||
"#\n",
|
||||
"# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
|
||||
"# you may not use this file except in compliance with the License.\n",
|
||||
@@ -31,26 +32,23 @@
|
||||
"source": [
|
||||
"# Vertex AI Model Garden - CLIP\n",
|
||||
"\n",
|
||||
"<table align=\"left\">\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_clip.ipynb\">\n",
|
||||
" <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
|
||||
"<table><tbody><tr>\n",
|
||||
" <td style=\"text-align: center\">\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/workbench/instances\">\n",
|
||||
" <img alt=\"Workbench logo\" src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" width=\"32px\"><br> Run in Workbench\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <td style=\"text-align: center\">\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/colab/import/https:%2F%2Fraw.githubusercontent.com%2FGoogleCloudPlatform%2Fvertex-ai-samples%2Fmain%2Fnotebooks%2Fcommunity%2Fmodel_garden%2Fmodel_garden_pytorch_clip.ipynb\">\n",
|
||||
" <img alt=\"Google Cloud Colab Enterprise logo\" src=\"https://lh3.googleusercontent.com/JmcxdQi-qOpctIvWKgPtrzZdJJK-J3sWE1RsfjZNwshCFgE_9fULcNpuXYTilIR2hjwN\" width=\"32px\"><br> Run in Colab Enterprise\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td style=\"text-align: center\">\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_clip.ipynb\">\n",
|
||||
" <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" <img alt=\"GitHub logo\" src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" width=\"32px\"><br> View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/notebooks/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/model_garden/model_garden_pytorch_clip.ipynb\">\n",
|
||||
" <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
|
||||
"Open in Vertex AI Workbench\n",
|
||||
" </a>\n",
|
||||
" (a Python-3 CPU notebook is recommended)\n",
|
||||
" </td>\n",
|
||||
"</table>"
|
||||
"</tr></tbody></table>"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -69,6 +67,10 @@
|
||||
"- Deploy the model on [Endpoint](https://cloud.google.com/vertex-ai/docs/predictions/using-private-endpoints).\n",
|
||||
"- Run online predictions for image captioning.\n",
|
||||
"\n",
|
||||
"### File a bug\n",
|
||||
"\n",
|
||||
"File a bug on [GitHub](https://github.com/GoogleCloudPlatform/vertex-ai-samples/issues/new) if you encounter any issue with the notebook.\n",
|
||||
"\n",
|
||||
"### Costs\n",
|
||||
"\n",
|
||||
"This tutorial uses billable components of Google Cloud:\n",
|
||||
@@ -76,194 +78,79 @@
|
||||
"* Vertex AI\n",
|
||||
"* Cloud Storage\n",
|
||||
"\n",
|
||||
"Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage pricing](https://cloud.google.com/storage/pricing), and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "264c07757582"
|
||||
},
|
||||
"source": [
|
||||
"## Setup environment\n",
|
||||
"\n",
|
||||
"**NOTE**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "d73ffa0c0b83"
|
||||
},
|
||||
"source": [
|
||||
"### Colab only"
|
||||
"Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing), [Cloud Storage pricing](https://cloud.google.com/storage/pricing), and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "2707b02ef5df"
|
||||
"cellView": "form",
|
||||
"id": "_53r3GqThdEP"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"if \"google.colab\" in str(get_ipython()):\n",
|
||||
" ! pip3 install --upgrade google-cloud-aiplatform\n",
|
||||
" from google.colab import auth as google_auth\n",
|
||||
"# @title Setup Google Cloud project\n",
|
||||
"\n",
|
||||
" google_auth.authenticate_user()\n",
|
||||
"# Import the necessary packages\n",
|
||||
"\n",
|
||||
" # Restart the notebook kernel after installs.\n",
|
||||
" import IPython\n",
|
||||
"# Upgrade Vertex AI SDK.\n",
|
||||
"! pip3 install --upgrade --quiet 'google-cloud-aiplatform>=1.84.0'\n",
|
||||
"\n",
|
||||
" app = IPython.Application.instance()\n",
|
||||
" app.kernel.do_shutdown(True)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "0f826ff482a2"
|
||||
},
|
||||
"source": [
|
||||
"### Setup Google Cloud project\n",
|
||||
"# @markdown 1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
|
||||
"\n",
|
||||
"1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
|
||||
"# @markdown 2. **[Optional]** [Create a Cloud Storage bucket](https://cloud.google.com/storage/docs/creating-buckets) for storing experiment outputs. Set the BUCKET_URI for the experiment environment. The specified Cloud Storage bucket (`BUCKET_URI`) should be located in the same region as where the notebook was launched. Note that a multi-region bucket (eg. \"us\") is not considered a match for a single region covered by the multi-region range (eg. \"us-central1\"). If not set, a unique GCS bucket will be created instead.\n",
|
||||
"\n",
|
||||
"1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
|
||||
"BUCKET_URI = \"gs://\" # @param {type:\"string\"}\n",
|
||||
"\n",
|
||||
"1. [Enable the Vertex AI API and Compute Engine API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component).\n",
|
||||
"# @markdown 3. **[Optional]** Set region. If not set, the region will be set automatically according to Colab Enterprise environment.\n",
|
||||
"\n",
|
||||
"1. [Create a Cloud Storage bucket](https://cloud.google.com/storage/docs/creating-buckets) for storing experiment outputs."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "8958ebc71868"
|
||||
},
|
||||
"source": [
|
||||
"Fill following variables for experiments environment:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "9db30f827a65"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Cloud project id.\n",
|
||||
"PROJECT_ID = \"\" # @param {type:\"string\"}\n",
|
||||
"REGION = \"\" # @param {type:\"string\"}\n",
|
||||
"\n",
|
||||
"# The region you want to launch jobs in.\n",
|
||||
"REGION = \"us-central1\" # @param {type:\"string\"}\n",
|
||||
"\n",
|
||||
"# The Cloud Storage bucket for storing experiments output. Fill it without the 'gs://' prefix.\n",
|
||||
"GCS_BUCKET = \"\" # @param {type:\"string\"}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "92f16e22c20b"
|
||||
},
|
||||
"source": [
|
||||
"Initialize Vertex AI API:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "1680c257acfb"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from google.cloud import aiplatform\n",
|
||||
"\n",
|
||||
"aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=GCS_BUCKET)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "6ca48b699d17"
|
||||
},
|
||||
"source": [
|
||||
"### Define constants"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "de9882ea89ea"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# The pre-built serving docker image. It contains serving scripts and models.\n",
|
||||
"SERVE_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-transformers-serve\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "10188266a5cd"
|
||||
},
|
||||
"source": [
|
||||
"### Define common functions"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "cac4478ae098"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import base64\n",
|
||||
"import importlib\n",
|
||||
"import os\n",
|
||||
"from datetime import datetime\n",
|
||||
"from io import BytesIO\n",
|
||||
"\n",
|
||||
"import requests\n",
|
||||
"from google.cloud import aiplatform\n",
|
||||
"from PIL import Image\n",
|
||||
"\n",
|
||||
"if os.environ.get(\"VERTEX_PRODUCT\") != \"COLAB_ENTERPRISE\":\n",
|
||||
" ! pip install --upgrade tensorflow\n",
|
||||
"! git clone https://github.com/GoogleCloudPlatform/vertex-ai-samples.git\n",
|
||||
"\n",
|
||||
"common_util = importlib.import_module(\n",
|
||||
" \"vertex-ai-samples.community-content.vertex_model_garden.model_oss.notebook_util.common_util\"\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def create_job_name(prefix):\n",
|
||||
" user = os.environ.get(\"USER\")\n",
|
||||
" now = datetime.now().strftime(\"%Y%m%d_%H%M%S\")\n",
|
||||
" job_name = f\"{prefix}-{user}-{now}\"\n",
|
||||
" return job_name\n",
|
||||
"# Get the default cloud project id.\n",
|
||||
"PROJECT_ID = os.environ[\"GOOGLE_CLOUD_PROJECT\"]\n",
|
||||
"\n",
|
||||
"# Get the default region for launching jobs.\n",
|
||||
"if not REGION:\n",
|
||||
" REGION = os.environ[\"GOOGLE_CLOUD_REGION\"]\n",
|
||||
"\n",
|
||||
"def download_image(url):\n",
|
||||
" response = requests.get(url)\n",
|
||||
" return Image.open(BytesIO(response.content))\n",
|
||||
"# Initialize Vertex AI API.\n",
|
||||
"print(\"Initializing Vertex AI API.\")\n",
|
||||
"aiplatform.init(project=PROJECT_ID, location=REGION)\n",
|
||||
"\n",
|
||||
"! gcloud config set project $PROJECT_ID\n",
|
||||
"\n",
|
||||
"def image_to_base64(image, format=\"JPEG\"):\n",
|
||||
" buffer = BytesIO()\n",
|
||||
" image.save(buffer, format=format)\n",
|
||||
" image_str = base64.b64encode(buffer.getvalue()).decode(\"utf-8\")\n",
|
||||
" return image_str\n",
|
||||
"aiplatform.init(project=PROJECT_ID, location=REGION)\n",
|
||||
"\n",
|
||||
"# The pre-built serving docker image. It contains serving scripts and models.\n",
|
||||
"SERVE_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-transformers-serve\"\n",
|
||||
"\n",
|
||||
"def base64_to_image(image_str):\n",
|
||||
" image = Image.open(BytesIO(base64.b64decode(image_str)))\n",
|
||||
" return image\n",
|
||||
"models, endpoints = {}, {}\n",
|
||||
"LABEL = \"endpoint\"\n",
|
||||
"\n",
|
||||
"import vertexai\n",
|
||||
"\n",
|
||||
"def image_grid(imgs, rows=2, cols=2):\n",
|
||||
" w, h = imgs[0].size\n",
|
||||
" grid = Image.new(\"RGB\", size=(cols * w, rows * h))\n",
|
||||
" for i, img in enumerate(imgs):\n",
|
||||
" grid.paste(img, box=(i % cols * w, i // cols * h))\n",
|
||||
" return grid\n",
|
||||
"vertexai.init(\n",
|
||||
" project=PROJECT_ID,\n",
|
||||
" location=REGION,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"version_id = \"clip-vit-base-patch32\"\n",
|
||||
"PUBLISHER_MODEL_NAME = f\"publishers/openai/models/clip-vit-base-patch32@{version_id}\"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def deploy_model(model_id, task):\n",
|
||||
@@ -284,7 +171,7 @@
|
||||
" serving_container_health_route=\"/ping\",\n",
|
||||
" serving_container_environment_variables=serving_env,\n",
|
||||
" artifact_uri=artifact_uri,\n",
|
||||
" model_garden_source_model_name=\"publishers/openai/models/clip-vit-base-patch32\"\n",
|
||||
" model_garden_source_model_name=\"publishers/openai/models/clip-vit-base-patch32\",\n",
|
||||
" )\n",
|
||||
" model.deploy(\n",
|
||||
" endpoint=endpoint,\n",
|
||||
@@ -293,147 +180,89 @@
|
||||
" accelerator_count=1,\n",
|
||||
" deploy_request_timeout=1800,\n",
|
||||
" system_labels={\n",
|
||||
" \"NOTEBOOK_NAME\": \"model_garden_pytorch_clip.ipynb\"\n",
|
||||
" \"NOTEBOOK_NAME\": \"model_garden_pytorch_clip.ipynb\",\n",
|
||||
" \"NOTEBOOK_ENVIRONMENT\": common_util.get_deploy_source(),\n",
|
||||
" },\n",
|
||||
" )\n",
|
||||
" return model, endpoint"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "d2d72ecdb8c9"
|
||||
},
|
||||
"source": [
|
||||
"## Upload and deploy models"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "9448c5f545fa"
|
||||
},
|
||||
"source": [
|
||||
"This section uploads the pre-trained model to Model Registry and deploys it on the Endpoint with 1 T4 GPU.\n",
|
||||
"\n",
|
||||
"The model deployment step will take ~15 minutes to complete."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "99f3c6b404b5"
|
||||
},
|
||||
"source": [
|
||||
"### Zero-shot image classification"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "b4b46c28d8b1"
|
||||
"cellView": "form",
|
||||
"id": "LQir_tczhdEP"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"model, endpoint = deploy_model(\n",
|
||||
" model_id=\"openai/clip-vit-base-patch32\", task=\"zero-shot-image-classification\"\n",
|
||||
"accelerator_type = \"NVIDIA_TESLA_T4\"\n",
|
||||
"machine_type = \"n1-standard-8\"\n",
|
||||
"accelerator_count = 1\n",
|
||||
"\n",
|
||||
"# @title [Option 1] Deploy with Model Garden SDK\n",
|
||||
"\n",
|
||||
"# @markdown Deploy with Gen AI model-centric SDK. This section uploads the prebuilt model to Model Registry and deploys it to a Vertex AI Endpoint. It takes 15 minutes to 1 hour to finish depending on the size of the model. See [use open models with Vertex AI](https://cloud.google.com/vertex-ai/generative-ai/docs/open-models/use-open-models) for documentation on other use cases.\n",
|
||||
"from vertexai.preview import model_garden\n",
|
||||
"\n",
|
||||
"model = model_garden.OpenModel(PUBLISHER_MODEL_NAME)\n",
|
||||
"endpoints[LABEL] = model.deploy(\n",
|
||||
" machine_type=machine_type,\n",
|
||||
" accelerator_type=accelerator_type,\n",
|
||||
" accelerator_count=accelerator_count,\n",
|
||||
" accept_eula=True, # Accept the End User License Agreement (EULA) on the model card before deploy. Otherwise, the deployment will be forbidden.\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "80b3fd2ace09"
|
||||
},
|
||||
"source": [
|
||||
"NOTE: The model weights will be downloaded after the deployment succeeds. Thus additional 5 minutes of waiting time is needed **after** the above model deployment step succeeds and before you run the next step below. Otherwise you might see a `ServiceUnavailable: 503 502:Bad Gateway` error when you send requests to the endpoint."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "6be655247cb1"
|
||||
"cellView": "form",
|
||||
"id": "vBvfVTWOhdEP"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"image1 = download_image(\"http://images.cocodataset.org/val2017/000000039769.jpg\")\n",
|
||||
"image2 = download_image(\"http://images.cocodataset.org/val2017/000000000285.jpg\")\n",
|
||||
"grid = image_grid([image1, image2], 1, 2)\n",
|
||||
"# @title [Option 2] Deploy with customized configs\n",
|
||||
"\n",
|
||||
"# @markdown This section uploads the pre-trained model to Model Registry and deploys it on the Endpoint with 1 T4 GPU.\n",
|
||||
"# @markdown The model deployment step will take ~15 minutes to complete.\n",
|
||||
"\n",
|
||||
"models[LABEL], endpoints[LABEL] = deploy_model(\n",
|
||||
" model_id=\"openai/clip-vit-base-patch32\", task=\"zero-shot-image-classification\"\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"image1 = common_util.download_image(\n",
|
||||
" \"http://images.cocodataset.org/val2017/000000039769.jpg\"\n",
|
||||
")\n",
|
||||
"image2 = common_util.download_image(\n",
|
||||
" \"http://images.cocodataset.org/val2017/000000000285.jpg\"\n",
|
||||
")\n",
|
||||
"grid = common_util.image_grid([image1, image2], 1, 2)\n",
|
||||
"display(grid)\n",
|
||||
"\n",
|
||||
"instances = [\n",
|
||||
" {\"image\": image_to_base64(image1), \"text\": \"two cats\"},\n",
|
||||
" {\"image\": image_to_base64(image2), \"text\": \"a bear\"},\n",
|
||||
" {\"image\": common_util.image_to_base64(image1), \"text\": \"two cats\"},\n",
|
||||
" {\"image\": common_util.image_to_base64(image2), \"text\": \"a bear\"},\n",
|
||||
"]\n",
|
||||
"preds = endpoint.predict(instances=instances).predictions\n",
|
||||
"print(preds)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "2ccf3714dbe9"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Undeploy model and delete endpoint.\n",
|
||||
"endpoint.delete(force=True)\n",
|
||||
"preds = endpoints[LABEL].predict(instances=instances).predictions\n",
|
||||
"print(preds)\n",
|
||||
"\n",
|
||||
"# Delete models.\n",
|
||||
"model.delete()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "ddf9e9ec7b58"
|
||||
},
|
||||
"source": [
|
||||
"### Image/text feature embedding"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "8897bbac6887"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"model, endpoint = deploy_model(\n",
|
||||
"models[LABEL], endpoints[LABEL] = deploy_model(\n",
|
||||
" model_id=\"openai/clip-vit-base-patch32\", task=\"feature-embedding\"\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "9f854c45f8f3"
|
||||
},
|
||||
"source": [
|
||||
"NOTE: The model weights will be downloaded after the deployment succeeds. Thus additional 5 minutes of waiting time is needed after the above model deployment step succeeds and before you run the next step below. Otherwise you might see a ServiceUnavailable: 503 502:Bad Gateway error when you send requests to the endpoint."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "2abd54335f36"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
")\n",
|
||||
"\n",
|
||||
"import numpy as np\n",
|
||||
"\n",
|
||||
"# Extract feature embedding of images.\n",
|
||||
"image = download_image(\"http://images.cocodataset.org/val2017/000000039769.jpg\")\n",
|
||||
"image = common_util.download_image(\n",
|
||||
" \"http://images.cocodataset.org/val2017/000000039769.jpg\"\n",
|
||||
")\n",
|
||||
"display(image)\n",
|
||||
"instances = [\n",
|
||||
" {\"image\": image_to_base64(image)},\n",
|
||||
" {\"image\": common_util.image_to_base64(image)},\n",
|
||||
"]\n",
|
||||
"preds = endpoint.predict(instances=instances).predictions\n",
|
||||
"preds = endpoints[LABEL].predict(instances=instances).predictions\n",
|
||||
"image_features = np.array(preds[0][\"image_features\"])\n",
|
||||
"print(image_features.shape)\n",
|
||||
"\n",
|
||||
@@ -442,7 +271,7 @@
|
||||
" {\"text\": \"two cats\"},\n",
|
||||
" {\"text\": \"hello world\"},\n",
|
||||
"]\n",
|
||||
"preds = endpoint.predict(instances=instances).predictions\n",
|
||||
"preds = endpoints[LABEL].predict(instances=instances).predictions\n",
|
||||
"text_features = np.array(preds[0][\"text_features\"])\n",
|
||||
"print(text_features.shape)"
|
||||
]
|
||||
@@ -451,15 +280,21 @@
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "712eb9d0b336"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @markdown Delete the experiment models and endpoints to recycle the resources\n",
|
||||
"# @markdown and avoid unnecessary continuous charges that may incur.\n",
|
||||
"\n",
|
||||
"# Undeploy model and delete endpoint.\n",
|
||||
"endpoint.delete(force=True)\n",
|
||||
"for endpoint in endpoints.values():\n",
|
||||
" endpoint.delete(force=True)\n",
|
||||
"\n",
|
||||
"# Delete models.\n",
|
||||
"model.delete()"
|
||||
"for model in models.values():\n",
|
||||
" model.delete()"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
@@ -210,8 +210,8 @@
|
||||
"\n",
|
||||
"model_id = os.path.join(VERTEX_AI_MODEL_GARDEN_CODE_LLAMA, model_name)\n",
|
||||
"\n",
|
||||
"# @markdown Set use_dedicated_endpoint to True if you want to use [dedicated endpoint](https://cloud.google.com/vertex-ai/docs/general/deployment#create-dedicated-endpoint).\n",
|
||||
"use_dedicated_endpoint = False # @param {type:\"boolean\"}\n",
|
||||
"# @markdown Set use_dedicated_endpoint to False if you don't want to use [dedicated endpoint](https://cloud.google.com/vertex-ai/docs/general/deployment#create-dedicated-endpoint). Note that [dedicated endpoint does not support VPC Service Controls](https://cloud.google.com/vertex-ai/docs/predictions/choose-endpoint-type), uncheck the box if you are using VPC-SC.\n",
|
||||
"use_dedicated_endpoint = True # @param {type:\"boolean\"}\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def deploy_model_vllm(\n",
|
||||
@@ -476,6 +476,7 @@
|
||||
" accelerator_count=accelerator_count,\n",
|
||||
" max_model_len=max_model_len,\n",
|
||||
" dtype=vllm_precision,\n",
|
||||
" use_dedicated_endpoint=use_dedicated_endpoint,\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
|
||||
@@ -59,12 +59,16 @@
|
||||
"source": [
|
||||
"## Overview\n",
|
||||
"\n",
|
||||
"This notebook demonstrates serving DeepSeek models with [vLLM](https://github.com/vllm-project/vllm) or [SGLang](https://github.com/sgl-project/sglang). [DeepSeek-V3](https://huggingface.co/deepseek-ai/DeepSeek-V3) is a strong Mixture-of-Experts (MoE) language model with 671B total parameters with 37B activated for each token. [DeepSeek-R1](https://huggingface.co/deepseek-ai/DeepSeek-R1) is one of the first-generation reasoning models introduced by DeepSeek and offers performance comparable to OpenAI-o1 across math, code, and reasoning tasks.\n",
|
||||
"This notebook demonstrates serving DeepSeek models with [vLLM](https://github.com/vllm-project/vllm), [SGLang](https://github.com/sgl-project/sglang), or [TensorRT-LLM](https://github.com/NVIDIA/TensorRT-LLM). [DeepSeek-V3](https://huggingface.co/deepseek-ai/DeepSeek-V3) is a strong Mixture-of-Experts (MoE) language model with 671B total parameters with 37B activated for each token. [DeepSeek-R1](https://huggingface.co/deepseek-ai/DeepSeek-R1) is one of the first-generation reasoning models introduced by DeepSeek and offers performance comparable to OpenAI-o1 across math, code, and reasoning tasks.\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"### Objective\n",
|
||||
"\n",
|
||||
"- Deploy DeepSeek-V3 and DeepSeek-R1 with vLLM or SGLang on GPU using single-host and multi-host serving, and [Spot VMs](https://cloud.google.com/compute/docs/instances/spot) (Optional). Multi-host GPU serving is a preview feature.\n",
|
||||
"- Deploy DeepSeek-V3 and DeepSeek-R1 with vLLM, SGLang, or TensorRT-LLM on GPU using single-host and multi-host serving, and [Spot VMs](https://cloud.google.com/compute/docs/instances/spot) (Optional). Multi-host GPU serving is a preview feature.\n",
|
||||
"\n",
|
||||
"### File a bug\n",
|
||||
"\n",
|
||||
"File a bug on [GitHub](https://github.com/GoogleCloudPlatform/vertex-ai-samples/issues/new) if you encounter any issue with the notebook.\n",
|
||||
"\n",
|
||||
"### Costs\n",
|
||||
"\n",
|
||||
@@ -106,7 +110,6 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"language": "python",
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "YXFGIp1l-qtT"
|
||||
@@ -133,6 +136,7 @@
|
||||
"\n",
|
||||
"# Import the necessary packages\n",
|
||||
"import importlib\n",
|
||||
"import json\n",
|
||||
"import os\n",
|
||||
"import time\n",
|
||||
"from typing import Tuple\n",
|
||||
@@ -179,6 +183,7 @@
|
||||
" )\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"LABEL = \"vllm_gpu\"\n",
|
||||
"models, endpoints = {}, {}\n",
|
||||
"\n",
|
||||
"# Get the default cloud project id.\n",
|
||||
@@ -188,20 +193,10 @@
|
||||
"if not REGION:\n",
|
||||
" REGION = os.environ[\"GOOGLE_CLOUD_REGION\"]\n",
|
||||
"\n",
|
||||
"# Enable the Vertex AI API and Compute Engine API, if not already.\n",
|
||||
"print(\"Enabling Vertex AI API and Compute Engine API.\")\n",
|
||||
"! gcloud services enable aiplatform.googleapis.com compute.googleapis.com\n",
|
||||
"\n",
|
||||
"# Initialize Vertex AI API.\n",
|
||||
"print(\"Initializing Vertex AI API.\")\n",
|
||||
"aiplatform.init(project=PROJECT_ID, location=REGION)\n",
|
||||
"\n",
|
||||
"# Gets the default SERVICE_ACCOUNT.\n",
|
||||
"shell_output = ! gcloud projects describe $PROJECT_ID\n",
|
||||
"project_number = shell_output[-1].split(\":\")[1].strip().replace(\"'\", \"\")\n",
|
||||
"SERVICE_ACCOUNT = f\"{project_number}-compute@developer.gserviceaccount.com\"\n",
|
||||
"print(\"Using this default Service Account:\", SERVICE_ACCOUNT)\n",
|
||||
"\n",
|
||||
"! gcloud config set project $PROJECT_ID\n",
|
||||
"\n",
|
||||
"import vertexai\n",
|
||||
@@ -230,7 +225,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"LABEL = \"vllm_gpu\"\n",
|
||||
"# @title Set the model variants\n",
|
||||
"\n",
|
||||
"# @markdown It's recommended to use the region selected by the deployment button on the model card. If the deployment button is not available, it's recommended to stay with the default region of the notebook.\n",
|
||||
"\n",
|
||||
@@ -247,9 +242,12 @@
|
||||
"else:\n",
|
||||
" model_user_id = \"deepseek-v3\"\n",
|
||||
"\n",
|
||||
"# fmt: off\n",
|
||||
"PUBLISHER_MODEL_NAME = f\"publishers/deepseek-ai/models/{model_user_id}@{base_model_name.lower()}\"\n",
|
||||
"# fmt: on"
|
||||
"PUBLISHER_MODEL_NAME = (\n",
|
||||
" f\"publishers/deepseek-ai/models/{model_user_id}@{base_model_name.lower()}\"\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# @markdown Set use_dedicated_endpoint to False if you don't want to use [dedicated endpoint](https://cloud.google.com/vertex-ai/docs/general/deployment#create-dedicated-endpoint). Note that [dedicated endpoint does not support VPC Service Controls](https://cloud.google.com/vertex-ai/docs/predictions/choose-endpoint-type), uncheck the box if you are using VPC-SC.\n",
|
||||
"use_dedicated_endpoint = True # @param {type:\"boolean\"}"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -269,10 +267,6 @@
|
||||
"# The pre-built serving docker image for vLLM past v0.7.3, https://github.com/vllm-project/vllm/commit/f6bb18fd9a19e5e4fb1991339638fc666d06b27a.\n",
|
||||
"VLLM_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-vllm-serve:20250312_0916_RC01\"\n",
|
||||
"\n",
|
||||
"# @markdown Use a [dedicated endpoint](https://cloud.google.com/vertex-ai/docs/general/deployment#create-dedicated-endpoint) for the deployment.\n",
|
||||
"# @markdown Only dedicated endpoints are supported at this moment.\n",
|
||||
"use_dedicated_endpoint = True # @param {type:\"boolean\"}\n",
|
||||
"\n",
|
||||
"# @markdown Choose whether to use a [Spot VM](https://cloud.google.com/compute/docs/instances/spot) for the deployment.\n",
|
||||
"is_spot = False # @param {type:\"boolean\"}\n",
|
||||
"\n",
|
||||
@@ -529,13 +523,14 @@
|
||||
" \"requiredReplicaCount\": required_replica_count,\n",
|
||||
" \"maxReplicaCount\": max_replica_count,\n",
|
||||
" },\n",
|
||||
" \"serviceAccount\": service_account,\n",
|
||||
" \"system_labels\": {\n",
|
||||
" \"NOTEBOOK_NAME\": \"model_garden_pytorch_deepseek_deployment.ipynb\",\n",
|
||||
" \"NOTEBOOK_ENVIRONMENT\": common_util.get_deploy_source(),\n",
|
||||
" },\n",
|
||||
" },\n",
|
||||
" }\n",
|
||||
" if service_account:\n",
|
||||
" data[\"deployedModel\"][\"serviceAccount\"] = service_account\n",
|
||||
" if is_spot:\n",
|
||||
" data[\"deployedModel\"][\"dedicatedResources\"][\"spot\"] = True\n",
|
||||
" if autoscale_by_gpu_duty_cycle_target > 0 or autoscale_by_cpu_usage_target > 0:\n",
|
||||
@@ -574,7 +569,6 @@
|
||||
" publisher=\"deepseek-ai\",\n",
|
||||
" publisher_model_id=(\"deepseek-v3\" if \"V3\" in model_id else \"deepseek-r1\"),\n",
|
||||
" base_model_id=hf_model_id,\n",
|
||||
" service_account=None,\n",
|
||||
" machine_type=machine_type,\n",
|
||||
" accelerator_type=accelerator_type,\n",
|
||||
" accelerator_count=accelerator_count,\n",
|
||||
@@ -681,9 +675,7 @@
|
||||
"\n",
|
||||
"if use_dedicated_endpoint:\n",
|
||||
" DEDICATED_ENDPOINT_DNS = endpoints[\"vllm_gpu\"].gca_resource.dedicated_endpoint_dns\n",
|
||||
"ENDPOINT_RESOURCE_NAME = \"projects/{}/locations/{}/endpoints/{}\".format(\n",
|
||||
" PROJECT_ID, REGION, endpoints[\"vllm_gpu\"].name\n",
|
||||
")\n",
|
||||
"ENDPOINT_RESOURCE_NAME = endpoints[\"vllm_gpu\"].resource_name\n",
|
||||
"\n",
|
||||
"# @markdown Because the DeepSeek-R1 model generates detailed reasoning steps, the output is expected to be long. We recommend using streaming for a better generation experience.\n",
|
||||
"# @title Chat Completions Inference\n",
|
||||
@@ -779,15 +771,14 @@
|
||||
"hf_model_id = model_id\n",
|
||||
"\n",
|
||||
"# The pre-built serving docker images.\n",
|
||||
"SGLANG_DOCKER_URI = \"us-docker.pkg.dev/deeplearning-platform-release/vertex-model-garden/sglang-serve.cu124.0-4.ubuntu2204.py310\"\n",
|
||||
"\n",
|
||||
"# @markdown Use a [dedicated endpoint](https://cloud.google.com/vertex-ai/docs/general/deployment#create-dedicated-endpoint) for the deployment.\n",
|
||||
"# @markdown Only dedicated endpoints are supported at this moment.\n",
|
||||
"use_dedicated_endpoint = True # @param {type:\"boolean\"}\n",
|
||||
"SGLANG_DOCKER_URI = \"us-docker.pkg.dev/deeplearning-platform-release/vertex-model-garden/sglang-serve.cu124.0-4.ubuntu2204.py310:20250427-1800-rc0\"\n",
|
||||
"\n",
|
||||
"# @markdown Choose whether to use a [Spot VM](https://cloud.google.com/compute/docs/instances/spot) for the deployment.\n",
|
||||
"is_spot = False # @param {type:\"boolean\"}\n",
|
||||
"\n",
|
||||
"# @markdown Set use_dedicated_endpoint to False if you don't want to use [dedicated endpoint](https://cloud.google.com/vertex-ai/docs/general/deployment#create-dedicated-endpoint). Note that [dedicated endpoint does not support VPC Service Controls](https://cloud.google.com/vertex-ai/docs/predictions/choose-endpoint-type), uncheck the box if you are using VPC-SC.\n",
|
||||
"use_dedicated_endpoint = True # @param {type:\"boolean\"}\n",
|
||||
"\n",
|
||||
"# @markdown Find Vertex AI prediction supported accelerators and regions at https://cloud.google.com/vertex-ai/docs/predictions/configure-compute.\n",
|
||||
"accelerator_type = \"NVIDIA_H200_141GB\" # @param [\"NVIDIA_H200_141GB\", \"NVIDIA_H100_80GB\"] {isTemplate:true}\n",
|
||||
"accelerator_count = 8\n",
|
||||
@@ -819,9 +810,11 @@
|
||||
"# @markdown 2. High throughput: This profile optimizes for high throughput on large batches of incoming requests.\n",
|
||||
"profile = \"Low latency\" # @param [\"Low latency\", \"High throughput\"] {isTemplate:true}\n",
|
||||
"\n",
|
||||
"# Set this value to the expected number of concurrent requests.\n",
|
||||
"torch_compile_max_bs = 4\n",
|
||||
"\n",
|
||||
"if profile == \"Low latency\":\n",
|
||||
" enable_torch_compile = True\n",
|
||||
" enable_flashinfer_mla = True\n",
|
||||
" disable_cuda_graph = False\n",
|
||||
" if base_model_name not in (\"DeepSeek-V3\", \"DeepSeek-V3-0324\", \"DeepSeek-R1\"):\n",
|
||||
" speculative_algorithm = None\n",
|
||||
@@ -837,7 +830,6 @@
|
||||
" dp_size = 1\n",
|
||||
"else:\n",
|
||||
" enable_torch_compile = False\n",
|
||||
" enable_flashinfer_mla = True\n",
|
||||
" disable_cuda_graph = False\n",
|
||||
" speculative_algorithm = None\n",
|
||||
" speculative_draft_model_path = \"\"\n",
|
||||
@@ -882,7 +874,7 @@
|
||||
" model_id: str,\n",
|
||||
" publisher: str,\n",
|
||||
" publisher_model_id: str,\n",
|
||||
" service_account: str,\n",
|
||||
" service_account: str = \"\",\n",
|
||||
" base_model_id: str = \"\",\n",
|
||||
" machine_type: str = \"g2-standard-8\",\n",
|
||||
" accelerator_type: str = \"NVIDIA_L4\",\n",
|
||||
@@ -893,6 +885,8 @@
|
||||
" dtype: str | None = None,\n",
|
||||
" enable_trust_remote_code: bool = False,\n",
|
||||
" enable_torch_compile: bool = False,\n",
|
||||
" torch_compile_max_bs: int | None = None,\n",
|
||||
" attention_backend: str = \"\",\n",
|
||||
" enable_flashinfer_mla: bool = False,\n",
|
||||
" disable_cuda_graph: bool = False,\n",
|
||||
" speculative_algorithm: str | None = None,\n",
|
||||
@@ -940,6 +934,11 @@
|
||||
"\n",
|
||||
" if enable_torch_compile:\n",
|
||||
" sglang_args.append(\"--enable-torch-compile\")\n",
|
||||
" if torch_compile_max_bs:\n",
|
||||
" sglang_args.append(f\"--torch-compile-max-bs={torch_compile_max_bs}\")\n",
|
||||
"\n",
|
||||
" if attention_backend:\n",
|
||||
" sglang_args.append(f\"--attention-backend={attention_backend}\")\n",
|
||||
"\n",
|
||||
" if enable_flashinfer_mla:\n",
|
||||
" sglang_args.append(\"--enable-flashinfer-mla\")\n",
|
||||
@@ -1017,13 +1016,14 @@
|
||||
" \"minReplicaCount\": 1,\n",
|
||||
" \"maxReplicaCount\": 1,\n",
|
||||
" },\n",
|
||||
" \"serviceAccount\": service_account,\n",
|
||||
" \"system_labels\": {\n",
|
||||
" \"NOTEBOOK_NAME\": \"model_garden_pytorch_deepseek_deployment.ipynb\",\n",
|
||||
" \"NOTEBOOK_ENVIRONMENT\": common_util.get_deploy_source(),\n",
|
||||
" },\n",
|
||||
" },\n",
|
||||
" }\n",
|
||||
" if service_account:\n",
|
||||
" data[\"deployedModel\"][\"serviceAccount\"] = service_account\n",
|
||||
" if is_spot:\n",
|
||||
" data[\"deployedModel\"][\"dedicatedResources\"][\"spot\"] = True\n",
|
||||
" response = requests.post(url, headers=headers, json=data)\n",
|
||||
@@ -1041,7 +1041,6 @@
|
||||
" model_id=model_id,\n",
|
||||
" publisher=\"deepseek-ai\",\n",
|
||||
" publisher_model_id=(\"deepseek-v3\" if \"V3\" in model_id else \"deepseek-r1\"),\n",
|
||||
" service_account=SERVICE_ACCOUNT,\n",
|
||||
" base_model_id=hf_model_id,\n",
|
||||
" machine_type=machine_type,\n",
|
||||
" accelerator_type=accelerator_type,\n",
|
||||
@@ -1049,7 +1048,8 @@
|
||||
" multihost_gpu_node_count=multihost_gpu_node_count,\n",
|
||||
" enable_trust_remote_code=True,\n",
|
||||
" enable_torch_compile=enable_torch_compile,\n",
|
||||
" enable_flashinfer_mla=enable_flashinfer_mla,\n",
|
||||
" torch_compile_max_bs=torch_compile_max_bs,\n",
|
||||
" attention_backend=\"fa3\",\n",
|
||||
" disable_cuda_graph=disable_cuda_graph,\n",
|
||||
" speculative_algorithm=speculative_algorithm,\n",
|
||||
" speculative_draft_model_path=speculative_draft_model_path,\n",
|
||||
@@ -1143,9 +1143,7 @@
|
||||
"\n",
|
||||
"if use_dedicated_endpoint:\n",
|
||||
" DEDICATED_ENDPOINT_DNS = endpoints[\"sglang_gpu\"].gca_resource.dedicated_endpoint_dns\n",
|
||||
"ENDPOINT_RESOURCE_NAME = \"projects/{}/locations/{}/endpoints/{}\".format(\n",
|
||||
" PROJECT_ID, REGION, endpoints[\"sglang_gpu\"].name\n",
|
||||
")\n",
|
||||
"ENDPOINT_RESOURCE_NAME = endpoints[\"sglang_gpu\"].resource_name\n",
|
||||
"\n",
|
||||
"# @markdown Because the DeepSeek-R1 / DeepSeek-V3 model generates detailed reasoning steps, the output is expected to be long. We recommend using streaming for a better generation experience.\n",
|
||||
"# @title Chat Completions Inference\n",
|
||||
@@ -1208,6 +1206,293 @@
|
||||
"# @markdown Click \"Show Code\" to see more details."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "cyg_QzpQD0DJ"
|
||||
},
|
||||
"source": [
|
||||
"## Deploy DeepSeek-V3 and DeepSeek-R1 with TensorRT-LLM"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "wfoBFRv_D4mB"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Deploy\n",
|
||||
"\n",
|
||||
"# @markdown This section uploads DeepSeek models to Model Registry and deploys them to a Vertex Prediction Endpoint. It takes ~1 hour to finish.\n",
|
||||
"\n",
|
||||
"# @markdown It's recommended to use the region selected by the deployment button on the model card. If the deployment button is not available, it's recommended to stay with the default region of the notebook.\n",
|
||||
"\n",
|
||||
"# @markdown Set the model to deploy.\n",
|
||||
"\n",
|
||||
"base_model_name = \"DeepSeek-R1\" # @param [\"DeepSeek-V3\", \"DeepSeek-V3-Base\", \"DeepSeek-V3-0324\", \"DeepSeek-R1\"] {isTemplate:true}\n",
|
||||
"model_id = \"deepseek-ai/\" + base_model_name\n",
|
||||
"hf_model_id = model_id\n",
|
||||
"\n",
|
||||
"# The pre-built serving docker images.\n",
|
||||
"TRTLLM_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/tensorrt-llm.cu128.0-18.ubuntu2404.py312:deepseek\"\n",
|
||||
"\n",
|
||||
"# @markdown Choose whether to use a [Spot VM](https://cloud.google.com/compute/docs/instances/spot) for the deployment.\n",
|
||||
"is_spot = False # @param {type:\"boolean\"}\n",
|
||||
"\n",
|
||||
"# @markdown Set use_dedicated_endpoint to False if you don't want to use [dedicated endpoint](https://cloud.google.com/vertex-ai/docs/general/deployment#create-dedicated-endpoint). Note that [dedicated endpoint does not support VPC Service Controls](https://cloud.google.com/vertex-ai/docs/predictions/choose-endpoint-type), uncheck the box if you are using VPC-SC.\n",
|
||||
"use_dedicated_endpoint = True # @param {type:\"boolean\"}\n",
|
||||
"\n",
|
||||
"# @markdown Find Vertex AI prediction supported accelerators and regions at https://cloud.google.com/vertex-ai/docs/predictions/configure-compute.\n",
|
||||
"trtllm_accelerator_type = \"NVIDIA_H200_141GB\" # @param [\"NVIDIA_H200_141GB\"] {isTemplate:true}\n",
|
||||
"accelerator_count = 8\n",
|
||||
"if trtllm_accelerator_type == \"NVIDIA_H200_141GB\":\n",
|
||||
" machine_type = \"a3-ultragpu-8g\"\n",
|
||||
" multihost_gpu_node_count = 1\n",
|
||||
" resource_id = \"custom_model_serving_nvidia_h200_gpus\"\n",
|
||||
"else:\n",
|
||||
" raise ValueError(\"Only NVIDIA_H200_141GB is supported for DeepSeek-R1.\")\n",
|
||||
"\n",
|
||||
"check_quota(\n",
|
||||
" project_id=PROJECT_ID,\n",
|
||||
" region=REGION,\n",
|
||||
" resource_id=resource_id,\n",
|
||||
" accelerator_count=int(accelerator_count * multihost_gpu_node_count),\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# 18K context length. This is the maximum supported by the current version of TensorRT-LLM on DeepSeek V3/R1 models.\n",
|
||||
"MAX_INPUT_LEN = 18000\n",
|
||||
"MAX_MODEL_LEN = 18000\n",
|
||||
"MAX_NUM_SEQS = 128\n",
|
||||
"GPU_MEMORY_UTILIZATION = 0.55\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def poll_operation(op_name: str) -> bool: # noqa: F811\n",
|
||||
" creds, _ = auth.default()\n",
|
||||
" auth_req = auth.transport.requests.Request()\n",
|
||||
" creds.refresh(auth_req)\n",
|
||||
" headers = {\n",
|
||||
" \"Authorization\": f\"Bearer {creds.token}\",\n",
|
||||
" }\n",
|
||||
" get_resp = requests.get(\n",
|
||||
" f\"https://{REGION}-aiplatform.googleapis.com/ui/{op_name}\",\n",
|
||||
" headers=headers,\n",
|
||||
" )\n",
|
||||
" opjs = get_resp.json()\n",
|
||||
" if \"error\" in opjs:\n",
|
||||
" raise ValueError(f\"Operation failed: {opjs['error']}\")\n",
|
||||
" return opjs.get(\"done\", False)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def poll_and_wait(op_name: str, total_wait: int, interval: int = 60): # noqa: F811\n",
|
||||
" waited = 0\n",
|
||||
" while not poll_operation(op_name):\n",
|
||||
" if waited > total_wait:\n",
|
||||
" raise TimeoutError(\"Operation timed out\")\n",
|
||||
" print(\n",
|
||||
" f\"\\rStill waiting for operation... Waited time in second: {waited:<6}\",\n",
|
||||
" end=\"\",\n",
|
||||
" flush=True,\n",
|
||||
" )\n",
|
||||
" waited += interval\n",
|
||||
" time.sleep(interval)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def deploy_model_tensorrt_llm_multihost(\n",
|
||||
" model_name: str,\n",
|
||||
" model_id: str,\n",
|
||||
" publisher: str,\n",
|
||||
" publisher_model_id: str,\n",
|
||||
" service_account: str = \"\",\n",
|
||||
" base_model_id: str = \"\",\n",
|
||||
" machine_type: str = \"g2-standard-8\",\n",
|
||||
" accelerator_type: str = \"NVIDIA_L4\",\n",
|
||||
" accelerator_count: int = 1,\n",
|
||||
" multihost_gpu_node_count: int = 1,\n",
|
||||
" gpu_memory_utilization: float | None = None,\n",
|
||||
" max_input_len: int | None = None,\n",
|
||||
" max_model_len: int | None = None,\n",
|
||||
" max_num_seqs: int | None = None,\n",
|
||||
" enable_trust_remote_code: bool = False,\n",
|
||||
" enable_chunked_prefill: bool = False,\n",
|
||||
" use_dedicated_endpoint: bool = False,\n",
|
||||
" is_spot: bool = True,\n",
|
||||
") -> Tuple[aiplatform.Model, aiplatform.Endpoint]:\n",
|
||||
" \"\"\"Deploys trained models with TensorRT-LLM on Vertex AI.\"\"\"\n",
|
||||
" endpoint = aiplatform.Endpoint.create(\n",
|
||||
" display_name=f\"{model_name}-endpoint\",\n",
|
||||
" dedicated_endpoint_enabled=use_dedicated_endpoint,\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" if not base_model_id:\n",
|
||||
" base_model_id = model_id\n",
|
||||
"\n",
|
||||
" container_args = [\n",
|
||||
" \"python\",\n",
|
||||
" \"api_server.py\",\n",
|
||||
" \"--host=0.0.0.0\",\n",
|
||||
" \"--port=7080\",\n",
|
||||
" f\"--model={model_id}\",\n",
|
||||
" f\"--tensor-parallel-size={accelerator_count * multihost_gpu_node_count}\",\n",
|
||||
" ]\n",
|
||||
"\n",
|
||||
" if gpu_memory_utilization:\n",
|
||||
" container_args.append(f\"--gpu-memory-utilization={gpu_memory_utilization}\")\n",
|
||||
"\n",
|
||||
" if max_input_len:\n",
|
||||
" container_args.append(f\"--max-input-len={max_input_len}\")\n",
|
||||
"\n",
|
||||
" if max_model_len:\n",
|
||||
" container_args.append(f\"--max-model-len={max_model_len}\")\n",
|
||||
"\n",
|
||||
" if max_num_seqs:\n",
|
||||
" container_args.append(f\"--max-num-seqs={max_num_seqs}\")\n",
|
||||
"\n",
|
||||
" if enable_trust_remote_code:\n",
|
||||
" container_args.append(\"--trust-remote-code=True\")\n",
|
||||
"\n",
|
||||
" if enable_chunked_prefill:\n",
|
||||
" container_args.append(\"--enable-chunked-prefill=True\")\n",
|
||||
"\n",
|
||||
" env_vars = {\n",
|
||||
" \"MODEL_ID\": base_model_id,\n",
|
||||
" \"DEPLOY_SOURCE\": \"notebook\",\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" # HF_TOKEN is not a compulsory field and may not be defined.\n",
|
||||
" try:\n",
|
||||
" if HF_TOKEN:\n",
|
||||
" env_vars[\"HF_TOKEN\"] = HF_TOKEN\n",
|
||||
" except NameError:\n",
|
||||
" pass\n",
|
||||
"\n",
|
||||
" model = aiplatform.Model.upload(\n",
|
||||
" display_name=model_name,\n",
|
||||
" serving_container_image_uri=TRTLLM_DOCKER_URI,\n",
|
||||
" serving_container_args=container_args,\n",
|
||||
" serving_container_ports=[7080],\n",
|
||||
" serving_container_predict_route=\"/v1/chat/completions\",\n",
|
||||
" serving_container_health_route=\"/health\",\n",
|
||||
" serving_container_environment_variables=env_vars,\n",
|
||||
" serving_container_shared_memory_size_mb=(16 * 1024), # 16 GB\n",
|
||||
" serving_container_deployment_timeout=7200,\n",
|
||||
" model_garden_source_model_name=(\n",
|
||||
" f\"publishers/{publisher}/models/{publisher_model_id}\"\n",
|
||||
" ),\n",
|
||||
" )\n",
|
||||
" print(\n",
|
||||
" f\"Deploying {model_name} on {machine_type} with {int(accelerator_count * multihost_gpu_node_count)} {accelerator_type} GPU(s).\"\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" creds, _ = auth.default()\n",
|
||||
" auth_req = auth.transport.requests.Request()\n",
|
||||
" creds.refresh(auth_req)\n",
|
||||
"\n",
|
||||
" url = f\"https://{REGION}-aiplatform.googleapis.com/ui/projects/{PROJECT_ID}/locations/{REGION}/endpoints/{endpoint.name}:deployModel\"\n",
|
||||
" headers = {\n",
|
||||
" \"Content-Type\": \"application/json\",\n",
|
||||
" \"Authorization\": f\"Bearer {creds.token}\",\n",
|
||||
" }\n",
|
||||
" data = {\n",
|
||||
" \"deployedModel\": {\n",
|
||||
" \"model\": model.resource_name,\n",
|
||||
" \"displayName\": model_name,\n",
|
||||
" \"dedicatedResources\": {\n",
|
||||
" \"machineSpec\": {\n",
|
||||
" \"machineType\": machine_type,\n",
|
||||
" \"multihostGpuNodeCount\": multihost_gpu_node_count,\n",
|
||||
" \"acceleratorType\": accelerator_type,\n",
|
||||
" \"acceleratorCount\": accelerator_count,\n",
|
||||
" },\n",
|
||||
" \"minReplicaCount\": 1,\n",
|
||||
" \"maxReplicaCount\": 1,\n",
|
||||
" },\n",
|
||||
" \"system_labels\": {\n",
|
||||
" \"NOTEBOOK_NAME\": \"model_garden_pytorch_deepseek_deployment.ipynb\",\n",
|
||||
" \"NOTEBOOK_ENVIRONMENT\": common_util.get_deploy_source(),\n",
|
||||
" },\n",
|
||||
" },\n",
|
||||
" }\n",
|
||||
" if service_account:\n",
|
||||
" data[\"deployedModel\"][\"serviceAccount\"] = service_account\n",
|
||||
" if is_spot:\n",
|
||||
" data[\"deployedModel\"][\"dedicatedResources\"][\"spot\"] = True\n",
|
||||
" response = requests.post(url, headers=headers, json=data)\n",
|
||||
" print(f\"Deploy Model response: {response.json()}\")\n",
|
||||
" if response.status_code != 200 or \"name\" not in response.json():\n",
|
||||
" raise ValueError(f\"Failed to deploy model: {response.text}\")\n",
|
||||
" poll_and_wait(response.json()[\"name\"], 7200)\n",
|
||||
" print(\"endpoint_name:\", endpoint.name)\n",
|
||||
"\n",
|
||||
" return model, endpoint\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"models[\"trtllm_gpu\"], endpoints[\"trtllm_gpu\"] = deploy_model_tensorrt_llm_multihost(\n",
|
||||
" model_name=common_util.get_job_name_with_datetime(prefix=\"deepseek-serve\"),\n",
|
||||
" model_id=model_id,\n",
|
||||
" publisher=\"deepseek-ai\",\n",
|
||||
" publisher_model_id=(\"deepseek-v3\" if \"V3\" in model_id else \"deepseek-r1\"),\n",
|
||||
" base_model_id=hf_model_id,\n",
|
||||
" machine_type=machine_type,\n",
|
||||
" accelerator_type=trtllm_accelerator_type,\n",
|
||||
" accelerator_count=accelerator_count,\n",
|
||||
" multihost_gpu_node_count=multihost_gpu_node_count,\n",
|
||||
" gpu_memory_utilization=GPU_MEMORY_UTILIZATION,\n",
|
||||
" max_input_len=MAX_INPUT_LEN,\n",
|
||||
" max_model_len=MAX_MODEL_LEN,\n",
|
||||
" max_num_seqs=MAX_NUM_SEQS,\n",
|
||||
" enable_trust_remote_code=True,\n",
|
||||
" use_dedicated_endpoint=use_dedicated_endpoint,\n",
|
||||
" is_spot=is_spot,\n",
|
||||
")\n",
|
||||
"# @markdown Click \"Show Code\" to see more details."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "sfnEWPRGFDGD"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Raw predict\n",
|
||||
"# @markdown Once deployment succeeds, you can send requests to the endpoint using `:rawPredict`. The OpenAI Client chat completions support is coming soon.\n",
|
||||
"endpoints[\"trtllm_gpu\"] = aiplatform.Endpoint(endpoints[\"trtllm_gpu\"].resource_name)\n",
|
||||
"\n",
|
||||
"# @markdown Fill out some request parameters:\n",
|
||||
"user_message = \"How is your day going?\" # @param {type: \"string\"}\n",
|
||||
"# @markdown If you encounter the issue like `ServiceUnavailable: 503 Took too long to respond when processing`, you can reduce the maximum number of output tokens, such as set `max_tokens` as 20.\n",
|
||||
"max_tokens = 50 # @param {type: \"integer\"}\n",
|
||||
"temperature = 1.0 # @param {type: \"number\"}\n",
|
||||
"\n",
|
||||
"# @markdown Now we can send a request.\n",
|
||||
"\n",
|
||||
"response = endpoints[\"trtllm_gpu\"].raw_predict(\n",
|
||||
" body=json.dumps(\n",
|
||||
" {\n",
|
||||
" \"model\": \"\",\n",
|
||||
" \"messages\": [\n",
|
||||
" {\n",
|
||||
" \"role\": \"user\",\n",
|
||||
" \"content\": user_message,\n",
|
||||
" }\n",
|
||||
" ],\n",
|
||||
" \"max_tokens\": max_tokens,\n",
|
||||
" \"temperature\": temperature,\n",
|
||||
" }\n",
|
||||
" ),\n",
|
||||
" headers={\"Content-Type\": \"application/json\"},\n",
|
||||
" use_dedicated_endpoint=use_dedicated_endpoint,\n",
|
||||
")\n",
|
||||
"print(response.json()[\"choices\"][0][\"message\"][\"content\"])\n",
|
||||
"\n",
|
||||
"# @markdown Click \"Show Code\" to see more details."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
|
||||
@@ -0,0 +1,373 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "7d9bbf86da5e"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Copyright 2025 Google LLC\n",
|
||||
"#\n",
|
||||
"# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
|
||||
"# you may not use this file except in compliance with the License.\n",
|
||||
"# You may obtain a copy of the License at\n",
|
||||
"#\n",
|
||||
"# https://www.apache.org/licenses/LICENSE-2.0\n",
|
||||
"#\n",
|
||||
"# Unless required by applicable law or agreed to in writing, software\n",
|
||||
"# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
|
||||
"# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
|
||||
"# See the License for the specific language governing permissions and\n",
|
||||
"# limitations under the License."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "99c1c3fc2ca5"
|
||||
},
|
||||
"source": [
|
||||
"# Vertex AI Model Garden - Dia-1.6B\n",
|
||||
"\n",
|
||||
"<table><tbody><tr>\n",
|
||||
" <td style=\"text-align: center\">\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/workbench/instances\">\n",
|
||||
" <img alt=\"Workbench logo\" src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" width=\"32px\"><br> Run in Workbench\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td style=\"text-align: center\">\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/colab/import/https:%2F%2Fraw.githubusercontent.com%2FGoogleCloudPlatform%2Fvertex-ai-samples%2Fmain%2Fnotebooks%2Fcommunity%2Fmodel_garden%2Fmodel_garden_pytorch_dia_1_6b.ipynb\">\n",
|
||||
" <img alt=\"Google Cloud Colab Enterprise logo\" src=\"https://lh3.googleusercontent.com/JmcxdQi-qOpctIvWKgPtrzZdJJK-J3sWE1RsfjZNwshCFgE_9fULcNpuXYTilIR2hjwN\" width=\"32px\"><br> Run in Colab Enterprise\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td style=\"text-align: center\">\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_dia_1_6b.ipynb\">\n",
|
||||
" <img alt=\"GitHub logo\" src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" width=\"32px\"><br> View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
"</tr></tbody></table>"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "3de7470326a2"
|
||||
},
|
||||
"source": [
|
||||
"## Overview\n",
|
||||
"\n",
|
||||
"This notebook demonstrates deploying the pre-trained [Dia-1.6B](https://huggingface.co/nari-labs/Dia-1.6B) model on Vertex AI for online prediction.\n",
|
||||
"\n",
|
||||
"### Objective\n",
|
||||
"\n",
|
||||
"- Upload the model to [Model Registry](https://cloud.google.com/vertex-ai/docs/model-registry/introduction).\n",
|
||||
"- Deploy the model on [Endpoint](https://cloud.google.com/vertex-ai/docs/predictions/using-private-endpoints).\n",
|
||||
"- Run online predictions for text-to-speech.\n",
|
||||
"\n",
|
||||
"### File a bug\n",
|
||||
"\n",
|
||||
"File a bug on [GitHub](https://github.com/GoogleCloudPlatform/vertex-ai-samples/issues/new) if you encounter any issue with the notebook.\n",
|
||||
"\n",
|
||||
"### Costs\n",
|
||||
"\n",
|
||||
"This tutorial uses billable components of Google Cloud:\n",
|
||||
"\n",
|
||||
"* Vertex AI\n",
|
||||
"* Cloud Storage\n",
|
||||
"\n",
|
||||
"Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing), [Cloud Storage pricing](https://cloud.google.com/storage/pricing), and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "264c07757582"
|
||||
},
|
||||
"source": [
|
||||
"## Run the notebook"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "ioensNKM8ned"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Setup Google Cloud project\n",
|
||||
"\n",
|
||||
"# @markdown 1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
|
||||
"\n",
|
||||
"# @markdown 2. **[Optional]** Set region. If not set, the region will be set automatically according to Colab Enterprise environment.\n",
|
||||
"\n",
|
||||
"REGION = \"\" # @param {type:\"string\"}\n",
|
||||
"\n",
|
||||
"# @markdown 3. If you want to run predictions with A100 80GB or H100 GPUs, we recommend using the regions listed below. **NOTE:** Make sure you have associated quota in selected regions. Click the links to see your current quota for each GPU type: [Nvidia A100 80GB](https://console.cloud.google.com/iam-admin/quotas?metric=aiplatform.googleapis.com%2Fcustom_model_serving_nvidia_a100_80gb_gpus), [Nvidia H100 80GB](https://console.cloud.google.com/iam-admin/quotas?metric=aiplatform.googleapis.com%2Fcustom_model_serving_nvidia_h100_gpus). You can request for quota following the instructions at [\"Request a higher quota\"](https://cloud.google.com/docs/quota/view-manage#requesting_higher_quota).\n",
|
||||
"\n",
|
||||
"# @markdown > | Machine Type | Accelerator Type | Recommended Regions |\n",
|
||||
"# @markdown | ----------- | ----------- | ----------- |\n",
|
||||
"# @markdown | a2-ultragpu-1g | 1 NVIDIA_A100_80GB | us-central1, us-east4, europe-west4, asia-southeast1, us-east4 |\n",
|
||||
"# @markdown | a3-highgpu-2g | 2 NVIDIA_H100_80GB | us-west1, asia-southeast1, europe-west4 |\n",
|
||||
"# @markdown | a3-highgpu-4g | 4 NVIDIA_H100_80GB | us-west1, asia-southeast1, europe-west4 |\n",
|
||||
"# @markdown | a3-highgpu-8g | 8 NVIDIA_H100_80GB | us-central1, europe-west4, us-west1, asia-southeast1 |\n",
|
||||
"\n",
|
||||
"# Upgrade Vertex AI SDK.\n",
|
||||
"! pip3 install --upgrade --quiet 'google-cloud-aiplatform>=1.84.0'\n",
|
||||
"\n",
|
||||
"import importlib\n",
|
||||
"import os\n",
|
||||
"\n",
|
||||
"from google.cloud import aiplatform\n",
|
||||
"\n",
|
||||
"if os.environ.get(\"VERTEX_PRODUCT\") != \"COLAB_ENTERPRISE\":\n",
|
||||
" ! pip install --upgrade tensorflow\n",
|
||||
"! git clone https://github.com/GoogleCloudPlatform/vertex-ai-samples.git\n",
|
||||
"\n",
|
||||
"common_util = importlib.import_module(\n",
|
||||
" \"vertex-ai-samples.community-content.vertex_model_garden.model_oss.notebook_util.common_util\"\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"models, endpoints = {}, {}\n",
|
||||
"LABEL = \"text-to-speech\"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Get the default cloud project id.\n",
|
||||
"PROJECT_ID = os.environ[\"GOOGLE_CLOUD_PROJECT\"]\n",
|
||||
"\n",
|
||||
"# Get the default region for launching jobs.\n",
|
||||
"if not REGION:\n",
|
||||
" REGION = os.environ[\"GOOGLE_CLOUD_REGION\"]\n",
|
||||
"\n",
|
||||
"# Initialize Vertex AI API.\n",
|
||||
"print(\"Initializing Vertex AI API.\")\n",
|
||||
"aiplatform.init(project=PROJECT_ID, location=REGION)\n",
|
||||
"\n",
|
||||
"! gcloud config set project $PROJECT_ID\n",
|
||||
"import vertexai\n",
|
||||
"\n",
|
||||
"vertexai.init(\n",
|
||||
" project=PROJECT_ID,\n",
|
||||
" location=REGION,\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "2707b02ef5df"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Set the model parameters\n",
|
||||
"\n",
|
||||
"MODEL_ID = \"nari-labs/Dia-1.6B\"\n",
|
||||
"TASK = \"text-to-speech\"\n",
|
||||
"PUBLISHER_MODEL_NAME = \"publishers/nari-labs/models/dia-1.6b@dia-1.6b\"\n",
|
||||
"\n",
|
||||
"machine_type = \"g2-standard-12\"\n",
|
||||
"accelerator_type = \"NVIDIA_L4\"\n",
|
||||
"accelerator_count = 1"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "qZ5z-4hzRzL8"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title [Option 1] Deploy with Model Garden SDK\n",
|
||||
"\n",
|
||||
"# @markdown Set use_dedicated_endpoint to False if you don't want to use [dedicated endpoint](https://cloud.google.com/vertex-ai/docs/general/deployment#create-dedicated-endpoint). Note that [dedicated endpoint does not support VPC Service Controls](https://cloud.google.com/vertex-ai/docs/predictions/choose-endpoint-type), uncheck the box if you are using VPC-SC.\n",
|
||||
"use_dedicated_endpoint = True # @param {type:\"boolean\"}\n",
|
||||
"# @markdown Deploy with Gen AI model-centric SDK. This section uploads the prebuilt model to Model Registry and deploys it to a Vertex AI Endpoint. It takes 15 minutes to 1 hour to finish depending on the size of the model. See [use open models with Vertex AI](https://cloud.google.com/vertex-ai/generative-ai/docs/open-models/use-open-models) for documentation on other use cases.\n",
|
||||
"from vertexai.preview import model_garden\n",
|
||||
"\n",
|
||||
"model = model_garden.OpenModel(PUBLISHER_MODEL_NAME)\n",
|
||||
"endpoints[LABEL] = model.deploy(\n",
|
||||
" machine_type=machine_type,\n",
|
||||
" accelerator_type=accelerator_type,\n",
|
||||
" accelerator_count=accelerator_count,\n",
|
||||
" use_dedicated_endpoint=use_dedicated_endpoint,\n",
|
||||
" accept_eula=True, # Accept the End User License Agreement (EULA) on the model card before deploy. Otherwise, the deployment will be forbidden.\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "lSD2g1pYYamO"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title [Option 2] Deploy with customized configs\n",
|
||||
"\n",
|
||||
"# @markdown This section uploads the Dia-1.6B model to Model Registry and deploys it on the Endpoint with NVIDIA_L4 accelerator.\n",
|
||||
"\n",
|
||||
"# @markdown The deployment takes ~10 minutes to finish.\n",
|
||||
"\n",
|
||||
"# The pre-built serving docker image. It contains serving scripts and models.\n",
|
||||
"SERVE_DOCKER_URI = \"us-docker.pkg.dev/deeplearning-platform-release/vertex-model-garden/pytorch-dia-serve.cu125.0-1.ubuntu2204.py310\"\n",
|
||||
"\n",
|
||||
"common_util.check_quota(\n",
|
||||
" project_id=PROJECT_ID,\n",
|
||||
" region=REGION,\n",
|
||||
" accelerator_type=accelerator_type,\n",
|
||||
" accelerator_count=accelerator_count,\n",
|
||||
" is_for_training=False,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def deploy_model(\n",
|
||||
" model_id,\n",
|
||||
" task,\n",
|
||||
" machine_type,\n",
|
||||
" accelerator_type,\n",
|
||||
" accelerator_count,\n",
|
||||
" use_dedicated_endpoint,\n",
|
||||
"):\n",
|
||||
" \"\"\"Create a Vertex AI Endpoint and deploy the specified model to the endpoint.\"\"\"\n",
|
||||
"\n",
|
||||
" model_name = model_id\n",
|
||||
"\n",
|
||||
" endpoint = aiplatform.Endpoint.create(\n",
|
||||
" display_name=f\"{model_name}-endpoint\",\n",
|
||||
" dedicated_endpoint_enabled=use_dedicated_endpoint,\n",
|
||||
" )\n",
|
||||
" serving_env = {\n",
|
||||
" \"MODEL_ID\": model_id,\n",
|
||||
" \"TASK\": task,\n",
|
||||
" \"DEPLOY_SOURCE\": \"notebook\",\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" model = aiplatform.Model.upload(\n",
|
||||
" display_name=model_name,\n",
|
||||
" serving_container_image_uri=SERVE_DOCKER_URI,\n",
|
||||
" serving_container_ports=[7080],\n",
|
||||
" serving_container_predict_route=\"/predict\",\n",
|
||||
" serving_container_health_route=\"/health\",\n",
|
||||
" serving_container_environment_variables=serving_env,\n",
|
||||
" model_garden_source_model_name=PUBLISHER_MODEL_NAME,\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" model.deploy(\n",
|
||||
" endpoint=endpoint,\n",
|
||||
" machine_type=machine_type,\n",
|
||||
" accelerator_type=accelerator_type,\n",
|
||||
" accelerator_count=accelerator_count,\n",
|
||||
" deploy_request_timeout=1800,\n",
|
||||
" system_labels={\n",
|
||||
" \"NOTEBOOK_NAME\": \"model_garden_pytorch_dia_1_6b.ipynb\",\n",
|
||||
" \"NOTEBOOK_ENVIRONMENT\": common_util.get_deploy_source(),\n",
|
||||
" },\n",
|
||||
" )\n",
|
||||
" return model, endpoint\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"models[LABEL], endpoints[LABEL] = deploy_model(\n",
|
||||
" model_id=MODEL_ID,\n",
|
||||
" task=TASK,\n",
|
||||
" machine_type=machine_type,\n",
|
||||
" accelerator_type=accelerator_type,\n",
|
||||
" accelerator_count=accelerator_count,\n",
|
||||
" use_dedicated_endpoint=use_dedicated_endpoint,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"print(\"endpoint_name:\", endpoints[LABEL].name)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "bb7adab99e41"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Predict\n",
|
||||
"\n",
|
||||
"# @markdown Once deployment succeeds, you can send requests to the endpoint with text prompts.\n",
|
||||
"\n",
|
||||
"# @markdown Example:\n",
|
||||
"\n",
|
||||
"# @markdown ```\n",
|
||||
"# @markdown text: \"[S1] Dia is an open weights text to dialogue model. [S2] You get full control over scripts and voices. [S1] Wow. Amazing. (laughs)\"\n",
|
||||
"# @markdown ```\n",
|
||||
"\n",
|
||||
"# @markdown You may adjust the parameters below to achieve best audio quality.\n",
|
||||
"\n",
|
||||
"import base64\n",
|
||||
"\n",
|
||||
"from IPython import display\n",
|
||||
"\n",
|
||||
"text = \"[S1] Dia is an open weights text to dialogue model. [S2] You get full control over scripts and voices. [S1] Wow. Amazing. (laughs)\" # @param {type: \"string\"}\n",
|
||||
"cfg_scale = 0.3 # @param {type:\"number\"}\n",
|
||||
"temperature = 1.3 # @param {type:\"number\"}\n",
|
||||
"top_p = 0.95 # @param {type:\"number\"}\n",
|
||||
"\n",
|
||||
"instances = [{\"text\": text}]\n",
|
||||
"parameters = {\n",
|
||||
" \"temperature\": temperature,\n",
|
||||
" \"top_p\": top_p,\n",
|
||||
" \"cfg_scale\": cfg_scale,\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"# The default num inference steps is set to 4 in the serving container, but\n",
|
||||
"# you can change it to your own preference for image quality in the request.\n",
|
||||
"response = endpoints[LABEL].predict(\n",
|
||||
" instances=instances,\n",
|
||||
" parameters=parameters,\n",
|
||||
" use_dedicated_endpoint=use_dedicated_endpoint,\n",
|
||||
")\n",
|
||||
"base64_audio = response.predictions[0][\"audio\"]\n",
|
||||
"display.Audio(base64.b64decode(base64_audio), rate=44100)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "6c460088b873"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Clean up resources\n",
|
||||
"# @markdown Delete the experiment models and endpoints to recycle the resources\n",
|
||||
"# @markdown and avoid unnecessary continuous charges that may incur.\n",
|
||||
"\n",
|
||||
"# Undeploy model and delete endpoint.\n",
|
||||
"for endpoint in endpoints.values():\n",
|
||||
" endpoint.delete(force=True)\n",
|
||||
"\n",
|
||||
"# Delete models.\n",
|
||||
"for model in models.values():\n",
|
||||
" model.delete()"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"name": "model_garden_pytorch_dia_1_6b.ipynb",
|
||||
"toc_visible": true
|
||||
},
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"name": "python3"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 0
|
||||
}
|
||||
@@ -29,28 +29,28 @@
|
||||
"id": "2bd716bf3e39"
|
||||
},
|
||||
"source": [
|
||||
"# Vertex AI Model Garden - Dolly V2\n",
|
||||
"# Vertex AI Model Garden - Dolly V2 [Deprecated]\n",
|
||||
"\n",
|
||||
"<table align=\"left\">\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_dolly_v2.ipynb\">\n",
|
||||
" <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_dolly_v2.ipynb\">\n",
|
||||
" <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
|
||||
"\u003ctable align=\"left\"\u003e\n",
|
||||
" \u003ctd\u003e\n",
|
||||
" \u003ca href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_dolly_v2.ipynb\"\u003e\n",
|
||||
" \u003cimg src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"\u003e Run in Colab\n",
|
||||
" \u003c/a\u003e\n",
|
||||
" \u003c/td\u003e\n",
|
||||
" \u003ctd\u003e\n",
|
||||
" \u003ca href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_dolly_v2.ipynb\"\u003e\n",
|
||||
" \u003cimg src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\"\u003e\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/notebooks/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/model_garden/model_garden_pytorch_dolly_v2.ipynb\">\n",
|
||||
" <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
|
||||
" \u003c/a\u003e\n",
|
||||
" \u003c/td\u003e\n",
|
||||
" \u003ctd\u003e\n",
|
||||
" \u003ca href=\"https://console.cloud.google.com/vertex-ai/notebooks/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/model_garden/model_garden_pytorch_dolly_v2.ipynb\"\u003e\n",
|
||||
" \u003cimg src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\"\u003e\n",
|
||||
"Open in Vertex AI Workbench\n",
|
||||
" </a>\n",
|
||||
" \u003c/a\u003e\n",
|
||||
" (a Python-3 CPU notebook is recommended)\n",
|
||||
" </td>\n",
|
||||
"</table>"
|
||||
" \u003c/td\u003e\n",
|
||||
"\u003c/table\u003e"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -61,7 +61,7 @@
|
||||
"source": [
|
||||
"## Overview\n",
|
||||
"\n",
|
||||
"This notebook demonstrates downloading and deploying the pre-trained [Dolly-V2-3b](https://huggingface.co/databricks/dolly-v2-3b), [Dolly-V2-7b](https://huggingface.co/databricks/dolly-v2-7b) & [Dolly-V2-12b](https://huggingface.co/databricks/dolly-v2-12b) model on Vertex AI for online prediction. It also demonstrates the evaluation of popular benchmark datasets through Vertex CustomJobs using [EleutherAI’s evaluation harness](https://github.com/EleutherAI/lm-evaluation-harness).\n",
|
||||
"This notebook demonstrates downloading and deploying the pre-trained [Dolly-V2-3b](https://huggingface.co/databricks/dolly-v2-3b), [Dolly-V2-7b](https://huggingface.co/databricks/dolly-v2-7b) \u0026 [Dolly-V2-12b](https://huggingface.co/databricks/dolly-v2-12b) model on Vertex AI for online prediction. It also demonstrates the evaluation of popular benchmark datasets through Vertex CustomJobs using [EleutherAI’s evaluation harness](https://github.com/EleutherAI/lm-evaluation-harness).\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"### Objective\n",
|
||||
@@ -322,7 +322,7 @@
|
||||
" return model, endpoint\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def get_job_name_with_datetime(prefix: str) -> str:\n",
|
||||
"def get_job_name_with_datetime(prefix: str) -\u003e str:\n",
|
||||
" \"\"\"Gets the job name with date time when triggering training or deployment\n",
|
||||
" jobs in Vertex AI.\n",
|
||||
" \"\"\"\n",
|
||||
@@ -463,7 +463,7 @@
|
||||
"if \"12b\" in MODEL_ID:\n",
|
||||
" accelerator_count = 2 # for dolly-v2-12b\n",
|
||||
"else:\n",
|
||||
" accelerator_count = 1 # for dolly-v2-3b & dolly-v2-7b\n",
|
||||
" accelerator_count = 1 # for dolly-v2-3b \u0026 dolly-v2-7b\n",
|
||||
"\n",
|
||||
"replica_count = 1"
|
||||
]
|
||||
|
||||
@@ -0,0 +1,366 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "7d9bbf86da5e"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Copyright 2025 Google LLC\n",
|
||||
"#\n",
|
||||
"# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
|
||||
"# you may not use this file except in compliance with the License.\n",
|
||||
"# You may obtain a copy of the License at\n",
|
||||
"#\n",
|
||||
"# https://www.apache.org/licenses/LICENSE-2.0\n",
|
||||
"#\n",
|
||||
"# Unless required by applicable law or agreed to in writing, software\n",
|
||||
"# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
|
||||
"# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
|
||||
"# See the License for the specific language governing permissions and\n",
|
||||
"# limitations under the License."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "99c1c3fc2ca5"
|
||||
},
|
||||
"source": [
|
||||
"# Vertex AI Model Garden - HiDream-I1\n",
|
||||
"\n",
|
||||
"<table><tbody><tr>\n",
|
||||
" <td style=\"text-align: center\">\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/workbench/instances\">\n",
|
||||
" <img alt=\"Workbench logo\" src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" width=\"32px\"><br> Run in Workbench\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td style=\"text-align: center\">\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/colab/import/https:%2F%2Fraw.githubusercontent.com%2FGoogleCloudPlatform%2Fvertex-ai-samples%2Fmain%2Fnotebooks%2Fcommunity%2Fmodel_garden%2Fmodel_garden_pytorch_hidream_i1.ipynb\">\n",
|
||||
" <img alt=\"Google Cloud Colab Enterprise logo\" src=\"https://lh3.googleusercontent.com/JmcxdQi-qOpctIvWKgPtrzZdJJK-J3sWE1RsfjZNwshCFgE_9fULcNpuXYTilIR2hjwN\" width=\"32px\"><br> Run in Colab Enterprise\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td style=\"text-align: center\">\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_hidream_i1.ipynb\">\n",
|
||||
" <img alt=\"GitHub logo\" src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" width=\"32px\"><br> View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
"</tr></tbody></table>"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "3de7470326a2"
|
||||
},
|
||||
"source": [
|
||||
"## Overview\n",
|
||||
"\n",
|
||||
"This notebook demonstrates deploying the pre-trained [HiDream-I1](https://huggingface.co/collections/HiDream-ai/hidream-i1-67f3e90dd509fed088a158b3) models on Vertex AI for online prediction.\n",
|
||||
"\n",
|
||||
"### Objective\n",
|
||||
"\n",
|
||||
"- Upload the model to [Model Registry](https://cloud.google.com/vertex-ai/docs/model-registry/introduction).\n",
|
||||
"- Deploy the model on [Endpoint](https://cloud.google.com/vertex-ai/docs/predictions/using-private-endpoints).\n",
|
||||
"- Run online predictions for text-to-image.\n",
|
||||
"\n",
|
||||
"### File a bug\n",
|
||||
"\n",
|
||||
"File a bug on [GitHub](https://github.com/GoogleCloudPlatform/vertex-ai-samples/issues/new) if you encounter any issue with the notebook.\n",
|
||||
"\n",
|
||||
"### Costs\n",
|
||||
"\n",
|
||||
"This tutorial uses billable components of Google Cloud:\n",
|
||||
"\n",
|
||||
"* Vertex AI\n",
|
||||
"* Cloud Storage\n",
|
||||
"\n",
|
||||
"Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing), [Cloud Storage pricing](https://cloud.google.com/storage/pricing), and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "264c07757582"
|
||||
},
|
||||
"source": [
|
||||
"## Run the notebook"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "ioensNKM8ned"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Setup Google Cloud project\n",
|
||||
"\n",
|
||||
"# @markdown 1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
|
||||
"\n",
|
||||
"# @markdown 2. **[Optional]** Set region. If not set, the region will be set automatically according to Colab Enterprise environment.\n",
|
||||
"\n",
|
||||
"REGION = \"\" # @param {type:\"string\"}\n",
|
||||
"\n",
|
||||
"# @markdown 3. If you want to run predictions with A100 80GB or H100 GPUs, we recommend using the regions listed below. **NOTE:** Make sure you have associated quota in selected regions. Click the links to see your current quota for each GPU type: [Nvidia A100 80GB](https://console.cloud.google.com/iam-admin/quotas?metric=aiplatform.googleapis.com%2Fcustom_model_serving_nvidia_a100_80gb_gpus), [Nvidia H100 80GB](https://console.cloud.google.com/iam-admin/quotas?metric=aiplatform.googleapis.com%2Fcustom_model_serving_nvidia_h100_gpus). You can request for quota following the instructions at [\"Request a higher quota\"](https://cloud.google.com/docs/quota/view-manage#requesting_higher_quota).\n",
|
||||
"\n",
|
||||
"# @markdown > | Machine Type | Accelerator Type | Recommended Regions |\n",
|
||||
"# @markdown | ----------- | ----------- | ----------- |\n",
|
||||
"# @markdown | a2-ultragpu-1g | 1 NVIDIA_A100_80GB | us-central1, us-east4, europe-west4, asia-southeast1, us-east4 |\n",
|
||||
"# @markdown | a3-highgpu-2g | 2 NVIDIA_H100_80GB | us-west1, asia-southeast1, europe-west4 |\n",
|
||||
"# @markdown | a3-highgpu-4g | 4 NVIDIA_H100_80GB | us-west1, asia-southeast1, europe-west4 |\n",
|
||||
"# @markdown | a3-highgpu-8g | 8 NVIDIA_H100_80GB | us-central1, europe-west4, us-west1, asia-southeast1 |\n",
|
||||
"\n",
|
||||
"# Upgrade Vertex AI SDK.\n",
|
||||
"! pip3 install --upgrade --quiet 'google-cloud-aiplatform>=1.84.0'\n",
|
||||
"\n",
|
||||
"import importlib\n",
|
||||
"import os\n",
|
||||
"\n",
|
||||
"from google.cloud import aiplatform\n",
|
||||
"\n",
|
||||
"if os.environ.get(\"VERTEX_PRODUCT\") != \"COLAB_ENTERPRISE\":\n",
|
||||
" ! pip install --upgrade tensorflow\n",
|
||||
"! git clone https://github.com/GoogleCloudPlatform/vertex-ai-samples.git\n",
|
||||
"\n",
|
||||
"common_util = importlib.import_module(\n",
|
||||
" \"vertex-ai-samples.community-content.vertex_model_garden.model_oss.notebook_util.common_util\"\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"models, endpoints = {}, {}\n",
|
||||
"LABEL = \"text-to-image-hidream\"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Get the default cloud project id.\n",
|
||||
"PROJECT_ID = os.environ[\"GOOGLE_CLOUD_PROJECT\"]\n",
|
||||
"\n",
|
||||
"# Get the default region for launching jobs.\n",
|
||||
"if not REGION:\n",
|
||||
" REGION = os.environ[\"GOOGLE_CLOUD_REGION\"]\n",
|
||||
"\n",
|
||||
"# Initialize Vertex AI API.\n",
|
||||
"print(\"Initializing Vertex AI API.\")\n",
|
||||
"aiplatform.init(project=PROJECT_ID, location=REGION)\n",
|
||||
"\n",
|
||||
"! gcloud config set project $PROJECT_ID\n",
|
||||
"import vertexai\n",
|
||||
"\n",
|
||||
"vertexai.init(\n",
|
||||
" project=PROJECT_ID,\n",
|
||||
" location=REGION,\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "2707b02ef5df"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Set the model parameters\n",
|
||||
"\n",
|
||||
"MODEL_ID = \"HiDream-ai/HiDream-I1-Full\" # @param [\"HiDream-ai/HiDream-I1-Full\", \"HiDream-ai/HiDream-I1-Dev\", \"HiDream-ai/HiDream-I1-Fast\"]\n",
|
||||
"TASK = \"text-to-image-hidream\"\n",
|
||||
"\n",
|
||||
"model_version = MODEL_ID.split(\"/\")[-1].lower()\n",
|
||||
"PUBLISHER_MODEL_NAME = f\"publishers/hidream-i1/models/hidream-i1-full@{model_version}\"\n",
|
||||
"\n",
|
||||
"ACCELERATOR_TYPE = \"NVIDIA_A100_80GB\" # @param [\"NVIDIA_A100_80GB\", \"NVIDIA_H100_80GB\"]\n",
|
||||
"\n",
|
||||
"if ACCELERATOR_TYPE == \"NVIDIA_A100_80GB\":\n",
|
||||
" machine_type = \"a2-ultragpu-1g\"\n",
|
||||
" accelerator_count = 1\n",
|
||||
"elif ACCELERATOR_TYPE == \"NVIDIA_H100_80GB\":\n",
|
||||
" machine_type = \"a3-highgpu-2g\"\n",
|
||||
" accelerator_count = 2\n",
|
||||
"else:\n",
|
||||
" raise ValueError(f\"Unsupported accelerator type: {ACCELERATOR_TYPE}\")\n",
|
||||
"accelerator_type = ACCELERATOR_TYPE"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "lSD2g1pYYamO"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Deploy to Vertex AI\n",
|
||||
"\n",
|
||||
"# @markdown This section uploads the HiDream-I1 model to Model Registry and deploys it on the Endpoint with selected accelerator type.\n",
|
||||
"\n",
|
||||
"# @markdown The deployment takes ~25 minutes to finish.\n",
|
||||
"\n",
|
||||
"# @markdown Set use_dedicated_endpoint to False if you don't want to use [dedicated endpoint](https://cloud.google.com/vertex-ai/docs/general/deployment#create-dedicated-endpoint). Note that [dedicated endpoint does not support VPC Service Controls](https://cloud.google.com/vertex-ai/docs/predictions/choose-endpoint-type), uncheck the box if you are using VPC-SC.\n",
|
||||
"use_dedicated_endpoint = True # @param {type:\"boolean\"}\n",
|
||||
"\n",
|
||||
"# The pre-built serving docker image. It contains serving scripts and models.\n",
|
||||
"SERVE_DOCKER_URI = \"us-docker.pkg.dev/deeplearning-platform-release/vertex-model-garden/pytorch-inference.cu125.0-4.ubuntu2204.py310\"\n",
|
||||
"\n",
|
||||
"common_util.check_quota(\n",
|
||||
" project_id=PROJECT_ID,\n",
|
||||
" region=REGION,\n",
|
||||
" accelerator_type=accelerator_type,\n",
|
||||
" accelerator_count=accelerator_count,\n",
|
||||
" is_for_training=False,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def deploy_model(\n",
|
||||
" model_id,\n",
|
||||
" task,\n",
|
||||
" machine_type,\n",
|
||||
" accelerator_type,\n",
|
||||
" accelerator_count,\n",
|
||||
" use_dedicated_endpoint,\n",
|
||||
"):\n",
|
||||
" \"\"\"Create a Vertex AI Endpoint and deploy the specified model to the endpoint.\"\"\"\n",
|
||||
"\n",
|
||||
" model_name = model_id\n",
|
||||
"\n",
|
||||
" endpoint = aiplatform.Endpoint.create(\n",
|
||||
" display_name=f\"{model_name}-endpoint\",\n",
|
||||
" dedicated_endpoint_enabled=use_dedicated_endpoint,\n",
|
||||
" )\n",
|
||||
" serving_env = {\n",
|
||||
" \"MODEL_ID\": model_id,\n",
|
||||
" \"TASK\": task,\n",
|
||||
" \"DEPLOY_SOURCE\": \"notebook\",\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" model = aiplatform.Model.upload(\n",
|
||||
" display_name=model_name,\n",
|
||||
" serving_container_image_uri=SERVE_DOCKER_URI,\n",
|
||||
" serving_container_ports=[7080],\n",
|
||||
" serving_container_predict_route=\"/predict\",\n",
|
||||
" serving_container_health_route=\"/health\",\n",
|
||||
" serving_container_environment_variables=serving_env,\n",
|
||||
" model_garden_source_model_name=PUBLISHER_MODEL_NAME,\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" model.deploy(\n",
|
||||
" endpoint=endpoint,\n",
|
||||
" machine_type=machine_type,\n",
|
||||
" accelerator_type=accelerator_type,\n",
|
||||
" accelerator_count=accelerator_count,\n",
|
||||
" deploy_request_timeout=1800,\n",
|
||||
" system_labels={\n",
|
||||
" \"NOTEBOOK_NAME\": \"model_garden_pytorch_hidream_i1.ipynb\",\n",
|
||||
" \"NOTEBOOK_ENVIRONMENT\": common_util.get_deploy_source(),\n",
|
||||
" },\n",
|
||||
" )\n",
|
||||
" return model, endpoint\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"models[LABEL], endpoints[LABEL] = deploy_model(\n",
|
||||
" model_id=MODEL_ID,\n",
|
||||
" task=TASK,\n",
|
||||
" machine_type=machine_type,\n",
|
||||
" accelerator_type=accelerator_type,\n",
|
||||
" accelerator_count=accelerator_count,\n",
|
||||
" use_dedicated_endpoint=use_dedicated_endpoint,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"print(\"endpoint_name:\", endpoints[LABEL].name)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "bb7adab99e41"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Predict\n",
|
||||
"\n",
|
||||
"# @markdown Once deployment succeeds, you can send requests to the endpoint with text prompts.\n",
|
||||
"\n",
|
||||
"# @markdown Example:\n",
|
||||
"\n",
|
||||
"# @markdown ```\n",
|
||||
"# @markdown text: \"A cat holding a sign that says hello world\"\n",
|
||||
"# @markdown ```\n",
|
||||
"\n",
|
||||
"# @markdown Recommended parameters:\n",
|
||||
"# @markdown - HiDream-I1-Full: num_inference_steps=50, guidance_scale=5.0\n",
|
||||
"# @markdown - HiDream-I1-Dev: num_inference_steps=28, guidance_scale=0.0\n",
|
||||
"# @markdown - HiDream-I1-Fast: num_inference_steps=16, guidance_scale=0.0\n",
|
||||
"\n",
|
||||
"# @markdown You may adjust the parameters below to achieve best image quality.\n",
|
||||
"\n",
|
||||
"text = \"A cat holding a sign that says hello world\" # @param {type: \"string\"}\n",
|
||||
"height = 1024 # @param {type:\"number\"}\n",
|
||||
"width = 1024 # @param {type:\"number\"}\n",
|
||||
"num_inference_steps = 50 # @param {type:\"number\"}\n",
|
||||
"guidance_scale = 5.0 # @param {type:\"number\"}\n",
|
||||
"\n",
|
||||
"instances = [{\"text\": text}]\n",
|
||||
"parameters = {\n",
|
||||
" \"height\": height,\n",
|
||||
" \"width\": width,\n",
|
||||
" \"num_inference_steps\": num_inference_steps,\n",
|
||||
" \"guidance_scale\": guidance_scale,\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"# The default num inference steps is set to 4 in the serving container, but\n",
|
||||
"# you can change it to your own preference for image quality in the request.\n",
|
||||
"response = endpoints[LABEL].predict(\n",
|
||||
" instances=instances,\n",
|
||||
" parameters=parameters,\n",
|
||||
" use_dedicated_endpoint=use_dedicated_endpoint,\n",
|
||||
")\n",
|
||||
"images = [\n",
|
||||
" common_util.base64_to_image(prediction.get(\"output\"))\n",
|
||||
" for prediction in response.predictions\n",
|
||||
"]\n",
|
||||
"common_util.image_grid(images, rows=1)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "6c460088b873"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Clean up resources\n",
|
||||
"# @markdown Delete the experiment models and endpoints to recycle the resources\n",
|
||||
"# @markdown and avoid unnecessary continuous charges that may incur.\n",
|
||||
"\n",
|
||||
"# Undeploy model and delete endpoint.\n",
|
||||
"for endpoint in endpoints.values():\n",
|
||||
" endpoint.delete(force=True)\n",
|
||||
"\n",
|
||||
"# Delete models.\n",
|
||||
"for model in models.values():\n",
|
||||
" model.delete()"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"name": "model_garden_pytorch_hidream_i1.ipynb",
|
||||
"toc_visible": true
|
||||
},
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"name": "python3"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 0
|
||||
}
|
||||
@@ -178,7 +178,10 @@
|
||||
"# The machine and accelerator specs for model deployment.\n",
|
||||
"accelerator_type = \"NVIDIA_L4\"\n",
|
||||
"machine_type = \"g2-standard-12\"\n",
|
||||
"accelerator_count = 1"
|
||||
"accelerator_count = 1\n",
|
||||
"\n",
|
||||
"# @markdown Set use_dedicated_endpoint to False if you don't want to use [dedicated endpoint](https://cloud.google.com/vertex-ai/docs/general/deployment#create-dedicated-endpoint). Note that [dedicated endpoint does not support VPC Service Controls](https://cloud.google.com/vertex-ai/docs/predictions/choose-endpoint-type), uncheck the box if you are using VPC-SC.\n",
|
||||
"use_dedicated_endpoint = True # @param {type:\"boolean\"}"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -200,6 +203,7 @@
|
||||
" machine_type=machine_type,\n",
|
||||
" accelerator_type=accelerator_type,\n",
|
||||
" accelerator_count=accelerator_count,\n",
|
||||
" use_dedicated_endpoint=use_dedicated_endpoint,\n",
|
||||
" accept_eula=True, # Accept the End User License Agreement (EULA) on the model card before deploy. Otherwise, the deployment will be forbidden.\n",
|
||||
")"
|
||||
]
|
||||
@@ -226,9 +230,13 @@
|
||||
" machine_type: str,\n",
|
||||
" accelerator_type: str,\n",
|
||||
" accelerator_count: int,\n",
|
||||
" use_dedicated_endpoint: bool = False,\n",
|
||||
"):\n",
|
||||
" model_name = \"instruct-pix2pix\"\n",
|
||||
" endpoint = aiplatform.Endpoint.create(display_name=f\"{model_name}-endpoint\")\n",
|
||||
" endpoint = aiplatform.Endpoint.create(\n",
|
||||
" display_name=f\"{model_name}-endpoint\",\n",
|
||||
" dedicated_endpoint_enabled=use_dedicated_endpoint,\n",
|
||||
" )\n",
|
||||
" serving_env = {\n",
|
||||
" \"MODEL_ID\": model_id,\n",
|
||||
" \"TASK\": task,\n",
|
||||
@@ -272,6 +280,7 @@
|
||||
" machine_type=machine_type,\n",
|
||||
" accelerator_type=accelerator_type,\n",
|
||||
" accelerator_count=accelerator_count,\n",
|
||||
" use_dedicated_endpoint=use_dedicated_endpoint,\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
@@ -302,7 +311,9 @@
|
||||
" \"num_inference_steps\": num_inference_steps,\n",
|
||||
" },\n",
|
||||
"]\n",
|
||||
"response = endpoints[LABEL].predict(instances=instances)\n",
|
||||
"response = endpoints[LABEL].predict(\n",
|
||||
" instances=instances, use_dedicated_endpoint=use_dedicated_endpoint\n",
|
||||
")\n",
|
||||
"images = [common_util.base64_to_image(image) for image in response.predictions]\n",
|
||||
"common_util.image_grid([init_image, images[0]], rows=1, cols=2)"
|
||||
]
|
||||
|
||||
@@ -380,9 +380,7 @@
|
||||
"models[\"vllm_fast\"], endpoints[\"vllm_fast\"] = fast_deploy(\n",
|
||||
" \"meta\", \"llama3_1\", \"llama-3.1-8b-instruct\"\n",
|
||||
")\n",
|
||||
"ENDPOINT_RESOURCE_NAME = \"projects/{}/locations/{}/endpoints/{}\".format(\n",
|
||||
" PROJECT_ID, REGION, endpoints[\"vllm_fast\"].name\n",
|
||||
")\n",
|
||||
"ENDPOINT_RESOURCE_NAME = endpoints[\"vllm_fast\"].resource_name\n",
|
||||
"BASE_URL = (\n",
|
||||
" f\"https://{REGION}-aiplatform.googleapis.com/v1beta1/{ENDPOINT_RESOURCE_NAME}\"\n",
|
||||
")\n",
|
||||
|
||||
@@ -111,7 +111,6 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"language": "python",
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "YXFGIp1l-qtT"
|
||||
@@ -778,10 +777,8 @@
|
||||
"REGION = FAST_DEPLOYMENT_REGION\n",
|
||||
"\n",
|
||||
"if use_dedicated_endpoint:\n",
|
||||
"DEDICATED_ENDPOINT_DNS = endpoints[\"vllm_fast\"].gca_resource.dedicated_endpoint_dns\n",
|
||||
"ENDPOINT_RESOURCE_NAME = \"projects/{}/locations/{}/endpoints/{}\".format(\n",
|
||||
" PROJECT_ID, REGION, endpoints[\"vllm_fast\"].name\n",
|
||||
")\n",
|
||||
" DEDICATED_ENDPOINT_DNS = endpoints[\"vllm_fast\"].gca_resource.dedicated_endpoint_dns\n",
|
||||
" ENDPOINT_RESOURCE_NAME = endpoints[\"vllm_fast\"].resource_name\n",
|
||||
"\n",
|
||||
"# @title Chat Completions Inference\n",
|
||||
"\n",
|
||||
@@ -1070,8 +1067,12 @@
|
||||
" vllm_args.append(f\"--model-type={model_type}\")\n",
|
||||
"\n",
|
||||
" if enable_llama_tool_parser:\n",
|
||||
" vllm_args.append(\"--enable-auto-tool-choice\")\n",
|
||||
" vllm_args.append(\"--tool-call-parser=vertex-llama-3\")\n",
|
||||
" if \"Llama-4\" not in model_id:\n",
|
||||
" vllm_args.append(\"--enable-auto-tool-choice\")\n",
|
||||
" vllm_args.append(\"--tool-call-parser=vertex-llama-3\")\n",
|
||||
" else:\n",
|
||||
" vllm_args.append(\"--enable-auto-tool-choice\")\n",
|
||||
" vllm_args.append(\"--tool-call-parser=llama3_json\")\n",
|
||||
"\n",
|
||||
" env_vars = {\n",
|
||||
" \"MODEL_ID\": base_model_id,\n",
|
||||
@@ -1263,9 +1264,7 @@
|
||||
"\n",
|
||||
"if use_dedicated_endpoint:\n",
|
||||
" DEDICATED_ENDPOINT_DNS = endpoints[LABEL].gca_resource.dedicated_endpoint_dns\n",
|
||||
"ENDPOINT_RESOURCE_NAME = \"projects/{}/locations/{}/endpoints/{}\".format(\n",
|
||||
" PROJECT_ID, REGION, endpoints[LABEL].name\n",
|
||||
")\n",
|
||||
"ENDPOINT_RESOURCE_NAME = endpoints[LABEL].resource_name\n",
|
||||
"\n",
|
||||
"# @title Chat Completions Inference\n",
|
||||
"\n",
|
||||
@@ -1564,9 +1563,7 @@
|
||||
"source": [
|
||||
"# @title Chat completion\n",
|
||||
"\n",
|
||||
"ENDPOINT_RESOURCE_NAME = \"projects/{}/locations/{}/endpoints/{}\".format(\n",
|
||||
" PROJECT_ID, REGION, endpoints[\"optimized_vllm_gpu\"].name\n",
|
||||
")\n",
|
||||
"ENDPOINT_RESOURCE_NAME = endpoints[\"optimized_vllm_gpu\"].resource_name\n",
|
||||
"\n",
|
||||
"# @title Chat Completions Inference\n",
|
||||
"\n",
|
||||
|
||||
@@ -244,7 +244,7 @@
|
||||
"# @markdown ---\n",
|
||||
"\n",
|
||||
"# @markdown ### Access Llama 3.1 models on Hugging Face for GPU based finetuning and serving\n",
|
||||
"# @markdown You must provide a Hugging Face User Access Token (read) to access the Llama 3.1 models. You can follow the [Hugging Face documentation](https://huggingface.co/docs/hub/en/security-tokens) to create a **read** access token and put it in the `HF_TOKEN` field below.\n",
|
||||
"# @markdown You must provide a Hugging Face User Access Token (with read access) to access the Llama 3.1 models. You can follow the [Hugging Face documentation](https://huggingface.co/docs/hub/en/security-tokens) to create a **read** access token and put it in the `HF_TOKEN` field below.\n",
|
||||
"\n",
|
||||
"HF_TOKEN = \"\" # @param {type:\"string\", isTemplate:true}\n",
|
||||
"# @markdown *--- Or ---*\n",
|
||||
@@ -706,6 +706,7 @@
|
||||
" eval_accelerator_type = \"NVIDIA_L4\"\n",
|
||||
" eval_accelerator_count = 2\n",
|
||||
" dws_kwargs = {}\n",
|
||||
" is_dynamic_workload_scheduler = False\n",
|
||||
"elif \"70b\" in base_model_id.lower():\n",
|
||||
" eval_machine_type = \"a2-ultragpu-4g\"\n",
|
||||
" eval_accelerator_type = \"NVIDIA_A100_80GB\"\n",
|
||||
@@ -714,6 +715,7 @@
|
||||
" \"max_wait_duration\": 1800, # 30 minutes\n",
|
||||
" \"scheduling_strategy\": gca_custom_job_compat.Scheduling.Strategy.FLEX_START,\n",
|
||||
" }\n",
|
||||
" is_dynamic_workload_scheduler = True\n",
|
||||
"elif \"405b\" in base_model_id.lower():\n",
|
||||
" print(\n",
|
||||
" \"405B model is not supported for evaluation. We will skip the evaluation job.\"\n",
|
||||
@@ -758,6 +760,7 @@
|
||||
" accelerator_type=eval_accelerator_type,\n",
|
||||
" accelerator_count=eval_accelerator_count,\n",
|
||||
" is_for_training=True,\n",
|
||||
" is_dynamic_workload_scheduler=is_dynamic_workload_scheduler,\n",
|
||||
" )\n",
|
||||
" # Pass evaluation arguments and launch job.\n",
|
||||
" lm_eval_job = aiplatform.CustomContainerTrainingJob(\n",
|
||||
|
||||
+1
-1
@@ -209,7 +209,7 @@
|
||||
"# @markdown 1. Open the [Llama 3.1 model](https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct) and [Qwen2.5](https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct) from [Hugging Face](https://huggingface.co/).\n",
|
||||
"# @markdown 2. Review and accept the agreement.\n",
|
||||
"# @markdown 3. After accepting the agreement, Llama 3.1 and Qwen2.5 models will be available for serving.\n",
|
||||
"# @markdown 4. You must provide a Hugging Face User Access Token (read) to access the models. You can follow the [Hugging Face documentation](https://huggingface.co/docs/hub/en/security-tokens) to create a **read** access token and put it in the `HF_TOKEN` field below.\n",
|
||||
"# @markdown 4. You must provide a Hugging Face User Access Token (with read access) to access the models. You can follow the [Hugging Face documentation](https://huggingface.co/docs/hub/en/security-tokens) to create a **read** access token and put it in the `HF_TOKEN` field below.\n",
|
||||
"\n",
|
||||
"HF_TOKEN = \"\" # @param {type:\"string\", isTemplate:true}\n",
|
||||
"assert HF_TOKEN, \"Provide a read HF_TOKEN to load models from Hugging Face\""
|
||||
|
||||
+1
-3
@@ -380,9 +380,7 @@
|
||||
"models[\"vllm_fast\"], endpoints[\"vllm_fast\"] = fast_deploy(\n",
|
||||
" \"meta\", \"llama3_1\", \"llama-3.1-8b-instruct\"\n",
|
||||
")\n",
|
||||
"ENDPOINT_RESOURCE_NAME = \"projects/{}/locations/{}/endpoints/{}\".format(\n",
|
||||
" PROJECT_ID, REGION, endpoints[\"vllm_fast\"].name\n",
|
||||
")\n",
|
||||
"ENDPOINT_RESOURCE_NAME = endpoints[\"vllm_fast\"].resource_name\n",
|
||||
"BASE_URL = (\n",
|
||||
" f\"https://{REGION}-aiplatform.googleapis.com/v1beta1/{ENDPOINT_RESOURCE_NAME}\"\n",
|
||||
")\n",
|
||||
|
||||
@@ -65,8 +65,7 @@
|
||||
"\n",
|
||||
"- Deploy Llama 3.2 1B and 3B with [Hex-LLM](https://cloud.google.com/vertex-ai/generative-ai/docs/open-models/use-hex-llm) on TPU.\n",
|
||||
"- Deploy Llama 3.2 1B and 3B Instruct with the Fast Deployment feature.\n",
|
||||
"- Deploy Llama 3.2 11B-vision and 90B-vision with [vLLM](https://github.com/vllm-project/vllm) on GPU with limited functions supported.\n",
|
||||
"- Deploy Llama 3.2 11B-vision and 90B-vision with reference server on GPU with full functions supported.\n",
|
||||
"- Deploy Llama 3.2 11B-vision and 90B-vision with [vLLM](https://github.com/vllm-project/vllm) on GPU.\n",
|
||||
"\n",
|
||||
"### File a bug\n",
|
||||
"\n",
|
||||
@@ -112,7 +111,6 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"language": "python",
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "YXFGIp1l-qtT"
|
||||
@@ -159,7 +157,6 @@
|
||||
" \"vertex-ai-samples.community-content.vertex_model_garden.model_oss.notebook_util.common_util\"\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"LABEL = \"vllm_gpu\"\n",
|
||||
"models, endpoints = {}, {}\n",
|
||||
"\n",
|
||||
"\n",
|
||||
@@ -780,9 +777,7 @@
|
||||
"\n",
|
||||
"if use_dedicated_endpoint:\n",
|
||||
" DEDICATED_ENDPOINT_DNS = endpoints[\"vllm_fast\"].gca_resource.dedicated_endpoint_dns\n",
|
||||
"ENDPOINT_RESOURCE_NAME = \"projects/{}/locations/{}/endpoints/{}\".format(\n",
|
||||
" PROJECT_ID, REGION, endpoints[\"vllm_fast\"].name\n",
|
||||
")\n",
|
||||
"ENDPOINT_RESOURCE_NAME = endpoints[\"vllm_fast\"].resource_name\n",
|
||||
"\n",
|
||||
"# @title Chat Completions Inference\n",
|
||||
"\n",
|
||||
@@ -865,7 +860,7 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Select the model variants\n",
|
||||
"\n",
|
||||
"LABEL = \"vllm_gpu\"\n",
|
||||
"base_model_name = \"Llama-3.2-11B-Vision-Instruct\" # @param [\"Llama-3.2-11B-Vision\", \"Llama-3.2-11B-Vision-Instruct\", \"Llama-3.2-90B-Vision\", \"Llama-3.2-90B-Vision-Instruct\"] {isTemplate:true}\n",
|
||||
"hf_model_id = \"meta-llama/\" + base_model_name\n",
|
||||
"model_id = os.path.join(VERTEX_AI_MODEL_GARDEN_LLAMA_3_2, base_model_name)\n",
|
||||
@@ -919,11 +914,17 @@
|
||||
"# @markdown Deploy with Gen AI model-centric SDK. This section uploads the prebuilt model to Model Registry and deploys it to a Vertex AI Endpoint. It takes 15 minutes to 1 hour to finish depending on the size of the model. See [use open models with Vertex AI](https://cloud.google.com/vertex-ai/generative-ai/docs/open-models/use-open-models) for documentation on other use cases.\n",
|
||||
"from vertexai.preview import model_garden\n",
|
||||
"\n",
|
||||
"if REGION == \"us-central1\":\n",
|
||||
" fast_tryout_enabled = True\n",
|
||||
"else:\n",
|
||||
" fast_tryout_enabled = False\n",
|
||||
"\n",
|
||||
"model = model_garden.OpenModel(PUBLISHER_MODEL_NAME)\n",
|
||||
"endpoints[LABEL] = model.deploy(\n",
|
||||
" machine_type=machine_type,\n",
|
||||
" accelerator_type=accelerator_type,\n",
|
||||
" accelerator_count=accelerator_count,\n",
|
||||
" fast_tryout_enabled=fast_tryout_enabled,\n",
|
||||
" use_dedicated_endpoint=use_dedicated_endpoint,\n",
|
||||
" accept_eula=True, # Accept the End User License Agreement (EULA) on the model card before deploy. Otherwise, the deployment will be forbidden.\n",
|
||||
")"
|
||||
@@ -1046,8 +1047,12 @@
|
||||
" vllm_args.append(f\"--model-type={model_type}\")\n",
|
||||
"\n",
|
||||
" if enable_llama_tool_parser:\n",
|
||||
" vllm_args.append(\"--enable-auto-tool-choice\")\n",
|
||||
" vllm_args.append(\"--tool-call-parser=vertex-llama-3\")\n",
|
||||
" if \"Llama-4\" not in model_id:\n",
|
||||
" vllm_args.append(\"--enable-auto-tool-choice\")\n",
|
||||
" vllm_args.append(\"--tool-call-parser=vertex-llama-3\")\n",
|
||||
" else:\n",
|
||||
" vllm_args.append(\"--enable-auto-tool-choice\")\n",
|
||||
" vllm_args.append(\"--tool-call-parser=llama3_json\")\n",
|
||||
"\n",
|
||||
" env_vars = {\n",
|
||||
" \"MODEL_ID\": base_model_id,\n",
|
||||
@@ -1178,9 +1183,7 @@
|
||||
"\n",
|
||||
"if use_dedicated_endpoint:\n",
|
||||
" DEDICATED_ENDPOINT_DNS = endpoints[LABEL].gca_resource.dedicated_endpoint_dns\n",
|
||||
"ENDPOINT_RESOURCE_NAME = \"projects/{}/locations/{}/endpoints/{}\".format(\n",
|
||||
" PROJECT_ID, REGION, endpoints[LABEL].name\n",
|
||||
")\n",
|
||||
"ENDPOINT_RESOURCE_NAME = endpoints[LABEL].resource_name\n",
|
||||
"\n",
|
||||
"# @title Chat Completions Inference\n",
|
||||
"\n",
|
||||
@@ -1255,9 +1258,7 @@
|
||||
"\n",
|
||||
"if use_dedicated_endpoint:\n",
|
||||
" DEDICATED_ENDPOINT_DNS = endpoints[LABEL].gca_resource.dedicated_endpoint_dns\n",
|
||||
"ENDPOINT_RESOURCE_NAME = \"projects/{}/locations/{}/endpoints/{}\".format(\n",
|
||||
" PROJECT_ID, REGION, endpoints[LABEL].name\n",
|
||||
")\n",
|
||||
"ENDPOINT_RESOURCE_NAME = endpoints[LABEL].resource_name\n",
|
||||
"\n",
|
||||
"# @title Chat Completions Inference\n",
|
||||
"\n",
|
||||
@@ -1314,256 +1315,6 @@
|
||||
"# @markdown Click \"Show Code\" to see more details."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "4AoiFsQWhYtM"
|
||||
},
|
||||
"source": [
|
||||
"## Deploy prebuilt Llama 3.2 11B-Vision and 90B-Vision with reference server"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "wXkUpLOXhYtM"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Deploy\n",
|
||||
"\n",
|
||||
"# @markdown This section uploads prebuilt Llama 3.2 models to Model Registry and deploys it to a Vertex AI Endpoint. It takes 15 minutes to 1 hour to finish depending on the size of the model.\n",
|
||||
"\n",
|
||||
"# @markdown The reference server can be used with full input formats supported.\n",
|
||||
"\n",
|
||||
"# @markdown Set the model to deploy.\n",
|
||||
"\n",
|
||||
"base_model_name = \"Llama-3.2-11B-Vision-Instruct-meta\" # @param [\"Llama-3.2-11B-Vision-meta\", \"Llama-3.2-11B-Vision-Instruct-meta\", \"Llama-3.2-90B-Vision-meta\", \"Llama-3.2-90B-Vision-Instruct-meta\"] {isTemplate:true}\n",
|
||||
"model_id = os.path.join(VERTEX_AI_MODEL_GARDEN_LLAMA_3_2, base_model_name)\n",
|
||||
"hf_model_id = \"meta-llama/\" + base_model_name\n",
|
||||
"\n",
|
||||
"# The pre-built serving docker images.\n",
|
||||
"REF_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-peft-serve:llama_ref_impl\"\n",
|
||||
"\n",
|
||||
"# @markdown Find Vertex AI prediction supported accelerators and regions at https://cloud.google.com/vertex-ai/docs/predictions/configure-compute.\n",
|
||||
"if \"11B\" in base_model_name:\n",
|
||||
" accelerator_type = \"NVIDIA_TESLA_A100\"\n",
|
||||
" machine_type = \"a2-highgpu-1g\"\n",
|
||||
" accelerator_count = 1\n",
|
||||
"elif \"90B\" in base_model_name:\n",
|
||||
" accelerator_type = \"NVIDIA_H100_80GB\"\n",
|
||||
" machine_type = \"a3-highgpu-8g\"\n",
|
||||
" accelerator_count = 8\n",
|
||||
"else:\n",
|
||||
" raise ValueError(f\"Recommended GPU setting not found for: {base_model_name}.\")\n",
|
||||
"\n",
|
||||
"common_util.check_quota(\n",
|
||||
" project_id=PROJECT_ID,\n",
|
||||
" region=REGION,\n",
|
||||
" accelerator_type=accelerator_type,\n",
|
||||
" accelerator_count=accelerator_count,\n",
|
||||
" is_for_training=False,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def deploy_model_ref(\n",
|
||||
" model_name: str,\n",
|
||||
" model_id: str,\n",
|
||||
" publisher: str,\n",
|
||||
" publisher_model_id: str,\n",
|
||||
" base_model_id: str = None,\n",
|
||||
" machine_type: str = \"g2-standard-8\",\n",
|
||||
" accelerator_type: str = \"NVIDIA_L4\",\n",
|
||||
" accelerator_count: int = 1,\n",
|
||||
" max_model_len: int = 4096,\n",
|
||||
") -> Tuple[aiplatform.Model, aiplatform.Endpoint]:\n",
|
||||
" \"\"\"Deploys models with reference server into Vertex AI.\"\"\"\n",
|
||||
" endpoint = aiplatform.Endpoint.create(\n",
|
||||
" display_name=f\"{model_name}-endpoint\",\n",
|
||||
" dedicated_endpoint_enabled=use_dedicated_endpoint,\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" if not base_model_id:\n",
|
||||
" base_model_id = model_id\n",
|
||||
"\n",
|
||||
" ref_args = [\n",
|
||||
" \"torchrun\",\n",
|
||||
" f\"--nproc_per_node={accelerator_count}\",\n",
|
||||
" \"server.py\",\n",
|
||||
" \"--port=8080\",\n",
|
||||
" f\"--model={model_id}\",\n",
|
||||
" f\"--tensor-parallel-size={accelerator_count}\",\n",
|
||||
" f\"--max-model-len={max_model_len}\",\n",
|
||||
" ]\n",
|
||||
"\n",
|
||||
" env_vars = {\n",
|
||||
" \"MODEL_ID\": base_model_id,\n",
|
||||
" \"DEPLOY_SOURCE\": \"notebook\",\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" # HF_TOKEN is not a compulsory field and may not be defined.\n",
|
||||
" try:\n",
|
||||
" if HF_TOKEN:\n",
|
||||
" env_vars[\"HF_TOKEN\"] = HF_TOKEN\n",
|
||||
" except NameError:\n",
|
||||
" pass\n",
|
||||
"\n",
|
||||
" model = aiplatform.Model.upload(\n",
|
||||
" display_name=model_name,\n",
|
||||
" serving_container_image_uri=REF_DOCKER_URI,\n",
|
||||
" serving_container_args=ref_args,\n",
|
||||
" serving_container_ports=[8080],\n",
|
||||
" serving_container_predict_route=\"/generate\",\n",
|
||||
" serving_container_health_route=\"/ping\",\n",
|
||||
" serving_container_environment_variables=env_vars,\n",
|
||||
" serving_container_shared_memory_size_mb=(16 * 1024), # 16 GB\n",
|
||||
" serving_container_deployment_timeout=7200,\n",
|
||||
" model_garden_source_model_name=(\n",
|
||||
" f\"publishers/{publisher}/models/{publisher_model_id}\"\n",
|
||||
" ),\n",
|
||||
" )\n",
|
||||
" print(\n",
|
||||
" f\"Deploying {model_name} on {machine_type} with {accelerator_count} {accelerator_type} GPU(s).\"\n",
|
||||
" )\n",
|
||||
" model.deploy(\n",
|
||||
" endpoint=endpoint,\n",
|
||||
" machine_type=machine_type,\n",
|
||||
" accelerator_type=accelerator_type,\n",
|
||||
" accelerator_count=accelerator_count,\n",
|
||||
" deploy_request_timeout=1800,\n",
|
||||
" system_labels={\n",
|
||||
" \"NOTEBOOK_NAME\": \"model_garden_pytorch_llama3_2_deployment.ipynb\",\n",
|
||||
" \"NOTEBOOK_ENVIRONMENT\": common_util.get_deploy_source(),\n",
|
||||
" },\n",
|
||||
" )\n",
|
||||
" print(\"endpoint_name:\", endpoint.name)\n",
|
||||
"\n",
|
||||
" return model, endpoint\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"max_model_len = 4096\n",
|
||||
"\n",
|
||||
"models[\"ref_gpu\"], endpoints[\"ref_gpu\"] = deploy_model_ref(\n",
|
||||
" model_name=common_util.get_job_name_with_datetime(prefix=\"llama3_2-serve-ref\"),\n",
|
||||
" model_id=model_id,\n",
|
||||
" publisher=\"meta\",\n",
|
||||
" publisher_model_id=\"llama3-2\",\n",
|
||||
" base_model_id=hf_model_id,\n",
|
||||
" machine_type=machine_type,\n",
|
||||
" accelerator_type=accelerator_type,\n",
|
||||
" accelerator_count=accelerator_count,\n",
|
||||
" max_model_len=max_model_len,\n",
|
||||
")\n",
|
||||
"# @markdown Click \"Show Code\" to see more details."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "Gl72VUchVmK5"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Chat completion for text-only models\n",
|
||||
"\n",
|
||||
"if use_dedicated_endpoint:\n",
|
||||
" DEDICATED_ENDPOINT_DNS = endpoints[\"ref_gpu\"].gca_resource.dedicated_endpoint_dns\n",
|
||||
"ENDPOINT_RESOURCE_NAME = \"projects/{}/locations/{}/endpoints/{}\".format(\n",
|
||||
" PROJECT_ID, REGION, endpoints[\"ref_gpu\"].name\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# @markdown Once deployment succeeds, you can send requests to the endpoint using the Vertex SDK.\n",
|
||||
"\n",
|
||||
"# @markdown Next fill out some request parameters:\n",
|
||||
"\n",
|
||||
"user_message = \"Tell me about large language models\" # @param {type: \"string\"}\n",
|
||||
"# @markdown If you encounter the issue like `ServiceUnavailable: 503 Took too long to respond when processing`, you can reduce the maximum number of output tokens, such as set `max_tokens` as 20.\n",
|
||||
"max_tokens = 50 # @param {type: \"integer\"}\n",
|
||||
"temperature = 1.0 # @param {type: \"number\"}\n",
|
||||
"top_p = 0.9 # @param {type: \"number\"}\n",
|
||||
"\n",
|
||||
"# @markdown Now we can send a request.\n",
|
||||
"\n",
|
||||
"instances = [\n",
|
||||
" {\n",
|
||||
" \"messages\": [{\"role\": \"user\", \"content\": user_message}],\n",
|
||||
" \"temperature\": temperature,\n",
|
||||
" \"max_tokens\": max_tokens,\n",
|
||||
" \"top_p\": top_p,\n",
|
||||
" }\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"model_response = endpoints[\"ref_gpu\"].predict(\n",
|
||||
" instances=instances, use_dedicated_endpoint=use_dedicated_endpoint\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"print(model_response.predictions[0])\n",
|
||||
"\n",
|
||||
"# @markdown Click \"Show Code\" to see more details."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "pY_hkPI2hYtM"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Chat completion for vision models\n",
|
||||
"\n",
|
||||
"if use_dedicated_endpoint:\n",
|
||||
" DEDICATED_ENDPOINT_DNS = endpoints[\"ref_gpu\"].gca_resource.dedicated_endpoint_dns\n",
|
||||
"ENDPOINT_RESOURCE_NAME = \"projects/{}/locations/{}/endpoints/{}\".format(\n",
|
||||
" PROJECT_ID, REGION, endpoints[\"ref_gpu\"].name\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# @markdown Once deployment succeeds, you can send requests to the endpoint using the Vertex SDK.\n",
|
||||
"\n",
|
||||
"# @markdown Next fill out some request parameters:\n",
|
||||
"\n",
|
||||
"user_image1 = \"https://upload.wikimedia.org/wikipedia/commons/thumb/8/84/Male_and_female_chicken_sitting_together.jpg/440px-Male_and_female_chicken_sitting_together.jpg\" # @param {type: \"string\"}\n",
|
||||
"user_image2 = \"https://upload.wikimedia.org/wikipedia/commons/thumb/9/91/Joseph_Crawhall_-_Spanish_Cock_And_Snail.jpg/337px-Joseph_Crawhall_-_Spanish_Cock_And_Snail.jpg\" # @param {type: \"string\"}\n",
|
||||
"user_message = \"How many chickens are there in these two images?\" # @param {type: \"string\"}\n",
|
||||
"# @markdown If you encounter the issue like `ServiceUnavailable: 503 Took too long to respond when processing`, you can reduce the maximum number of output tokens, such as set `max_tokens` as 20.\n",
|
||||
"max_tokens = 50 # @param {type: \"integer\"}\n",
|
||||
"temperature = 1.0 # @param {type: \"number\"}\n",
|
||||
"top_p = 0.9 # @param {type: \"number\"}\n",
|
||||
"\n",
|
||||
"# @markdown Now we can send a request.\n",
|
||||
"\n",
|
||||
"instances = [\n",
|
||||
" {\n",
|
||||
" \"messages\": [\n",
|
||||
" {\n",
|
||||
" \"role\": \"user\",\n",
|
||||
" \"content\": [\n",
|
||||
" {\"type\": \"image_url\", \"image_url\": {\"url\": user_image1}},\n",
|
||||
" {\"type\": \"image_url\", \"image_url\": {\"url\": user_image2}},\n",
|
||||
" {\"type\": \"text\", \"text\": user_message},\n",
|
||||
" ],\n",
|
||||
" },\n",
|
||||
" ],\n",
|
||||
" \"temperature\": temperature,\n",
|
||||
" \"max_tokens\": max_tokens,\n",
|
||||
" \"top_p\": top_p,\n",
|
||||
" }\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"model_response = endpoints[\"ref_gpu\"].predict(\n",
|
||||
" instances=instances, use_dedicated_endpoint=use_dedicated_endpoint\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"print(model_response.predictions[0])\n",
|
||||
"\n",
|
||||
"# @markdown Click \"Show Code\" to see more details."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
|
||||
@@ -59,12 +59,12 @@
|
||||
"source": [
|
||||
"## Overview\n",
|
||||
"\n",
|
||||
"This notebook demonstrates downloading, deploying, and serving prebuilt Llama 3.3 model with [vLLM](https://github.com/vllm-project/vllm).\n",
|
||||
"This notebook demonstrates downloading, deploying, and serving prebuilt Llama 3.3 model with [vLLM](https://github.com/vllm-project/vllm) or [TensorRT-LLM](https://github.com/NVIDIA/TensorRT-LLM).\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"### Objective\n",
|
||||
"\n",
|
||||
"- Deploy Llama 3.3 70B Instruct with vLLM on GPU, optionally with dynamic LoRA adapters.\n",
|
||||
"- Deploy Llama 3.3 70B Instruct with vLLM (optionally with dynamic LoRA adapters) or TensorRT-LLM on GPU.\n",
|
||||
"\n",
|
||||
"### File a bug\n",
|
||||
"\n",
|
||||
@@ -136,8 +136,10 @@
|
||||
"! git clone https://github.com/GoogleCloudPlatform/vertex-ai-samples.git\n",
|
||||
"\n",
|
||||
"import importlib\n",
|
||||
"import json\n",
|
||||
"import os\n",
|
||||
"import re\n",
|
||||
"import time\n",
|
||||
"from typing import Tuple\n",
|
||||
"\n",
|
||||
"import requests\n",
|
||||
@@ -217,9 +219,7 @@
|
||||
"\n",
|
||||
"# @markdown Set the model to deploy.\n",
|
||||
"\n",
|
||||
"# fmt: off\n",
|
||||
"base_model_name = \"Llama-3.3-70B-Instruct\" # @param [\"Llama-3.3-70B-Instruct\"] {isTemplate:true}\n",
|
||||
"# fmt: on\n",
|
||||
"\n",
|
||||
"model_id = os.path.join(VERTEX_AI_MODEL_GARDEN_LLAMA_3_3, base_model_name)\n",
|
||||
"ENABLE_DYNAMIC_LORA = True # @param {type:\"boolean\", isTemplate:true}\n",
|
||||
@@ -401,8 +401,12 @@
|
||||
" vllm_args.append(f\"--model-type={model_type}\")\n",
|
||||
"\n",
|
||||
" if enable_llama_tool_parser:\n",
|
||||
" vllm_args.append(\"--enable-auto-tool-choice\")\n",
|
||||
" vllm_args.append(\"--tool-call-parser=vertex-llama-3\")\n",
|
||||
" if \"Llama-4\" not in model_id:\n",
|
||||
" vllm_args.append(\"--enable-auto-tool-choice\")\n",
|
||||
" vllm_args.append(\"--tool-call-parser=vertex-llama-3\")\n",
|
||||
" else:\n",
|
||||
" vllm_args.append(\"--enable-auto-tool-choice\")\n",
|
||||
" vllm_args.append(\"--tool-call-parser=llama3_json\")\n",
|
||||
"\n",
|
||||
" env_vars = {\n",
|
||||
" \"MODEL_ID\": base_model_id,\n",
|
||||
@@ -591,9 +595,7 @@
|
||||
"\n",
|
||||
"if use_dedicated_endpoint:\n",
|
||||
" DEDICATED_ENDPOINT_DNS = endpoints[\"vllm_gpu\"].gca_resource.dedicated_endpoint_dns\n",
|
||||
"ENDPOINT_RESOURCE_NAME = \"projects/{}/locations/{}/endpoints/{}\".format(\n",
|
||||
" PROJECT_ID, REGION, endpoints[\"vllm_gpu\"].name\n",
|
||||
")\n",
|
||||
"ENDPOINT_RESOURCE_NAME = endpoints[\"vllm_gpu\"].resource_name\n",
|
||||
"\n",
|
||||
"# @title Chat Completions Inference\n",
|
||||
"\n",
|
||||
@@ -655,6 +657,284 @@
|
||||
"# @markdown Click \"Show Code\" to see more details."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "fQL54XQV1Fq2"
|
||||
},
|
||||
"source": [
|
||||
"## Deploy Llama 3.3 70B Instruct FP8 with TensorRT-LLM"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "Apv7MP2-1I1B"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Deploy\n",
|
||||
"\n",
|
||||
"# @markdown This section uploads Llama 3.3 70B Instruct FP8 model to Model Registry and deploys it to a Vertex Prediction Endpoint. It takes ~1 hour to finish.\n",
|
||||
"\n",
|
||||
"# @markdown The [Llama 3.3 70B Instruct FP8 model](https://huggingface.co/nvidia/Llama-3.3-70B-Instruct-FP8) is provided by Nvidia.\n",
|
||||
"\n",
|
||||
"# @markdown It's recommended to use the region selected by the deployment button on the model card. If the deployment button is not available, it's recommended to stay with the default region of the notebook.\n",
|
||||
"\n",
|
||||
"# @markdown Set the model to deploy.\n",
|
||||
"\n",
|
||||
"base_model_name = \"Llama-3.3-70B-Instruct-FP8\" # @param [\"Llama-3.3-70B-Instruct-FP8\"] {isTemplate:true}\n",
|
||||
"model_id = os.path.join(VERTEX_AI_MODEL_GARDEN_LLAMA_3_3, base_model_name)\n",
|
||||
"\n",
|
||||
"# The pre-built serving docker images.\n",
|
||||
"TRTLLM_DOCKER_URI = \"us-docker.pkg.dev/deeplearning-platform-release/vertex-model-garden/tensorrt-llm.cu128.0-18.ubuntu2404.py312:20250514-1800-rc0\"\n",
|
||||
"\n",
|
||||
"# @markdown Choose whether to use a [Spot VM](https://cloud.google.com/compute/docs/instances/spot) for the deployment.\n",
|
||||
"is_spot = False # @param {type:\"boolean\"}\n",
|
||||
"\n",
|
||||
"# @markdown Set use_dedicated_endpoint to False if you don't want to use [dedicated endpoint](https://cloud.google.com/vertex-ai/docs/general/deployment#create-dedicated-endpoint). Note that [dedicated endpoint does not support VPC Service Controls](https://cloud.google.com/vertex-ai/docs/predictions/choose-endpoint-type), uncheck the box if you are using VPC-SC.\n",
|
||||
"use_dedicated_endpoint = True # @param {type:\"boolean\"}\n",
|
||||
"\n",
|
||||
"accelerator_type = \"NVIDIA_H100_80GB\"\n",
|
||||
"accelerator_count = 8\n",
|
||||
"machine_type = \"a3-highgpu-8g\"\n",
|
||||
"multihost_gpu_node_count = 1\n",
|
||||
"\n",
|
||||
"common_util.check_quota(\n",
|
||||
" project_id=PROJECT_ID,\n",
|
||||
" region=REGION,\n",
|
||||
" accelerator_type=accelerator_type,\n",
|
||||
" accelerator_count=accelerator_count,\n",
|
||||
" is_for_training=False,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# 128K context length.\n",
|
||||
"MAX_MODEL_LEN = 131072\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def poll_operation(op_name: str) -> bool: # noqa: F811\n",
|
||||
" creds, _ = auth.default()\n",
|
||||
" auth_req = auth.transport.requests.Request()\n",
|
||||
" creds.refresh(auth_req)\n",
|
||||
" headers = {\n",
|
||||
" \"Authorization\": f\"Bearer {creds.token}\",\n",
|
||||
" }\n",
|
||||
" get_resp = requests.get(\n",
|
||||
" f\"https://{REGION}-aiplatform.googleapis.com/ui/{op_name}\",\n",
|
||||
" headers=headers,\n",
|
||||
" )\n",
|
||||
" opjs = get_resp.json()\n",
|
||||
" if \"error\" in opjs:\n",
|
||||
" raise ValueError(f\"Operation failed: {opjs['error']}\")\n",
|
||||
" return opjs.get(\"done\", False)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def poll_and_wait(op_name: str, total_wait: int, interval: int = 60): # noqa: F811\n",
|
||||
" waited = 0\n",
|
||||
" while not poll_operation(op_name):\n",
|
||||
" if waited > total_wait:\n",
|
||||
" raise TimeoutError(\"Operation timed out\")\n",
|
||||
" print(\n",
|
||||
" f\"\\rStill waiting for operation... Waited time in second: {waited:<6}\",\n",
|
||||
" end=\"\",\n",
|
||||
" flush=True,\n",
|
||||
" )\n",
|
||||
" waited += interval\n",
|
||||
" time.sleep(interval)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def deploy_model_tensorrt_llm_multihost(\n",
|
||||
" model_name: str,\n",
|
||||
" model_id: str,\n",
|
||||
" publisher: str,\n",
|
||||
" publisher_model_id: str,\n",
|
||||
" service_account: str = \"\",\n",
|
||||
" base_model_id: str = \"\",\n",
|
||||
" machine_type: str = \"g2-standard-8\",\n",
|
||||
" accelerator_type: str = \"NVIDIA_L4\",\n",
|
||||
" accelerator_count: int = 1,\n",
|
||||
" multihost_gpu_node_count: int = 1,\n",
|
||||
" gpu_memory_utilization: float | None = None,\n",
|
||||
" max_input_len: int | None = None,\n",
|
||||
" max_model_len: int | None = None,\n",
|
||||
" max_num_seqs: int | None = None,\n",
|
||||
" enable_trust_remote_code: bool = False,\n",
|
||||
" enable_chunked_prefill: bool = False,\n",
|
||||
" use_dedicated_endpoint: bool = False,\n",
|
||||
" is_spot: bool = True,\n",
|
||||
") -> Tuple[aiplatform.Model, aiplatform.Endpoint]:\n",
|
||||
" \"\"\"Deploys trained models with TensorRT-LLM on Vertex AI.\"\"\"\n",
|
||||
" endpoint = aiplatform.Endpoint.create(\n",
|
||||
" display_name=f\"{model_name}-endpoint\",\n",
|
||||
" dedicated_endpoint_enabled=use_dedicated_endpoint,\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" if not base_model_id:\n",
|
||||
" base_model_id = model_id\n",
|
||||
"\n",
|
||||
" container_args = [\n",
|
||||
" \"python\",\n",
|
||||
" \"api_server.py\",\n",
|
||||
" \"--host=0.0.0.0\",\n",
|
||||
" \"--port=7080\",\n",
|
||||
" f\"--model={model_id}\",\n",
|
||||
" f\"--tensor-parallel-size={accelerator_count * multihost_gpu_node_count}\",\n",
|
||||
" ]\n",
|
||||
"\n",
|
||||
" if gpu_memory_utilization:\n",
|
||||
" container_args.append(f\"--gpu-memory-utilization={gpu_memory_utilization}\")\n",
|
||||
"\n",
|
||||
" if max_input_len:\n",
|
||||
" container_args.append(f\"--max-input-len={max_input_len}\")\n",
|
||||
"\n",
|
||||
" if max_model_len:\n",
|
||||
" container_args.append(f\"--max-model-len={max_model_len}\")\n",
|
||||
"\n",
|
||||
" if max_num_seqs:\n",
|
||||
" container_args.append(f\"--max-num-seqs={max_num_seqs}\")\n",
|
||||
"\n",
|
||||
" if enable_trust_remote_code:\n",
|
||||
" container_args.append(\"--trust-remote-code=True\")\n",
|
||||
"\n",
|
||||
" if enable_chunked_prefill:\n",
|
||||
" container_args.append(\"--enable-chunked-prefill=True\")\n",
|
||||
"\n",
|
||||
" env_vars = {\n",
|
||||
" \"MODEL_ID\": base_model_id,\n",
|
||||
" \"DEPLOY_SOURCE\": \"notebook\",\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" # HF_TOKEN is not a compulsory field and may not be defined.\n",
|
||||
" try:\n",
|
||||
" if HF_TOKEN:\n",
|
||||
" env_vars[\"HF_TOKEN\"] = HF_TOKEN\n",
|
||||
" except NameError:\n",
|
||||
" pass\n",
|
||||
"\n",
|
||||
" model = aiplatform.Model.upload(\n",
|
||||
" display_name=model_name,\n",
|
||||
" serving_container_image_uri=TRTLLM_DOCKER_URI,\n",
|
||||
" serving_container_args=container_args,\n",
|
||||
" serving_container_ports=[7080],\n",
|
||||
" serving_container_predict_route=\"/v1/chat/completions\",\n",
|
||||
" serving_container_health_route=\"/health\",\n",
|
||||
" serving_container_environment_variables=env_vars,\n",
|
||||
" serving_container_shared_memory_size_mb=(16 * 1024), # 16 GB\n",
|
||||
" serving_container_deployment_timeout=7200,\n",
|
||||
" model_garden_source_model_name=(\n",
|
||||
" f\"publishers/{publisher}/models/{publisher_model_id}\"\n",
|
||||
" ),\n",
|
||||
" )\n",
|
||||
" print(\n",
|
||||
" f\"Deploying {model_name} on {machine_type} with {int(accelerator_count * multihost_gpu_node_count)} {accelerator_type} GPU(s).\"\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" creds, _ = auth.default()\n",
|
||||
" auth_req = auth.transport.requests.Request()\n",
|
||||
" creds.refresh(auth_req)\n",
|
||||
"\n",
|
||||
" url = f\"https://{REGION}-aiplatform.googleapis.com/ui/projects/{PROJECT_ID}/locations/{REGION}/endpoints/{endpoint.name}:deployModel\"\n",
|
||||
" headers = {\n",
|
||||
" \"Content-Type\": \"application/json\",\n",
|
||||
" \"Authorization\": f\"Bearer {creds.token}\",\n",
|
||||
" }\n",
|
||||
" data = {\n",
|
||||
" \"deployedModel\": {\n",
|
||||
" \"model\": model.resource_name,\n",
|
||||
" \"displayName\": model_name,\n",
|
||||
" \"dedicatedResources\": {\n",
|
||||
" \"machineSpec\": {\n",
|
||||
" \"machineType\": machine_type,\n",
|
||||
" \"multihostGpuNodeCount\": multihost_gpu_node_count,\n",
|
||||
" \"acceleratorType\": accelerator_type,\n",
|
||||
" \"acceleratorCount\": accelerator_count,\n",
|
||||
" },\n",
|
||||
" \"minReplicaCount\": 1,\n",
|
||||
" \"maxReplicaCount\": 1,\n",
|
||||
" },\n",
|
||||
" \"system_labels\": {\n",
|
||||
" \"NOTEBOOK_NAME\": \"model_garden_pytorch_llama3_3_deployment.ipynb\",\n",
|
||||
" \"NOTEBOOK_ENVIRONMENT\": common_util.get_deploy_source(),\n",
|
||||
" },\n",
|
||||
" },\n",
|
||||
" }\n",
|
||||
" if service_account:\n",
|
||||
" data[\"deployedModel\"][\"serviceAccount\"] = service_account\n",
|
||||
" if is_spot:\n",
|
||||
" data[\"deployedModel\"][\"dedicatedResources\"][\"spot\"] = True\n",
|
||||
" response = requests.post(url, headers=headers, json=data)\n",
|
||||
" print(f\"Deploy Model response: {response.json()}\")\n",
|
||||
" if response.status_code != 200 or \"name\" not in response.json():\n",
|
||||
" raise ValueError(f\"Failed to deploy model: {response.text}\")\n",
|
||||
" poll_and_wait(response.json()[\"name\"], 7200)\n",
|
||||
" print(\"endpoint_name:\", endpoint.name)\n",
|
||||
"\n",
|
||||
" return model, endpoint\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"models[\"trtllm_gpu\"], endpoints[\"trtllm_gpu\"] = deploy_model_tensorrt_llm_multihost(\n",
|
||||
" model_name=common_util.get_job_name_with_datetime(prefix=\"llama3-3-serve\"),\n",
|
||||
" model_id=model_id,\n",
|
||||
" publisher=\"meta-llama\",\n",
|
||||
" publisher_model_id=\"llama-3.3-70b-instruct\",\n",
|
||||
" base_model_id=\"meta-llama/Llama-3.3-70B-Instruct\",\n",
|
||||
" machine_type=machine_type,\n",
|
||||
" accelerator_type=accelerator_type,\n",
|
||||
" accelerator_count=accelerator_count,\n",
|
||||
" multihost_gpu_node_count=multihost_gpu_node_count,\n",
|
||||
" max_model_len=MAX_MODEL_LEN,\n",
|
||||
" enable_chunked_prefill=True,\n",
|
||||
" use_dedicated_endpoint=use_dedicated_endpoint,\n",
|
||||
" is_spot=is_spot,\n",
|
||||
")\n",
|
||||
"# @markdown Click \"Show Code\" to see more details."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "tcLecC90K6Br"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Raw predict\n",
|
||||
"# @markdown Once deployment succeeds, you can send requests to the endpoint using `:rawPredict`. The OpenAI Client chat completions support is coming soon.\n",
|
||||
"endpoints[\"trtllm_gpu\"] = aiplatform.Endpoint(endpoints[\"trtllm_gpu\"].resource_name)\n",
|
||||
"\n",
|
||||
"# @markdown Fill out some request parameters:\n",
|
||||
"user_message = \"How is your day going?\" # @param {type: \"string\"}\n",
|
||||
"# @markdown If you encounter the issue like `ServiceUnavailable: 503 Took too long to respond when processing`, you can reduce the maximum number of output tokens, such as set `max_tokens` as 20.\n",
|
||||
"max_tokens = 50 # @param {type: \"integer\"}\n",
|
||||
"temperature = 1.0 # @param {type: \"number\"}\n",
|
||||
"\n",
|
||||
"# @markdown Now we can send a request.\n",
|
||||
"\n",
|
||||
"response = endpoints[\"trtllm_gpu\"].raw_predict(\n",
|
||||
" body=json.dumps(\n",
|
||||
" {\n",
|
||||
" \"model\": \"\",\n",
|
||||
" \"messages\": [\n",
|
||||
" {\n",
|
||||
" \"role\": \"user\",\n",
|
||||
" \"content\": user_message,\n",
|
||||
" }\n",
|
||||
" ],\n",
|
||||
" \"max_tokens\": max_tokens,\n",
|
||||
" \"temperature\": temperature,\n",
|
||||
" }\n",
|
||||
" ),\n",
|
||||
" headers={\"Content-Type\": \"application/json\"},\n",
|
||||
" use_dedicated_endpoint=use_dedicated_endpoint,\n",
|
||||
")\n",
|
||||
"print(response.json()[\"choices\"][0][\"message\"][\"content\"])\n",
|
||||
"\n",
|
||||
"# @markdown Click \"Show Code\" to see more details."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
|
||||
@@ -243,7 +243,7 @@
|
||||
"# @markdown ---\n",
|
||||
"\n",
|
||||
"# @markdown ### Access Llama 3.3 models on Hugging Face for GPU based finetuning and serving\n",
|
||||
"# @markdown You must provide a Hugging Face User Access Token (read) to access the Llama 3.3 models. You can follow the [Hugging Face documentation](https://huggingface.co/docs/hub/en/security-tokens) to create a **read** access token and put it in the `HF_TOKEN` field below.\n",
|
||||
"# @markdown You must provide a Hugging Face User Access Token (with read access) to access the Llama 3.3 models. You can follow the [Hugging Face documentation](https://huggingface.co/docs/hub/en/security-tokens) to create a **read** access token and put it in the `HF_TOKEN` field below.\n",
|
||||
"\n",
|
||||
"HF_TOKEN = \"\" # @param {type:\"string\", isTemplate:true}\n",
|
||||
"# @markdown *--- Or ---*\n",
|
||||
@@ -737,6 +737,7 @@
|
||||
" accelerator_type=eval_accelerator_type,\n",
|
||||
" accelerator_count=eval_accelerator_count,\n",
|
||||
" is_for_training=True,\n",
|
||||
" is_dynamic_workload_scheduler=True,\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" # Pass evaluation arguments and launch job.\n",
|
||||
|
||||
@@ -544,9 +544,7 @@
|
||||
"source": [
|
||||
"if use_dedicated_endpoint:\n",
|
||||
" DEDICATED_ENDPOINT_DNS = endpoints[\"vllm_gpu\"].gca_resource.dedicated_endpoint_dns\n",
|
||||
"ENDPOINT_RESOURCE_NAME = \"projects/{}/locations/{}/endpoints/{}\".format(\n",
|
||||
" PROJECT_ID, REGION, endpoints[\"vllm_gpu\"].name\n",
|
||||
")\n",
|
||||
"ENDPOINT_RESOURCE_NAME = endpoints[\"vllm_gpu\"].resource_name\n",
|
||||
"\n",
|
||||
"# @title Chat Completions Inference\n",
|
||||
"\n",
|
||||
|
||||
@@ -222,7 +222,7 @@
|
||||
"# @markdown ---\n",
|
||||
"\n",
|
||||
"# @markdown ### Access Llama 3 models on Hugging Face for GPU based finetuning and serving\n",
|
||||
"# @markdown You must provide a Hugging Face User Access Token (read) to access the Llama 3 models. You can follow the [Hugging Face documentation](https://huggingface.co/docs/hub/en/security-tokens) to create a **read** access token and put it in the `HF_TOKEN` field below.\n",
|
||||
"# @markdown You must provide a Hugging Face User Access Token (with read access) to access the Llama 3 models. You can follow the [Hugging Face documentation](https://huggingface.co/docs/hub/en/security-tokens) to create a **read** access token and put it in the `HF_TOKEN` field below.\n",
|
||||
"\n",
|
||||
"HF_TOKEN = \"\" # @param {type:\"string\", isTemplate:true}\n",
|
||||
"if LOAD_MODEL_FROM == \"Hugging Face\":\n",
|
||||
|
||||
@@ -59,11 +59,12 @@
|
||||
"source": [
|
||||
"## Overview\n",
|
||||
"\n",
|
||||
"This notebook demonstrates deploying Llama 4 models on GPU.\n",
|
||||
"This notebook demonstrates deploying Llama 4 models on GPUs and TPUs.\n",
|
||||
"\n",
|
||||
"### Objective\n",
|
||||
"\n",
|
||||
"- Deploy Llama 4 models with [vLLM](https://github.com/vllm-project/vllm) on GPU.\n",
|
||||
"- Deploy Llama 4 models with [Hex-LLM](https://cloud.google.com/vertex-ai/generative-ai/docs/open-models/use-hex-llm) on TPU.\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"### File a bug\n",
|
||||
@@ -226,7 +227,7 @@
|
||||
"PUBLISHER_MODEL_NAME = \"publishers/meta/models/llama4\"\n",
|
||||
"\n",
|
||||
"# The pre-built serving docker image.\n",
|
||||
"VLLM_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-vllm-serve:20250405_1205_RC01\"\n",
|
||||
"VLLM_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-vllm-serve:20250417_0916_RC01\"\n",
|
||||
"\n",
|
||||
"# @markdown Set use_dedicated_endpoint to False if you don't want to use [dedicated endpoint](https://cloud.google.com/vertex-ai/docs/general/deployment#create-dedicated-endpoint). Note that [dedicated endpoint does not support VPC Service Controls](https://cloud.google.com/vertex-ai/docs/predictions/choose-endpoint-type), uncheck the box if you are using VPC-SC.\n",
|
||||
"use_dedicated_endpoint = True # @param {type:\"boolean\"}\n",
|
||||
@@ -374,8 +375,12 @@
|
||||
" vllm_args.append(f\"--model-type={model_type}\")\n",
|
||||
"\n",
|
||||
" if enable_llama_tool_parser:\n",
|
||||
" vllm_args.append(\"--enable-auto-tool-choice\")\n",
|
||||
" vllm_args.append(\"--tool-call-parser=vertex-llama-3\")\n",
|
||||
" if \"Llama-4\" not in model_id:\n",
|
||||
" vllm_args.append(\"--enable-auto-tool-choice\")\n",
|
||||
" vllm_args.append(\"--tool-call-parser=vertex-llama-3\")\n",
|
||||
" else:\n",
|
||||
" vllm_args.append(\"--enable-auto-tool-choice\")\n",
|
||||
" vllm_args.append(\"--tool-call-parser=llama3_json\")\n",
|
||||
"\n",
|
||||
" env_vars = {\n",
|
||||
" \"MODEL_ID\": base_model_id,\n",
|
||||
@@ -480,6 +485,7 @@
|
||||
" gpu_memory_utilization=gpu_memory_utilization,\n",
|
||||
" max_model_len=max_model_len,\n",
|
||||
" enforce_eager=False,\n",
|
||||
" enable_llama_tool_parser=True,\n",
|
||||
" use_dedicated_endpoint=use_dedicated_endpoint,\n",
|
||||
")\n",
|
||||
"# @markdown Click \"Show Code\" to see more details."
|
||||
@@ -563,9 +569,7 @@
|
||||
"\n",
|
||||
"if use_dedicated_endpoint:\n",
|
||||
" DEDICATED_ENDPOINT_DNS = endpoints[LABEL].gca_resource.dedicated_endpoint_dns\n",
|
||||
"ENDPOINT_RESOURCE_NAME = \"projects/{}/locations/{}/endpoints/{}\".format(\n",
|
||||
" PROJECT_ID, REGION, endpoints[LABEL].name\n",
|
||||
")\n",
|
||||
"ENDPOINT_RESOURCE_NAME = endpoints[LABEL].resource_name\n",
|
||||
"\n",
|
||||
"# @title Chat Completions Inference\n",
|
||||
"\n",
|
||||
@@ -640,9 +644,426 @@
|
||||
"\n",
|
||||
"if use_dedicated_endpoint:\n",
|
||||
" DEDICATED_ENDPOINT_DNS = endpoints[LABEL].gca_resource.dedicated_endpoint_dns\n",
|
||||
"ENDPOINT_RESOURCE_NAME = \"projects/{}/locations/{}/endpoints/{}\".format(\n",
|
||||
" PROJECT_ID, REGION, endpoints[LABEL].name\n",
|
||||
"ENDPOINT_RESOURCE_NAME = endpoints[LABEL].resource_name\n",
|
||||
"\n",
|
||||
"# @title Chat Completions Inference\n",
|
||||
"\n",
|
||||
"# @markdown Once deployment succeeds, you can send requests to the endpoint using the OpenAI SDK.\n",
|
||||
"\n",
|
||||
"# @markdown First you will need to install the SDK and some auth-related dependencies.\n",
|
||||
"\n",
|
||||
"! pip install -qU openai google-auth requests\n",
|
||||
"\n",
|
||||
"# @markdown Next fill out some request parameters:\n",
|
||||
"\n",
|
||||
"user_image = \"https://upload.wikimedia.org/wikipedia/commons/thumb/c/cb/The_Blue_Marble_%28remastered%29.jpg/580px-The_Blue_Marble_%28remastered%29.jpg\" # @param {type: \"string\"}\n",
|
||||
"user_message = \"What is in the image?\" # @param {type: \"string\"}\n",
|
||||
"# @markdown If you encounter the issue like `ServiceUnavailable: 503 Took too long to respond when processing`, you can reduce the maximum number of output tokens, such as set `max_tokens` as 20.\n",
|
||||
"max_tokens = 50 # @param {type: \"integer\"}\n",
|
||||
"temperature = 1.0 # @param {type: \"number\"}\n",
|
||||
"\n",
|
||||
"# @markdown Now we can send a request.\n",
|
||||
"\n",
|
||||
"import google.auth\n",
|
||||
"import openai\n",
|
||||
"\n",
|
||||
"creds, project = google.auth.default()\n",
|
||||
"auth_req = google.auth.transport.requests.Request()\n",
|
||||
"creds.refresh(auth_req)\n",
|
||||
"\n",
|
||||
"BASE_URL = (\n",
|
||||
" f\"https://{REGION}-aiplatform.googleapis.com/v1beta1/{ENDPOINT_RESOURCE_NAME}\"\n",
|
||||
")\n",
|
||||
"try:\n",
|
||||
" if use_dedicated_endpoint:\n",
|
||||
" BASE_URL = f\"https://{DEDICATED_ENDPOINT_DNS}/v1beta1/{ENDPOINT_RESOURCE_NAME}\"\n",
|
||||
"except NameError:\n",
|
||||
" pass\n",
|
||||
"\n",
|
||||
"client = openai.OpenAI(base_url=BASE_URL, api_key=creds.token)\n",
|
||||
"\n",
|
||||
"model_response = client.chat.completions.create(\n",
|
||||
" model=\"\",\n",
|
||||
" messages=[\n",
|
||||
" {\n",
|
||||
" \"role\": \"user\",\n",
|
||||
" \"content\": [\n",
|
||||
" {\"type\": \"image_url\", \"image_url\": {\"url\": user_image}},\n",
|
||||
" {\"type\": \"text\", \"text\": user_message},\n",
|
||||
" ],\n",
|
||||
" }\n",
|
||||
" ],\n",
|
||||
" temperature=temperature,\n",
|
||||
" max_tokens=max_tokens,\n",
|
||||
")\n",
|
||||
"print(model_response)\n",
|
||||
"\n",
|
||||
"# @markdown Click \"Show Code\" to see more details."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "U4tRt3ZtClzH"
|
||||
},
|
||||
"source": [
|
||||
"## Deploy Llama 4 models with Hex-LLM"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "pIoNTUDsCr8m"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Deploy\n",
|
||||
"\n",
|
||||
"# @markdown This section uploads prebuilt Llama 4 models to Model Registry and deploys it to a Vertex AI Endpoint. It takes 15 minutes to 1 hour to finish depending on the size of the model.\n",
|
||||
"\n",
|
||||
"# @markdown Select one of the model variations.\n",
|
||||
"base_model_name = \"Llama-4-Scout-17B-16E-Instruct\" # @param [\"Llama-4-Scout-17B-16E\", \"Llama-4-Scout-17B-16E-Instruct\"] {isTemplate:true}\n",
|
||||
"TPU_DEPLOYMENT_REGION = \"us-west1\" # @param [\"us-west1\"] {isTemplate:true}\n",
|
||||
"model_path_prefix = \"meta\"\n",
|
||||
"hf_model_id = \"meta-llama/\" + base_model_name\n",
|
||||
"model_id = os.path.join(VERTEX_AI_MODEL_GARDEN_LLAMA_4, base_model_name)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# The pre-built serving docker images.\n",
|
||||
"HEXLLM_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai-restricted/vertex-vision-model-garden-dockers/hex-llm-serve:llama4\"\n",
|
||||
"\n",
|
||||
"# @markdown Find Vertex AI prediction TPUv5e machine types in\n",
|
||||
"# @markdown https://cloud.google.com/vertex-ai/docs/predictions/use-tpu#deploy_a_model.\n",
|
||||
"\n",
|
||||
"# @markdown | Model Version | Default Max Model Length | Default TPU configuration |\n",
|
||||
"# @markdown |----------------------------|------------------|-----------------------------|\n",
|
||||
"# @markdown | Llama-4-Scout-17B-16E | 8192 | 4 x 4 TPU_V5e ct5lp-hightpu-4t |\n",
|
||||
"# @markdown | Llama-4-Scout-17B-16E-Instruct | 8192 | 4 x 4 TPU_V5e ct5lp-hightpu-4t |\n",
|
||||
"\n",
|
||||
"# Note: 1 TPU V5 chip has only one core.\n",
|
||||
"tpu_type = \"TPU_V5e\"\n",
|
||||
"\n",
|
||||
"if \"Llama-4\" in model_id:\n",
|
||||
" tpu_count = 16\n",
|
||||
" tpu_topo = \"4x4\"\n",
|
||||
" num_hosts = 4\n",
|
||||
" max_model_len = 8192\n",
|
||||
" machine_type = \"ct5lp-hightpu-4t\"\n",
|
||||
"else:\n",
|
||||
" raise ValueError(f\"Unsupported MODEL_ID: {model_id}\")\n",
|
||||
"\n",
|
||||
"common_util.check_quota(\n",
|
||||
" project_id=PROJECT_ID,\n",
|
||||
" region=TPU_DEPLOYMENT_REGION,\n",
|
||||
" accelerator_type=tpu_type,\n",
|
||||
" accelerator_count=tpu_count,\n",
|
||||
" is_for_training=False,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# Server parameters.\n",
|
||||
"tensor_parallel_size = tpu_count\n",
|
||||
"\n",
|
||||
"# Fraction of HBM memory allocated for KV cache after model loading. A larger value improves throughput but gives higher risk of TPU out-of-memory errors with long prompts.\n",
|
||||
"hbm_utilization_factor = 0.85\n",
|
||||
"\n",
|
||||
"max_running_seqs = 256\n",
|
||||
"decode_seqs_padding = 128\n",
|
||||
"\n",
|
||||
"# Endpoint configurations.\n",
|
||||
"min_replica_count = 1\n",
|
||||
"max_replica_count = 1\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def deploy_model_hexllm(\n",
|
||||
" model_name: str,\n",
|
||||
" model_id: str,\n",
|
||||
" publisher: str,\n",
|
||||
" publisher_model_id: str,\n",
|
||||
" service_account: str = None,\n",
|
||||
" base_model_id: str = None,\n",
|
||||
" data_parallel_size: int = 1,\n",
|
||||
" tensor_parallel_size: int = 1,\n",
|
||||
" machine_type: str = \"ct5lp-hightpu-1t\",\n",
|
||||
" tpu_topology: str = \"1x1\",\n",
|
||||
" disagg_topology: str = None,\n",
|
||||
" hbm_utilization_factor: float = 0.6,\n",
|
||||
" max_running_seqs: int = 256,\n",
|
||||
" decode_seqs_padding: int = None,\n",
|
||||
" max_model_len: int = 4096,\n",
|
||||
" enable_prefix_cache_hbm: bool = False,\n",
|
||||
" endpoint_id: str = \"\",\n",
|
||||
" min_replica_count: int = 1,\n",
|
||||
" max_replica_count: int = 1,\n",
|
||||
" use_dedicated_endpoint: bool = False,\n",
|
||||
") -> Tuple[aiplatform.Model, aiplatform.Endpoint]:\n",
|
||||
" \"\"\"Deploys models with Hex-LLM on TPU in Vertex AI.\"\"\"\n",
|
||||
" if endpoint_id:\n",
|
||||
" aip_endpoint_name = (\n",
|
||||
" f\"projects/{PROJECT_ID}/locations/{REGION}/endpoints/{endpoint_id}\"\n",
|
||||
" )\n",
|
||||
" endpoint = aiplatform.Endpoint(aip_endpoint_name)\n",
|
||||
" else:\n",
|
||||
" endpoint = aiplatform.Endpoint.create(\n",
|
||||
" display_name=f\"{model_name}-endpoint\",\n",
|
||||
" location=TPU_DEPLOYMENT_REGION,\n",
|
||||
" dedicated_endpoint_enabled=use_dedicated_endpoint,\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" if not base_model_id:\n",
|
||||
" base_model_id = model_id\n",
|
||||
"\n",
|
||||
" if not tensor_parallel_size:\n",
|
||||
" tensor_parallel_size = int(machine_type[-2])\n",
|
||||
"\n",
|
||||
" num_hosts = int(tpu_topology.split(\"x\")[0])\n",
|
||||
"\n",
|
||||
" # Learn more about the supported arguments and environment variables at https://cloud.google.com/vertex-ai/generative-ai/docs/open-models/use-hex-llm#config-server.\n",
|
||||
" hexllm_args = [\n",
|
||||
" \"--host=0.0.0.0\",\n",
|
||||
" \"--port=7080\",\n",
|
||||
" f\"--model={model_id}\",\n",
|
||||
" f\"--data_parallel_size={data_parallel_size}\",\n",
|
||||
" f\"--tensor_parallel_size={tensor_parallel_size}\",\n",
|
||||
" f\"--num_hosts={num_hosts}\",\n",
|
||||
" f\"--hbm_utilization_factor={hbm_utilization_factor}\",\n",
|
||||
" f\"--max_running_seqs={max_running_seqs}\",\n",
|
||||
" f\"--max_model_len={max_model_len}\",\n",
|
||||
" ]\n",
|
||||
"\n",
|
||||
" if decode_seqs_padding is not None:\n",
|
||||
" hexllm_args.append(f\"--decode_seqs_padding={decode_seqs_padding}\")\n",
|
||||
"\n",
|
||||
" if disagg_topology:\n",
|
||||
" hexllm_args.append(f\"--disagg_topo={disagg_topology}\")\n",
|
||||
" if enable_prefix_cache_hbm and not disagg_topology:\n",
|
||||
" hexllm_args.append(\"--enable_prefix_cache_hbm\")\n",
|
||||
"\n",
|
||||
" env_vars = {\n",
|
||||
" \"MODEL_ID\": base_model_id,\n",
|
||||
" \"HEX_LLM_LOG_LEVEL\": \"info\",\n",
|
||||
" \"DEPLOY_SOURCE\": \"notebook\",\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" # HF_TOKEN is not a compulsory field and may not be defined.\n",
|
||||
" try:\n",
|
||||
" if HF_TOKEN:\n",
|
||||
" env_vars.update({\"HF_TOKEN\": HF_TOKEN})\n",
|
||||
" except:\n",
|
||||
" pass\n",
|
||||
"\n",
|
||||
" model = aiplatform.Model.upload(\n",
|
||||
" display_name=model_name,\n",
|
||||
" serving_container_image_uri=HEXLLM_DOCKER_URI,\n",
|
||||
" serving_container_command=[\"python\", \"-m\", \"hex_llm.server.api_server\"],\n",
|
||||
" serving_container_args=hexllm_args,\n",
|
||||
" serving_container_ports=[7080],\n",
|
||||
" serving_container_predict_route=\"/generate\",\n",
|
||||
" serving_container_health_route=\"/ping\",\n",
|
||||
" serving_container_environment_variables=env_vars,\n",
|
||||
" serving_container_shared_memory_size_mb=(16 * 1024), # 16 GB\n",
|
||||
" serving_container_deployment_timeout=7200,\n",
|
||||
" location=TPU_DEPLOYMENT_REGION,\n",
|
||||
" model_garden_source_model_name=(\n",
|
||||
" f\"publishers/{publisher}/models/{publisher_model_id}\"\n",
|
||||
" ),\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" model.deploy(\n",
|
||||
" endpoint=endpoint,\n",
|
||||
" machine_type=machine_type,\n",
|
||||
" tpu_topology=tpu_topology if num_hosts > 1 else None,\n",
|
||||
" deploy_request_timeout=1800,\n",
|
||||
" service_account=service_account,\n",
|
||||
" min_replica_count=min_replica_count,\n",
|
||||
" max_replica_count=max_replica_count,\n",
|
||||
" system_labels={\n",
|
||||
" \"NOTEBOOK_NAME\": \"model_garden_pytorch_llama4_deployment.ipynb\",\n",
|
||||
" \"NOTEBOOK_ENVIRONMENT\": common_util.get_deploy_source(),\n",
|
||||
" },\n",
|
||||
" )\n",
|
||||
" return model, endpoint\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# @markdown Set `use_dedicated_endpoint` to False if you don't want to use [dedicated endpoint](https://cloud.google.com/vertex-ai/docs/general/deployment#create-dedicated-endpoint).\n",
|
||||
"use_dedicated_endpoint = True # @param {type:\"boolean\"}\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"models[\"hexllm_tpu\"], endpoints[\"hexllm_tpu\"] = deploy_model_hexllm(\n",
|
||||
" model_name=common_util.get_job_name_with_datetime(prefix=\"llama4-serve-hexllm\"),\n",
|
||||
" model_id=model_id,\n",
|
||||
" publisher=\"meta\",\n",
|
||||
" publisher_model_id=\"llama4\",\n",
|
||||
" tensor_parallel_size=tensor_parallel_size,\n",
|
||||
" machine_type=machine_type,\n",
|
||||
" tpu_topology=tpu_topo,\n",
|
||||
" hbm_utilization_factor=hbm_utilization_factor,\n",
|
||||
" max_running_seqs=max_running_seqs,\n",
|
||||
" decode_seqs_padding=decode_seqs_padding,\n",
|
||||
" max_model_len=max_model_len,\n",
|
||||
" min_replica_count=min_replica_count,\n",
|
||||
" max_replica_count=max_replica_count,\n",
|
||||
" use_dedicated_endpoint=use_dedicated_endpoint,\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "jp77ifJ6CvZ4"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Predict\n",
|
||||
"\n",
|
||||
"# @markdown **Note that the first few prompts will take longer to execute.**\n",
|
||||
"# @markdown **Note that the framework supports an 8k context window, which can process a combination of both image and text inputs up to this limit (in tokens).**\n",
|
||||
"\n",
|
||||
"# @markdown Once deployment succeeds, you can send requests to the endpoint with text prompts based on your `template`. Note that the first few prompts will take longer to execute.\n",
|
||||
"\n",
|
||||
"# @markdown Additionally, you can moderate the generated text with Vertex AI. See [Moderate text documentation](https://cloud.google.com/natural-language/docs/moderating-text) for more details.\n",
|
||||
"\n",
|
||||
"# @markdown Example:\n",
|
||||
"\n",
|
||||
"# @markdown ```\n",
|
||||
"# @markdown > What is a car?\n",
|
||||
"# @markdown > A car is a four-wheeled vehicle designed for the transportation of passengers and their belongings.\n",
|
||||
"# @markdown ```\n",
|
||||
"\n",
|
||||
"# @markdown Additionally, you can moderate the generated text with Vertex AI. See [Moderate text documentation](https://cloud.google.com/natural-language/docs/moderating-text) for more details.\n",
|
||||
"\n",
|
||||
"# Loads an existing endpoint instance using the endpoint name:\n",
|
||||
"# - Using `endpoint_name = endpoint.name` allows us to get the endpoint\n",
|
||||
"# name of the endpoint `endpoint` created in the cell above.\n",
|
||||
"# - Alternatively, you can set `endpoint_name = \"1234567890123456789\"` to load\n",
|
||||
"# an existing endpoint with the ID 1234567890123456789.\n",
|
||||
"# You may uncomment the code below to load an existing endpoint:\n",
|
||||
"# endpoint_name = endpoint_without_peft.name\n",
|
||||
"# # endpoint_name = \"\" # @param {type:\"string\"}\n",
|
||||
"# aip_endpoint_name = (\n",
|
||||
"# f\"projects/{PROJECT_ID}/locations/{REGION}/endpoints/{endpoint_name}\"\n",
|
||||
"# )\n",
|
||||
"# endpoint = aiplatform.Endpoint(aip_endpoint_name)\n",
|
||||
"\n",
|
||||
"prompt = \"What is a car?\" # @param {type: \"string\"}\n",
|
||||
"# @markdown If you encounter the issue like `ServiceUnavailable: 503 Took too long to respond when processing`, you can reduce the maximum number of output tokens, such as set `max_tokens` as 20.\n",
|
||||
"max_tokens = 50 # @param {type: \"integer\"}\n",
|
||||
"temperature = 1.0 # @param {type: \"number\"}\n",
|
||||
"top_p = 1.0 # @param {type: \"number\"}\n",
|
||||
"top_k = 1 # @param {type: \"integer\"}\n",
|
||||
"\n",
|
||||
"# Overrides parameters for inferences.\n",
|
||||
"instances = [\n",
|
||||
" {\n",
|
||||
" \"prompt\": prompt,\n",
|
||||
" \"max_tokens\": max_tokens,\n",
|
||||
" \"temperature\": temperature,\n",
|
||||
" \"top_p\": top_p,\n",
|
||||
" \"top_k\": top_k,\n",
|
||||
" },\n",
|
||||
"]\n",
|
||||
"response = endpoints[\"hexllm_tpu\"].predict(\n",
|
||||
" instances=instances, use_dedicated_endpoint=use_dedicated_endpoint\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"for prediction in response.predictions:\n",
|
||||
" print(prediction)\n",
|
||||
"\n",
|
||||
"# @markdown Click \"Show Code\" to see more details."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "B6WA_XX-I2NL"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Chat completion with text-only requests\n",
|
||||
"\n",
|
||||
"if use_dedicated_endpoint:\n",
|
||||
" DEDICATED_ENDPOINT_DNS = endpoints[\"hexllm_tpu\"].gca_resource.dedicated_endpoint_dns\n",
|
||||
"ENDPOINT_RESOURCE_NAME = endpoints[\"hexllm_tpu\"].resource_name\n",
|
||||
"\n",
|
||||
"# @title Chat Completions Inference\n",
|
||||
"\n",
|
||||
"# @markdown Once deployment succeeds, you can send requests to the endpoint using the OpenAI SDK.\n",
|
||||
"\n",
|
||||
"# @markdown First you will need to install the SDK and some auth-related dependencies.\n",
|
||||
"\n",
|
||||
"! pip install -qU openai google-auth requests\n",
|
||||
"\n",
|
||||
"# @markdown Next fill out some request parameters:\n",
|
||||
"\n",
|
||||
"user_message = \"How is your day going?\" # @param {type: \"string\"}\n",
|
||||
"# @markdown If you encounter the issue like `ServiceUnavailable: 503 Took too long to respond when processing`, you can reduce the maximum number of output tokens, such as set `max_tokens` as 20.\n",
|
||||
"max_tokens = 50 # @param {type: \"integer\"}\n",
|
||||
"temperature = 1.0 # @param {type: \"number\"}\n",
|
||||
"stream = False # @param {type: \"boolean\"}\n",
|
||||
"\n",
|
||||
"# @markdown Now we can send a request.\n",
|
||||
"\n",
|
||||
"import google.auth\n",
|
||||
"import openai\n",
|
||||
"\n",
|
||||
"creds, project = google.auth.default()\n",
|
||||
"auth_req = google.auth.transport.requests.Request()\n",
|
||||
"creds.refresh(auth_req)\n",
|
||||
"\n",
|
||||
"BASE_URL = (\n",
|
||||
" f\"https://{REGION}-aiplatform.googleapis.com/v1beta1/{ENDPOINT_RESOURCE_NAME}\"\n",
|
||||
")\n",
|
||||
"try:\n",
|
||||
" if use_dedicated_endpoint:\n",
|
||||
" BASE_URL = f\"https://{DEDICATED_ENDPOINT_DNS}/v1beta1/{ENDPOINT_RESOURCE_NAME}\"\n",
|
||||
"except NameError:\n",
|
||||
" pass\n",
|
||||
"\n",
|
||||
"client = openai.OpenAI(base_url=BASE_URL, api_key=creds.token)\n",
|
||||
"\n",
|
||||
"model_response = client.chat.completions.create(\n",
|
||||
" model=\"\",\n",
|
||||
" messages=[{\"role\": \"user\", \"content\": user_message}],\n",
|
||||
" temperature=temperature,\n",
|
||||
" max_tokens=max_tokens,\n",
|
||||
" stream=stream,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"if stream:\n",
|
||||
" usage = None\n",
|
||||
" contents = []\n",
|
||||
" for chunk in model_response:\n",
|
||||
" if chunk.usage is not None:\n",
|
||||
" usage = chunk.usage\n",
|
||||
" continue\n",
|
||||
" print(chunk.choices[0].delta.content, end=\"\")\n",
|
||||
" contents.append(chunk.choices[0].delta.content)\n",
|
||||
" print(f\"\\n\\n{usage}\")\n",
|
||||
"else:\n",
|
||||
" print(model_response)\n",
|
||||
"\n",
|
||||
"# @markdown Click \"Show Code\" to see more details."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "mAbnF4uEJCGk"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Chat completion for multi-modal requests\n",
|
||||
"\n",
|
||||
"if use_dedicated_endpoint:\n",
|
||||
" DEDICATED_ENDPOINT_DNS = endpoints[\"hexllm_tpu\"].gca_resource.dedicated_endpoint_dns\n",
|
||||
"ENDPOINT_RESOURCE_NAME = endpoints[\"hexllm_tpu\"].resource_name\n",
|
||||
"\n",
|
||||
"# @title Chat Completions Inference\n",
|
||||
"\n",
|
||||
|
||||
@@ -157,7 +157,7 @@
|
||||
" location=REGION,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# @markdown You must provide a Hugging Face User Access Token (read) to access models from Hugging Face. You can follow the [Hugging Face documentation](https://huggingface.co/docs/hub/en/security-tokens) to create a **read** access token and put it in the `HF_TOKEN` field below.\n",
|
||||
"# @markdown You must provide a Hugging Face User Access Token (with read access) to access models from Hugging Face. You can follow the [Hugging Face documentation](https://huggingface.co/docs/hub/en/security-tokens) to create a **read** access token and put it in the `HF_TOKEN` field below.\n",
|
||||
"\n",
|
||||
"HF_TOKEN = \"\" # @param {type:\"string\"}\n",
|
||||
"assert HF_TOKEN, \"Provide a read HF_TOKEN to load models from Hugging Face.\"\n",
|
||||
|
||||
@@ -739,9 +739,7 @@
|
||||
" DEDICATED_ENDPOINT_DNS = endpoints[\n",
|
||||
" \"optimized_vllm_gpu\"\n",
|
||||
" ].gca_resource.dedicated_endpoint_dns\n",
|
||||
"ENDPOINT_RESOURCE_NAME = \"projects/{}/locations/{}/endpoints/{}\".format(\n",
|
||||
" PROJECT_ID, REGION, endpoints[\"optimized_vllm_gpu\"].name\n",
|
||||
")\n",
|
||||
"ENDPOINT_RESOURCE_NAME = endpoints[\"optimized_vllm_gpu\"].resource_name\n",
|
||||
"\n",
|
||||
"# @title Chat Completions Inference\n",
|
||||
"\n",
|
||||
|
||||
+81
-146
@@ -90,107 +90,11 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Setup Google Cloud project\n",
|
||||
"# @title Request for quota\n",
|
||||
"\n",
|
||||
"# @markdown 1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
|
||||
"# @markdown By default, the quota for A100_80GB and H100 deployment `Custom model serving per region` is 0. You need to request quota following the instructions at [\"Request a higher quota\"](https://cloud.google.com/docs/quota/view-manage#requesting_higher_quota).\n",
|
||||
"\n",
|
||||
"# @markdown 2. **[Optional]** [Create a Cloud Storage bucket](https://cloud.google.com/storage/docs/creating-buckets) for storing experiment outputs. Set the BUCKET_URI for the experiment environment. The specified Cloud Storage bucket (`BUCKET_URI`) should be located in the same region as where the notebook was launched. Note that a multi-region bucket (eg. \"us\") is not considered a match for a single region covered by the multi-region range (eg. \"us-central1\"). If not set, a unique GCS bucket will be created instead.\n",
|
||||
"\n",
|
||||
"BUCKET_URI = \"gs://\" # @param {type:\"string\"}\n",
|
||||
"\n",
|
||||
"# @markdown 3. **[Optional]** Set region. If not set, the region will be set automatically according to Colab Enterprise environment.\n",
|
||||
"\n",
|
||||
"REGION = \"\" # @param {type:\"string\"}\n",
|
||||
"\n",
|
||||
"# @markdown 4. If you want to run predictions with A100 80GB or H100 GPUs, we recommend using the regions listed below. **NOTE:** Make sure you have associated quota in selected regions. Click the links to see your current quota for each GPU type: [Nvidia A100 80GB](https://console.cloud.google.com/iam-admin/quotas?metric=aiplatform.googleapis.com%2Fcustom_model_serving_nvidia_a100_80gb_gpus), [Nvidia H100 80GB](https://console.cloud.google.com/iam-admin/quotas?metric=aiplatform.googleapis.com%2Fcustom_model_serving_nvidia_h100_gpus). You can request for quota following the instructions at [\"Request a higher quota\"](https://cloud.google.com/docs/quota/view-manage#requesting_higher_quota).\n",
|
||||
"\n",
|
||||
"# @markdown > | Machine Type | Accelerator Type | Recommended Regions |\n",
|
||||
"# @markdown | ----------- | ----------- | ----------- |\n",
|
||||
"# @markdown | a2-ultragpu-1g | 1 NVIDIA_A100_80GB | us-central1, us-east4, europe-west4, asia-southeast1, us-east4 |\n",
|
||||
"# @markdown | a3-highgpu-2g | 2 NVIDIA_H100_80GB | us-west1, asia-southeast1, europe-west4 |\n",
|
||||
"# @markdown | a3-highgpu-4g | 4 NVIDIA_H100_80GB | us-west1, asia-southeast1, europe-west4 |\n",
|
||||
"# @markdown | a3-highgpu-8g | 8 NVIDIA_H100_80GB | us-central1, europe-west4, us-west1, asia-southeast1 |\n",
|
||||
"\n",
|
||||
"# Import the necessary packages\n",
|
||||
"# Upgrade Vertex AI SDK.\n",
|
||||
"! pip3 install --upgrade --quiet 'google-cloud-aiplatform>=1.64.0'\n",
|
||||
"! git clone https://github.com/GoogleCloudPlatform/vertex-ai-samples.git\n",
|
||||
"\n",
|
||||
"import datetime\n",
|
||||
"import importlib\n",
|
||||
"import os\n",
|
||||
"import re\n",
|
||||
"import uuid\n",
|
||||
"from typing import Tuple\n",
|
||||
"\n",
|
||||
"from google.cloud import aiplatform\n",
|
||||
"\n",
|
||||
"common_util = importlib.import_module(\n",
|
||||
" \"vertex-ai-samples.community-content.vertex_model_garden.model_oss.notebook_util.common_util\"\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Get the default cloud project id.\n",
|
||||
"PROJECT_ID = os.environ[\"GOOGLE_CLOUD_PROJECT\"]\n",
|
||||
"\n",
|
||||
"# Get the default region for launching jobs.\n",
|
||||
"if not REGION:\n",
|
||||
" if not os.environ.get(\"GOOGLE_CLOUD_REGION\"):\n",
|
||||
" raise ValueError(\n",
|
||||
" \"REGION must be set. See\"\n",
|
||||
" \" https://cloud.google.com/vertex-ai/docs/general/locations for\"\n",
|
||||
" \" available cloud locations.\"\n",
|
||||
" )\n",
|
||||
" REGION = os.environ[\"GOOGLE_CLOUD_REGION\"]\n",
|
||||
"\n",
|
||||
"# Enable the Vertex AI API and Compute Engine API, if not already.\n",
|
||||
"print(\"Enabling Vertex AI API and Compute Engine API.\")\n",
|
||||
"! gcloud services enable aiplatform.googleapis.com compute.googleapis.com\n",
|
||||
"\n",
|
||||
"# Cloud Storage bucket for storing the experiment artifacts.\n",
|
||||
"# A unique GCS bucket will be created for the purpose of this notebook. If you\n",
|
||||
"# prefer using your own GCS bucket, change the value yourself below.\n",
|
||||
"now = datetime.datetime.now().strftime(\"%Y%m%d%H%M%S\")\n",
|
||||
"BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n",
|
||||
"\n",
|
||||
"if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n",
|
||||
" BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n",
|
||||
" BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n",
|
||||
" ! gsutil mb -l {REGION} {BUCKET_URI}\n",
|
||||
"else:\n",
|
||||
" assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n",
|
||||
" shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n",
|
||||
" bucket_region = shell_output[0].strip().lower()\n",
|
||||
" if bucket_region != REGION:\n",
|
||||
" raise ValueError(\n",
|
||||
" \"Bucket region %s is different from notebook region %s\"\n",
|
||||
" % (bucket_region, REGION)\n",
|
||||
" )\n",
|
||||
"print(f\"Using this GCS Bucket: {BUCKET_URI}\")\n",
|
||||
"\n",
|
||||
"STAGING_BUCKET = os.path.join(BUCKET_URI, \"temporal\")\n",
|
||||
"MODEL_BUCKET = os.path.join(BUCKET_URI, \"Prompt-Guard-86M\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Initialize Vertex AI API.\n",
|
||||
"print(\"Initializing Vertex AI API.\")\n",
|
||||
"aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=STAGING_BUCKET)\n",
|
||||
"\n",
|
||||
"# Gets the default SERVICE_ACCOUNT.\n",
|
||||
"shell_output = ! gcloud projects describe $PROJECT_ID\n",
|
||||
"project_number = shell_output[-1].split(\":\")[1].strip().replace(\"'\", \"\")\n",
|
||||
"SERVICE_ACCOUNT = f\"{project_number}-compute@developer.gserviceaccount.com\"\n",
|
||||
"print(\"Using this default Service Account:\", SERVICE_ACCOUNT)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n",
|
||||
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n",
|
||||
"\n",
|
||||
"! gcloud config set project $PROJECT_ID\n",
|
||||
"! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n",
|
||||
"! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\"\n",
|
||||
"\n",
|
||||
"models, endpoints = {}, {}"
|
||||
"# @markdown For better chance to get resources, we recommend to request A100_80GB quota in the regions `us-central1, us-east1`, and request H100 quota in the regions `us-central1, us-west1`."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -202,57 +106,87 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Access Prompt Guard\n",
|
||||
"# @title Setup Google Cloud project\n",
|
||||
"\n",
|
||||
"# @markdown For GPU based serving, choose between accessing the Prompt Guard model on [Hugging Face](https://huggingface.co/)\n",
|
||||
"# @markdown or Vertex AI as described below.\n",
|
||||
"# @markdown 1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
|
||||
"\n",
|
||||
"# @markdown If you already obtained access to Prompt Guard on [Hugging Face](https://huggingface.co/), you can load the model from there.\n",
|
||||
"# @markdown Alternatively, you can also load the original Prompt Guard model for serving from Vertex AI after accepting the agreement.\n",
|
||||
"# @markdown 2. **[Optional]** Set region. If not set, the region will be set automatically according to Colab Enterprise environment.\n",
|
||||
"\n",
|
||||
"# @markdown **Only select and fill one of the following sections.**\n",
|
||||
"# fmt: off\n",
|
||||
"LOAD_MODEL_FROM = \"Hugging Face\" # @param [\"Hugging Face\", \"Google Cloud\"] {isTemplate:true}\n",
|
||||
"# fmt: on\n",
|
||||
"REGION = \"\" # @param {type:\"string\"}\n",
|
||||
"\n",
|
||||
"# @markdown ---\n",
|
||||
"# @markdown 3. If you want to run predictions with A100 80GB or H100 GPUs, we recommend using the regions listed below. **NOTE:** Make sure you have associated quota in selected regions. Click the links to see your current quota for each GPU type: [Nvidia A100 80GB](https://console.cloud.google.com/iam-admin/quotas?metric=aiplatform.googleapis.com%2Fcustom_model_serving_nvidia_a100_80gb_gpus), [Nvidia H100 80GB](https://console.cloud.google.com/iam-admin/quotas?metric=aiplatform.googleapis.com%2Fcustom_model_serving_nvidia_h100_gpus). You can request for quota following the instructions at [\"Request a higher quota\"](https://cloud.google.com/docs/quota/view-manage#requesting_higher_quota).\n",
|
||||
"\n",
|
||||
"# @markdown ### Access Prompt Guard on Hugging Face for GPU based serving\n",
|
||||
"# @markdown You must provide a Hugging Face User Access Token (read) to access Prompt Guard. You can follow the [Hugging Face documentation](https://huggingface.co/docs/hub/en/security-tokens) to create a **read** access token and put it in the `HF_TOKEN` field below.\n",
|
||||
"# @markdown > | Machine Type | Accelerator Type | Recommended Regions |\n",
|
||||
"# @markdown | ----------- | ----------- | ----------- |\n",
|
||||
"# @markdown | a2-ultragpu-1g | 1 NVIDIA_A100_80GB | us-central1, us-east4, europe-west4, asia-southeast1, us-east4 |\n",
|
||||
"# @markdown | a3-highgpu-2g | 2 NVIDIA_H100_80GB | us-west1, asia-southeast1, europe-west4 |\n",
|
||||
"# @markdown | a3-highgpu-4g | 4 NVIDIA_H100_80GB | us-west1, asia-southeast1, europe-west4 |\n",
|
||||
"# @markdown | a3-highgpu-8g | 8 NVIDIA_H100_80GB | us-central1, europe-west4, us-west1, asia-southeast1 |\n",
|
||||
"\n",
|
||||
"HF_TOKEN = \"\" # @param {type:\"string\", isTemplate:true}\n",
|
||||
"# Import the necessary packages\n",
|
||||
"\n",
|
||||
"# @markdown *--- Or ---*\n",
|
||||
"# @markdown ### Access Prompt Guard on Vertex AI for GPU based serving\n",
|
||||
"# @markdown The original model from Meta is converted into the Hugging Face format for serving in Vertex AI.\n",
|
||||
"# Upgrade Vertex AI SDK.\n",
|
||||
"! pip3 install --upgrade --quiet 'google-cloud-aiplatform>=1.84.0'\n",
|
||||
"! git clone https://github.com/GoogleCloudPlatform/vertex-ai-samples.git\n",
|
||||
"\n",
|
||||
"import importlib\n",
|
||||
"import os\n",
|
||||
"import re\n",
|
||||
"from typing import Tuple\n",
|
||||
"\n",
|
||||
"from google.cloud import aiplatform\n",
|
||||
"\n",
|
||||
"if os.environ.get(\"VERTEX_PRODUCT\") != \"COLAB_ENTERPRISE\":\n",
|
||||
" ! pip install --upgrade tensorflow\n",
|
||||
"! git clone https://github.com/GoogleCloudPlatform/vertex-ai-samples.git\n",
|
||||
"\n",
|
||||
"common_util = importlib.import_module(\n",
|
||||
" \"vertex-ai-samples.community-content.vertex_model_garden.model_oss.notebook_util.common_util\"\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"LABEL = \"pytorch_inference_gpu\"\n",
|
||||
"models, endpoints = {}, {}\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Get the default cloud project id.\n",
|
||||
"PROJECT_ID = os.environ[\"GOOGLE_CLOUD_PROJECT\"]\n",
|
||||
"\n",
|
||||
"# Get the default region for launching jobs.\n",
|
||||
"if not REGION:\n",
|
||||
" REGION = os.environ[\"GOOGLE_CLOUD_REGION\"]\n",
|
||||
"\n",
|
||||
"# Initialize Vertex AI API.\n",
|
||||
"print(\"Initializing Vertex AI API.\")\n",
|
||||
"aiplatform.init(project=PROJECT_ID, location=REGION)\n",
|
||||
"\n",
|
||||
"! gcloud config set project $PROJECT_ID\n",
|
||||
"import vertexai\n",
|
||||
"\n",
|
||||
"vertexai.init(\n",
|
||||
" project=PROJECT_ID,\n",
|
||||
" location=REGION,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# @markdown # Access Llama Prompt Guard models on Vertex AI\n",
|
||||
"# @markdown The original models from Meta are converted into the Hugging Face format for serving in Vertex AI.\n",
|
||||
"# @markdown Accept the model agreement to access the models:\n",
|
||||
"# @markdown 1. Open the [Prompt Guard model card](https://console.cloud.google.com/vertex-ai/publishers/meta/model-garden/prompt-guard) from [Vertex AI Model Garden](https://cloud.google.com/model-garden).\n",
|
||||
"# @markdown 1. Open the [Llama Prompt Guard model card](https://console.cloud.google.com/vertex-ai/publishers/meta/model-garden/prompt-guard) from [Vertex AI Model Garden](https://cloud.google.com/model-garden).\n",
|
||||
"# @markdown 2. Review and accept the agreement in the pop-up window on the model card page. If you have previously accepted the model agreement, there will not be a pop-up window on the model card page and this step is not needed.\n",
|
||||
"# @markdown 3. After accepting the agreement of Prompt Guard, a `gs://` URI containing the Prompt Guard model artifacts will be shared.\n",
|
||||
"# @markdown 3. After accepting the agreement, a `gs://` URI containing Llama Prompt Guard models will be shared.\n",
|
||||
"# @markdown 4. Paste the URI in the `VERTEX_AI_MODEL_GARDEN_PROMPT_GUARD` field below.\n",
|
||||
"# @markdown 5. The Llama Prompt Guard models will be copied into `BUCKET_URI`.\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"VERTEX_AI_MODEL_GARDEN_PROMPT_GUARD = \"\" # @param {type:\"string\", isTemplate:true}\n",
|
||||
"if LOAD_MODEL_FROM == \"Hugging Face\":\n",
|
||||
" assert (\n",
|
||||
" HF_TOKEN\n",
|
||||
" ), \"Provide a read HF_TOKEN to load models from Hugging Face, or select a different model source.\"\n",
|
||||
"else:\n",
|
||||
" assert (\n",
|
||||
" VERTEX_AI_MODEL_GARDEN_PROMPT_GUARD\n",
|
||||
" ), \"Click the agreement of Prompt Guard in Vertex AI Model Garden via https://console.cloud.google.com/vertex-ai/publishers/meta/model-garden/prompt-guard, and get the GCS path of Prompt Guard model artifacts.\"\n",
|
||||
" parsed_gcs_url = re.search(\"gs://.*?(?=[ ]|$)\", VERTEX_AI_MODEL_GARDEN_PROMPT_GUARD)\n",
|
||||
" if parsed_gcs_url:\n",
|
||||
" VERTEX_AI_MODEL_GARDEN_PROMPT_GUARD = parsed_gcs_url.group()\n",
|
||||
" assert VERTEX_AI_MODEL_GARDEN_PROMPT_GUARD.startswith(\n",
|
||||
" \"gs://\"\n",
|
||||
" ), \"VERTEX_AI_MODEL_GARDEN_PROMPT_GUARD is expected to be a GCS URI and must start with `gs://`.\"\n",
|
||||
" print(\n",
|
||||
" \"Copying Prompt Guard model artifacts from\",\n",
|
||||
" VERTEX_AI_MODEL_GARDEN_PROMPT_GUARD,\n",
|
||||
" \"to \",\n",
|
||||
" MODEL_BUCKET,\n",
|
||||
" )\n",
|
||||
" ! gsutil -m cp -R $VERTEX_AI_MODEL_GARDEN_PROMPT_GUARD/* $MODEL_BUCKET\n",
|
||||
"assert (\n",
|
||||
" VERTEX_AI_MODEL_GARDEN_PROMPT_GUARD\n",
|
||||
"), \"Click the agreement in Vertex AI Model Garden at https://console.cloud.google.com/vertex-ai/publishers/meta/model-garden/prompt-guard, and get the GCS path of Llama Prompt Guard model artifacts.\"\n",
|
||||
"parsed_gcs_url = re.search(\"gs://.*?(?=[ ]|$)\", VERTEX_AI_MODEL_GARDEN_PROMPT_GUARD)\n",
|
||||
"if parsed_gcs_url:\n",
|
||||
" VERTEX_AI_MODEL_GARDEN_PROMPT_GUARD = parsed_gcs_url.group()\n",
|
||||
"assert VERTEX_AI_MODEL_GARDEN_PROMPT_GUARD.startswith(\n",
|
||||
" \"gs://\"\n",
|
||||
"), \"VERTEX_AI_MODEL_GARDEN_PROMPT_GUARD is expected to be a GCS URI and must start with `gs://`.\"\n",
|
||||
"\n",
|
||||
"# @markdown ---\n",
|
||||
"# @markdown Click \"Show Code\" to see more details."
|
||||
@@ -285,14 +219,15 @@
|
||||
"# @markdown NVIDIA_L4 GPUs are used for demonstration. The serving efficiency of L4 GPUs is inferior to that of A100 GPUs, but L4 GPUs are nevertheless good serving solutions if you do not have A100 quota.\n",
|
||||
"\n",
|
||||
"# @markdown Find Vertex AI prediction supported accelerators and regions at https://cloud.google.com/vertex-ai/docs/predictions/configure-compute.\n",
|
||||
"\n",
|
||||
"if LOAD_MODEL_FROM == \"Hugging Face\":\n",
|
||||
" model_id = \"meta-llama/Prompt-Guard-86M\"\n",
|
||||
"else:\n",
|
||||
" model_id = MODEL_BUCKET\n",
|
||||
"base_model_name = \"Llama-Prompt-Guard-2-22M\" # @param [\"Prompt-Guard-86M\", \"Llama-Prompt-Guard-2-22M\", \"Llama-Prompt-Guard-2-86M\"] {allow-input: true, isTemplate: true}\n",
|
||||
"model_id = os.path.join(VERTEX_AI_MODEL_GARDEN_PROMPT_GUARD, base_model_name)\n",
|
||||
"hf_model_id = \"meta-llama/\" + base_model_name\n",
|
||||
"version_id = base_model_name.lower()\n",
|
||||
"PUBLISHER_MODEL_NAME = f\"publishers/meta/models/prompt-guard@{version_id}\"\n",
|
||||
"\n",
|
||||
"# The pre-built serving docker image.\n",
|
||||
"SERVE_DOCKER_URI = \"us-docker.pkg.dev/deeplearning-platform-release/gcr.io/huggingface-pytorch-inference-cu121.2-2.transformers.4-41.ubuntu2204.py311\"\n",
|
||||
"SERVE_DOCKER_URI = \"us-docker.pkg.dev/deeplearning-platform-release/gcr.io/huggingface-pytorch-inference-cu121.2-3.transformers.4-46.ubuntu2204.py311\"\n",
|
||||
"SERVICE_ACCOUNT = None\n",
|
||||
"\n",
|
||||
"accelerator_type = \"NVIDIA_L4\" # @param [\"NVIDIA_L4\", \"NVIDIA_A100_80GB\"]\n",
|
||||
"\n",
|
||||
@@ -387,9 +322,9 @@
|
||||
" return model, endpoint\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"models[\"pytorch_inference_gpu\"], endpoints[\"pytorch_inference_gpu\"] = deploy_model(\n",
|
||||
"models[LABEL], endpoints[LABEL] = deploy_model(\n",
|
||||
" model_name=common_util.get_job_name_with_datetime(prefix=\"prompt-guard-serve\"),\n",
|
||||
" model_id=model_id,\n",
|
||||
" model_id=model_id if is_gcs_path(model_id) else hf_model_id,\n",
|
||||
" publisher=\"meta\",\n",
|
||||
" publisher_model_id=\"prompt-guard\",\n",
|
||||
" task=task,\n",
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -62,6 +62,10 @@
|
||||
"- Download and deploy prebuilt Qwen2/Qwen2.5 models\n",
|
||||
"- Deploy Qwen2/Qwen2.5 with [vLLM](https://github.com/vllm-project/vllm) to improve serving throughput\n",
|
||||
"\n",
|
||||
"### File a bug\n",
|
||||
"\n",
|
||||
"File a bug on [GitHub](https://github.com/GoogleCloudPlatform/vertex-ai-samples/issues/new) if you encounter any issue with the notebook.\n",
|
||||
"\n",
|
||||
"### Costs\n",
|
||||
"\n",
|
||||
"This tutorial uses billable components of Google Cloud:\n",
|
||||
@@ -94,15 +98,11 @@
|
||||
"\n",
|
||||
"# @markdown 1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
|
||||
"\n",
|
||||
"# @markdown 2. **[Optional]** [Create a Cloud Storage bucket](https://cloud.google.com/storage/docs/creating-buckets) for storing experiment outputs. Set the BUCKET_URI for the experiment environment. The specified Cloud Storage bucket (`BUCKET_URI`) should be located in the same region as where the notebook was launched. Note that a multi-region bucket (eg. \"us\") is not considered a match for a single region covered by the multi-region range (eg. \"us-central1\"). If not set, a unique GCS bucket will be created instead.\n",
|
||||
"\n",
|
||||
"BUCKET_URI = \"gs://\" # @param {type:\"string\"}\n",
|
||||
"\n",
|
||||
"# @markdown 3. **[Optional]** Set region. If not set, the region will be set automatically according to Colab Enterprise environment.\n",
|
||||
"# @markdown 2. **[Optional]** Set region. If not set, the region will be set automatically according to Colab Enterprise environment.\n",
|
||||
"\n",
|
||||
"REGION = \"\" # @param {type:\"string\"}\n",
|
||||
"\n",
|
||||
"# @markdown 4. If you want to run predictions with A100 80GB or H100 GPUs, we recommend using the regions listed below. **NOTE:** Make sure you have associated quota in selected regions. Click the links to see your current quota for each GPU type: [Nvidia A100 80GB](https://console.cloud.google.com/iam-admin/quotas?metric=aiplatform.googleapis.com%2Fcustom_model_serving_nvidia_a100_80gb_gpus), [Nvidia H100 80GB](https://console.cloud.google.com/iam-admin/quotas?metric=aiplatform.googleapis.com%2Fcustom_model_serving_nvidia_h100_gpus). You can request for quota following the instructions at [\"Request a higher quota\"](https://cloud.google.com/docs/quota/view-manage#requesting_higher_quota).\n",
|
||||
"# @markdown 3. If you want to run predictions with A100 80GB or H100 GPUs, we recommend using the regions listed below. **NOTE:** Make sure you have associated quota in selected regions. Click the links to see your current quota for each GPU type: [Nvidia A100 80GB](https://console.cloud.google.com/iam-admin/quotas?metric=aiplatform.googleapis.com%2Fcustom_model_serving_nvidia_a100_80gb_gpus), [Nvidia H100 80GB](https://console.cloud.google.com/iam-admin/quotas?metric=aiplatform.googleapis.com%2Fcustom_model_serving_nvidia_h100_gpus). You can request for quota following the instructions at [\"Request a higher quota\"](https://cloud.google.com/docs/quota/view-manage#requesting_higher_quota).\n",
|
||||
"\n",
|
||||
"# @markdown > | Machine Type | Accelerator Type | Recommended Regions |\n",
|
||||
"# @markdown | ----------- | ----------- | ----------- |\n",
|
||||
@@ -112,84 +112,47 @@
|
||||
"# @markdown | a3-highgpu-8g | 8 NVIDIA_H100_80GB | us-central1, europe-west4, us-west1, asia-southeast1 |\n",
|
||||
"\n",
|
||||
"# Upgrade Vertex AI SDK.\n",
|
||||
"! pip3 install --upgrade --quiet 'google-cloud-aiplatform>=1.64.0'\n",
|
||||
"! pip3 install --upgrade --quiet 'google-cloud-aiplatform>=1.84.0'\n",
|
||||
"\n",
|
||||
"# Import the necessary packages\n",
|
||||
"import datetime\n",
|
||||
"\n",
|
||||
"import importlib\n",
|
||||
"import os\n",
|
||||
"import uuid\n",
|
||||
"from typing import Tuple\n",
|
||||
"\n",
|
||||
"from google.cloud import aiplatform\n",
|
||||
"\n",
|
||||
"! git clone https://github.com/GoogleCloudPlatform/vertex-ai-samples.git\n",
|
||||
"\n",
|
||||
"models, endpoints = {}, {}\n",
|
||||
"LABEL = \"vllm_gpu\"\n",
|
||||
"\n",
|
||||
"if os.environ.get(\"VERTEX_PRODUCT\") != \"COLAB_ENTERPRISE\":\n",
|
||||
" ! pip install --upgrade tensorflow\n",
|
||||
"! git clone https://github.com/GoogleCloudPlatform/vertex-ai-samples.git\n",
|
||||
"\n",
|
||||
"common_util = importlib.import_module(\n",
|
||||
" \"vertex-ai-samples.community-content.vertex_model_garden.model_oss.notebook_util.common_util\"\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Get the default cloud project id.\n",
|
||||
"PROJECT_ID = os.environ[\"GOOGLE_CLOUD_PROJECT\"]\n",
|
||||
"\n",
|
||||
"# Get the default region for launching jobs.\n",
|
||||
"if not REGION:\n",
|
||||
" if not os.environ.get(\"GOOGLE_CLOUD_REGION\"):\n",
|
||||
" raise ValueError(\n",
|
||||
" \"REGION must be set. See\"\n",
|
||||
" \" https://cloud.google.com/vertex-ai/docs/general/locations for\"\n",
|
||||
" \" available cloud locations.\"\n",
|
||||
" )\n",
|
||||
" REGION = os.environ[\"GOOGLE_CLOUD_REGION\"]\n",
|
||||
"\n",
|
||||
"# Enable the Vertex AI API and Compute Engine API, if not already.\n",
|
||||
"print(\"Enabling Vertex AI API and Compute Engine API.\")\n",
|
||||
"! gcloud services enable aiplatform.googleapis.com compute.googleapis.com\n",
|
||||
"\n",
|
||||
"# Cloud Storage bucket for storing the experiment artifacts.\n",
|
||||
"# A unique GCS bucket will be created for the purpose of this notebook. If you\n",
|
||||
"# prefer using your own GCS bucket, change the value yourself below.\n",
|
||||
"now = datetime.datetime.now().strftime(\"%Y%m%d%H%M%S\")\n",
|
||||
"BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n",
|
||||
"\n",
|
||||
"if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n",
|
||||
" BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n",
|
||||
" BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n",
|
||||
" ! gsutil mb -l {REGION} {BUCKET_URI}\n",
|
||||
"else:\n",
|
||||
" assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n",
|
||||
" shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n",
|
||||
" bucket_region = shell_output[0].strip().lower()\n",
|
||||
" if bucket_region != REGION:\n",
|
||||
" raise ValueError(\n",
|
||||
" \"Bucket region %s is different from notebook region %s\"\n",
|
||||
" % (bucket_region, REGION)\n",
|
||||
" )\n",
|
||||
"print(f\"Using this GCS Bucket: {BUCKET_URI}\")\n",
|
||||
"\n",
|
||||
"STAGING_BUCKET = os.path.join(BUCKET_URI, \"temporal\")\n",
|
||||
"MODEL_BUCKET = os.path.join(BUCKET_URI, \"qwen2\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Initialize Vertex AI API.\n",
|
||||
"print(\"Initializing Vertex AI API.\")\n",
|
||||
"aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=STAGING_BUCKET)\n",
|
||||
"\n",
|
||||
"# Gets the default SERVICE_ACCOUNT.\n",
|
||||
"shell_output = ! gcloud projects describe $PROJECT_ID\n",
|
||||
"project_number = shell_output[-1].split(\":\")[1].strip().replace(\"'\", \"\")\n",
|
||||
"SERVICE_ACCOUNT = f\"{project_number}-compute@developer.gserviceaccount.com\"\n",
|
||||
"print(\"Using this default Service Account:\", SERVICE_ACCOUNT)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n",
|
||||
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n",
|
||||
"aiplatform.init(project=PROJECT_ID, location=REGION)\n",
|
||||
"\n",
|
||||
"! gcloud config set project $PROJECT_ID\n",
|
||||
"! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n",
|
||||
"! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\""
|
||||
"\n",
|
||||
"import vertexai\n",
|
||||
"\n",
|
||||
"vertexai.init(\n",
|
||||
" project=PROJECT_ID,\n",
|
||||
" location=REGION,\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -211,20 +174,18 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Deploy\n",
|
||||
"\n",
|
||||
"# @markdown This section uploads prebuilt Qwen2/Qwen2.5 models to Model Registry and deploys it to a Vertex AI Endpoint. It takes 15 to 30 minutes to finish depending on the size of the model.\n",
|
||||
"\n",
|
||||
"MODEL_ID = \"Qwen2.5-0.5B-Instruct\" # @param [\"Qwen2-0.5B-Instruct\", \"Qwen2-1.5B-Instruct\", \"Qwen2-7B-Instruct\", \"Qwen2.5-0.5B-Instruct\", \"Qwen2.5-1.5B-Instruct\", \"Qwen2.5-7B-Instruct\", \"Qwen2.5-14B-Instruct\", \"Qwen2.5-32B-Instruct\"] {isTemplate: true}\n",
|
||||
"model_path_prefix = \"Qwen\"\n",
|
||||
"model_id = os.path.join(model_path_prefix, MODEL_ID)\n",
|
||||
"\n",
|
||||
"# The pre-built serving docker image for vLLM.\n",
|
||||
"PUBLISHER_MODEL_NAME = f\"publishers/qwen/models/qwen2@{MODEL_ID}\"\n",
|
||||
"\n",
|
||||
"VLLM_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-vllm-serve:20241008_0916_RC00\"\n",
|
||||
"\n",
|
||||
"# @markdown Set `use_dedicated_endpoint` to False if you don't want to use [dedicated endpoint](https://cloud.google.com/vertex-ai/docs/general/deployment#create-dedicated-endpoint).\n",
|
||||
"# @markdown Set use_dedicated_endpoint to False if you don't want to use [dedicated endpoint](https://cloud.google.com/vertex-ai/docs/general/deployment#create-dedicated-endpoint). Note that [dedicated endpoint does not support VPC Service Controls](https://cloud.google.com/vertex-ai/docs/predictions/choose-endpoint-type), uncheck the box if you are using VPC-SC.\n",
|
||||
"use_dedicated_endpoint = True # @param {type:\"boolean\"}\n",
|
||||
"accelerator_type = \"NVIDIA_A100_80GB\" # @param [\"NVIDIA_L4\", \"NVIDIA_TESLA_A100\", \"NVIDIA_A100_80GB\"] {isTemplate: true}\n",
|
||||
"\n",
|
||||
"accelerator_type = \"NVIDIA_L4\" # @param [\"NVIDIA_L4\", \"NVIDIA_TESLA_A100\", \"NVIDIA_A100_80GB\"] {isTemplate: true}\n",
|
||||
"vllm_dtype = \"bfloat16\"\n",
|
||||
"max_model_len = 32768 # Maximum context length.\n",
|
||||
"gpu_memory_utilization = 0.85\n",
|
||||
@@ -298,6 +259,45 @@
|
||||
"else:\n",
|
||||
" raise ValueError(\"Invalid model id: %s\" % MODEL_ID)\n",
|
||||
"\n",
|
||||
"# @markdown Click \"Show Code\" to see more details."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "Y8_2LKmZPHQG"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title [Option 1] Deploy with Model Garden SDK\n",
|
||||
"\n",
|
||||
"# @markdown Deploy with Gen AI model-centric SDK. This section uploads the prebuilt model to Model Registry and deploys it to a Vertex AI Endpoint. It takes 15 minutes to 1 hour to finish depending on the size of the model. See [use open models with Vertex AI](https://cloud.google.com/vertex-ai/generative-ai/docs/open-models/use-open-models) for documentation on other use cases.\n",
|
||||
"from vertexai.preview import model_garden\n",
|
||||
"\n",
|
||||
"model = model_garden.OpenModel(PUBLISHER_MODEL_NAME)\n",
|
||||
"endpoints[LABEL] = model.deploy(\n",
|
||||
" machine_type=machine_type,\n",
|
||||
" accelerator_type=accelerator_type,\n",
|
||||
" accelerator_count=accelerator_count,\n",
|
||||
" use_dedicated_endpoint=use_dedicated_endpoint,\n",
|
||||
" accept_eula=True, # Accept the End User License Agreement (EULA) on the model card before deploy. Otherwise, the deployment will be forbidden.\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "GxUAAhJKPHQG"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title [Option 2] Deploy with customized configs\n",
|
||||
"\n",
|
||||
"# @markdown This section uploads prebuilt Qwen2/Qwen2.5 models to Model Registry and deploys it to a Vertex AI Endpoint. It takes 15 to 30 minutes to finish depending on the size of the model.\n",
|
||||
"\n",
|
||||
"common_util.check_quota(\n",
|
||||
" project_id=PROJECT_ID,\n",
|
||||
@@ -313,7 +313,6 @@
|
||||
" model_id: str,\n",
|
||||
" publisher: str,\n",
|
||||
" publisher_model_id: str,\n",
|
||||
" service_account: str,\n",
|
||||
" base_model_id: str = None,\n",
|
||||
" machine_type: str = \"g2-standard-8\",\n",
|
||||
" accelerator_type: str = \"NVIDIA_L4\",\n",
|
||||
@@ -424,7 +423,6 @@
|
||||
" accelerator_type=accelerator_type,\n",
|
||||
" accelerator_count=accelerator_count,\n",
|
||||
" deploy_request_timeout=1800,\n",
|
||||
" service_account=service_account,\n",
|
||||
" system_labels={\n",
|
||||
" \"NOTEBOOK_NAME\": \"model_garden_pytorch_qwen2_deployment.ipynb\",\n",
|
||||
" \"NOTEBOOK_ENVIRONMENT\": common_util.get_deploy_source(),\n",
|
||||
@@ -440,7 +438,6 @@
|
||||
" model_id=model_id,\n",
|
||||
" publisher=\"qwen\",\n",
|
||||
" publisher_model_id=\"qwen2\",\n",
|
||||
" service_account=SERVICE_ACCOUNT,\n",
|
||||
" machine_type=machine_type,\n",
|
||||
" accelerator_type=accelerator_type,\n",
|
||||
" accelerator_count=accelerator_count,\n",
|
||||
@@ -448,9 +445,7 @@
|
||||
" gpu_memory_utilization=gpu_memory_utilization,\n",
|
||||
" dtype=vllm_dtype,\n",
|
||||
" use_dedicated_endpoint=use_dedicated_endpoint,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# @markdown Click \"Show Code\" to see more details."
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -536,16 +531,14 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Deploy\n",
|
||||
"\n",
|
||||
"# @markdown This section uploads prebuilt Qwen2 & Qwen2.5 models to Model Registry and deploys it to a Vertex AI Endpoint. It takes 15 minutes to 1 hour to finish depending on the size of the model.\n",
|
||||
"\n",
|
||||
"# @markdown Select one of the four model variations.\n",
|
||||
"MODEL_ID = \"Qwen2.5-0.5B-Instruct\" # @param [\"Qwen2-0.5B-Instruct\", \"Qwen2-1.5B-Instruct\", \"Qwen2-7B-Instruct\", \"Qwen2.5-0.5B-Instruct\", \"Qwen2.5-1.5B-Instruct\", \"Qwen2.5-7B-Instruct\", \"Qwen2.5-14B-Instruct\", \"Qwen2.5-32B-Instruct-AWQ\"] {isTemplate: true}\n",
|
||||
"TPU_DEPLOYMENT_REGION = \"us-west1\" # @param [\"us-west1\"] {isTemplate:true}\n",
|
||||
"model_path_prefix = \"Qwen\"\n",
|
||||
"model_id = os.path.join(model_path_prefix, MODEL_ID)\n",
|
||||
"\n",
|
||||
"PUBLISHER_MODEL_NAME = f\"publishers/qwen/models/qwen2@{MODEL_ID}\"\n",
|
||||
"\n",
|
||||
"# The pre-built serving docker images.\n",
|
||||
"HEXLLM_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai-restricted/vertex-vision-model-garden-dockers/hex-llm-serve:20241210_2323_RC00\"\n",
|
||||
"\n",
|
||||
@@ -584,7 +577,21 @@
|
||||
"\n",
|
||||
"# Endpoint configurations.\n",
|
||||
"min_replica_count = 1\n",
|
||||
"max_replica_count = 1\n",
|
||||
"max_replica_count = 1"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "ZOqzKtyrPHQH"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Deploy\n",
|
||||
"\n",
|
||||
"# @markdown This section uploads prebuilt Qwen2 & Qwen2.5 models to Model Registry and deploys it to a Vertex AI Endpoint. It takes 15 minutes to 1 hour to finish depending on the size of the model.\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def deploy_model_hexllm(\n",
|
||||
@@ -592,7 +599,6 @@
|
||||
" model_id: str,\n",
|
||||
" publisher: str,\n",
|
||||
" publisher_model_id: str,\n",
|
||||
" service_account: str = None,\n",
|
||||
" base_model_id: str = None,\n",
|
||||
" data_parallel_size: int = 1,\n",
|
||||
" tensor_parallel_size: int = 1,\n",
|
||||
@@ -681,7 +687,6 @@
|
||||
" machine_type=machine_type,\n",
|
||||
" tpu_topology=tpu_topology if num_hosts > 1 else None,\n",
|
||||
" deploy_request_timeout=1800,\n",
|
||||
" service_account=service_account,\n",
|
||||
" min_replica_count=min_replica_count,\n",
|
||||
" max_replica_count=max_replica_count,\n",
|
||||
" system_labels={\n",
|
||||
@@ -697,7 +702,6 @@
|
||||
" model_id=model_id,\n",
|
||||
" publisher=\"qwen\",\n",
|
||||
" publisher_model_id=\"qwen2\",\n",
|
||||
" service_account=SERVICE_ACCOUNT,\n",
|
||||
" tensor_parallel_size=tpu_count,\n",
|
||||
" machine_type=machine_type,\n",
|
||||
" tpu_topology=tpu_topo,\n",
|
||||
@@ -802,11 +806,7 @@
|
||||
"\n",
|
||||
"# Delete models.\n",
|
||||
"for model in models.values():\n",
|
||||
" model.delete()\n",
|
||||
"\n",
|
||||
"delete_bucket = False # @param {type:\"boolean\"}\n",
|
||||
"if delete_bucket:\n",
|
||||
" ! gsutil -m rm -r $BUCKET_NAME"
|
||||
" model.delete()"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
@@ -0,0 +1,782 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "SgQ6t5bqZVlH"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Copyright 2025 Google LLC\n",
|
||||
"#\n",
|
||||
"# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
|
||||
"# you may not use this file except in compliance with the License.\n",
|
||||
"# You may obtain a copy of the License at\n",
|
||||
"#\n",
|
||||
"# https://www.apache.org/licenses/LICENSE-2.0\n",
|
||||
"#\n",
|
||||
"# Unless required by applicable law or agreed to in writing, software\n",
|
||||
"# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
|
||||
"# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
|
||||
"# See the License for the specific language governing permissions and\n",
|
||||
"# limitations under the License."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "99c1c3fc2ca5"
|
||||
},
|
||||
"source": [
|
||||
"# Vertex AI Model Garden - Qwen3 (Deployment)\n",
|
||||
"\n",
|
||||
"<table><tbody><tr>\n",
|
||||
" <td style=\"text-align: center\">\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/workbench/instances\">\n",
|
||||
" <img alt=\"Workbench logo\" src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" width=\"32px\"><br> Run in Workbench\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td style=\"text-align: center\">\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/colab/import/https:%2F%2Fraw.githubusercontent.com%2FGoogleCloudPlatform%2Fvertex-ai-samples%2Fmain%2Fnotebooks%2Fcommunity%2Fmodel_garden%2Fmodel_garden_pytorch_qwen3_deployment.ipynb\">\n",
|
||||
" <img alt=\"Google Cloud Colab Enterprise logo\" src=\"https://lh3.googleusercontent.com/JmcxdQi-qOpctIvWKgPtrzZdJJK-J3sWE1RsfjZNwshCFgE_9fULcNpuXYTilIR2hjwN\" width=\"32px\"><br> Run in Colab Enterprise\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td style=\"text-align: center\">\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_qwen3_deployment.ipynb\">\n",
|
||||
" <img alt=\"GitHub logo\" src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" width=\"32px\"><br> View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
"</tr></tbody></table>"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "3de7470326a2"
|
||||
},
|
||||
"source": [
|
||||
"## Overview\n",
|
||||
"\n",
|
||||
"This notebook demonstrates serving Qwen3 models with [SGLang](https://github.com/sgl-project/sglang). [Qwen3](https://huggingface.co/collections/Qwen/qwen3-67dd247413f0e2e4f653967f) is the latest generation of large language models in Qwen series, offering a comprehensive suite of dense and mixture-of-experts (MoE) models. Built upon extensive training, Qwen3 delivers groundbreaking advancements in reasoning, instruction-following, agent capabilities, and multilingual support\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"### Objective\n",
|
||||
"\n",
|
||||
"- Deploy Qwen3 with SGLang on GPU using single-host serving, and [Spot VMs](https://cloud.google.com/compute/docs/instances/spot) (Optional).\n",
|
||||
"\n",
|
||||
"### File a bug\n",
|
||||
"\n",
|
||||
"File a bug on [GitHub](https://github.com/GoogleCloudPlatform/vertex-ai-samples/issues/new) if you encounter any issue with the notebook.\n",
|
||||
"\n",
|
||||
"### Costs\n",
|
||||
"\n",
|
||||
"This tutorial uses billable components of Google Cloud:\n",
|
||||
"\n",
|
||||
"* Vertex AI\n",
|
||||
"* Cloud Storage\n",
|
||||
"\n",
|
||||
"Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing), [Cloud Storage pricing](https://cloud.google.com/storage/pricing), and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "264c07757582"
|
||||
},
|
||||
"source": [
|
||||
"## Before you begin"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "ax7zWynUDcjk"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Request for quota\n",
|
||||
"\n",
|
||||
"# @markdown To deploy the largest variant of the Qwen3 models, you need 1 host of 8 x H100 machine. Check that you have sufficient quota:\n",
|
||||
"# @markdown - For Spot VM quota, check [`CustomModelServingPreemptibleH100GPUsPerProjectPerRegion`](https://console.cloud.google.com/iam-admin/quotas?metric=aiplatform.googleapis.com%2Fcustom_model_serving_preemptible_nvidia_h100_gpus).\n",
|
||||
"# @markdown - For regular VM quota, check [`CustomModelServingH100GPUsPerProjectPerRegion`](https://console.cloud.google.com/iam-admin/quotas?metric=aiplatform.googleapis.com%2Fcustom_model_serving_nvidia_h100_gpus).\n",
|
||||
"#\n",
|
||||
"# @markdown If you don't have sufficient quota, request for quota following the instructions at [\"Request a higher quota\"](https://cloud.google.com/docs/quota/view-manage#requesting_higher_quota)."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "YXFGIp1l-qtT"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Setup Google Cloud project\n",
|
||||
"\n",
|
||||
"# @markdown 1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
|
||||
"\n",
|
||||
"# @markdown 2. **[Optional]** Set region. If not set, the region will be set automatically according to Colab Enterprise environment.\n",
|
||||
"\n",
|
||||
"REGION = \"\" # @param {type:\"string\"}\n",
|
||||
"\n",
|
||||
"# @markdown 3. If you want to run predictions with A100 80GB or H100 GPUs, we recommend using the regions listed below. **NOTE:** Make sure you have associated quota in selected regions. Click the links to see your current quota for each GPU type: [Nvidia A100 80GB](https://console.cloud.google.com/iam-admin/quotas?metric=aiplatform.googleapis.com%2Fcustom_model_serving_nvidia_a100_80gb_gpus), [Nvidia H100 80GB](https://console.cloud.google.com/iam-admin/quotas?metric=aiplatform.googleapis.com%2Fcustom_model_serving_nvidia_h100_gpus). You can request for quota following the instructions at [\"Request a higher quota\"](https://cloud.google.com/docs/quota/view-manage#requesting_higher_quota).\n",
|
||||
"\n",
|
||||
"# @markdown > | Machine Type | Accelerator Type | Recommended Regions |\n",
|
||||
"# @markdown | ----------- | ----------- | ----------- |\n",
|
||||
"# @markdown | a2-ultragpu-1g | 1 NVIDIA_A100_80GB | us-central1, us-east4, europe-west4, asia-southeast1, us-east4 |\n",
|
||||
"# @markdown | a3-highgpu-2g | 2 NVIDIA_H100_80GB | us-west1, asia-southeast1, europe-west4 |\n",
|
||||
"# @markdown | a3-highgpu-4g | 4 NVIDIA_H100_80GB | us-west1, asia-southeast1, europe-west4 |\n",
|
||||
"# @markdown | a3-highgpu-8g | 8 NVIDIA_H100_80GB | us-central1, europe-west4, us-west1, asia-southeast1 |\n",
|
||||
"\n",
|
||||
"# Upgrade Vertex AI SDK.\n",
|
||||
"! pip3 install --upgrade --quiet 'google-cloud-aiplatform>=1.84.0'\n",
|
||||
"\n",
|
||||
"# Import the necessary packages\n",
|
||||
"import importlib\n",
|
||||
"import os\n",
|
||||
"import time\n",
|
||||
"from typing import Tuple\n",
|
||||
"\n",
|
||||
"import requests\n",
|
||||
"from google import auth\n",
|
||||
"from google.cloud import aiplatform\n",
|
||||
"\n",
|
||||
"# Upgrade Vertex AI SDK.\n",
|
||||
"if os.environ.get(\"VERTEX_PRODUCT\") != \"COLAB_ENTERPRISE\":\n",
|
||||
" ! pip install --upgrade tensorflow\n",
|
||||
"! git clone https://github.com/GoogleCloudPlatform/vertex-ai-samples.git\n",
|
||||
"\n",
|
||||
"common_util = importlib.import_module(\n",
|
||||
" \"vertex-ai-samples.community-content.vertex_model_garden.model_oss.notebook_util.common_util\"\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def check_quota(\n",
|
||||
" project_id: str,\n",
|
||||
" region: str,\n",
|
||||
" resource_id: str,\n",
|
||||
" accelerator_count: int,\n",
|
||||
"):\n",
|
||||
" \"\"\"Checks if the project and the region has the required quota.\"\"\"\n",
|
||||
" quota = common_util.get_quota(project_id, region, resource_id)\n",
|
||||
" quota_request_instruction = (\n",
|
||||
" \"Either use \"\n",
|
||||
" \"a different region or request additional quota. Follow \"\n",
|
||||
" \"instructions here \"\n",
|
||||
" \"https://cloud.google.com/docs/quotas/view-manage#requesting_higher_quota\"\n",
|
||||
" \" to check quota in a region or request additional quota for \"\n",
|
||||
" \"your project.\"\n",
|
||||
" )\n",
|
||||
" if quota == -1:\n",
|
||||
" raise ValueError(\n",
|
||||
" f\"Quota not found for: {resource_id} in {region}.\"\n",
|
||||
" f\" {quota_request_instruction}\"\n",
|
||||
" )\n",
|
||||
" if quota < accelerator_count:\n",
|
||||
" raise ValueError(\n",
|
||||
" f\"Quota not enough for {resource_id} in {region}: {quota} <\"\n",
|
||||
" f\" {accelerator_count}. {quota_request_instruction}\"\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"LABEL = \"sglang_gpu\"\n",
|
||||
"models, endpoints = {}, {}\n",
|
||||
"\n",
|
||||
"# Get the default cloud project id.\n",
|
||||
"PROJECT_ID = os.environ[\"GOOGLE_CLOUD_PROJECT\"]\n",
|
||||
"\n",
|
||||
"# Get the default region for launching jobs.\n",
|
||||
"if not REGION:\n",
|
||||
" REGION = os.environ[\"GOOGLE_CLOUD_REGION\"]\n",
|
||||
"\n",
|
||||
"# Initialize Vertex AI API.\n",
|
||||
"print(\"Initializing Vertex AI API.\")\n",
|
||||
"aiplatform.init(project=PROJECT_ID, location=REGION)\n",
|
||||
"\n",
|
||||
"! gcloud config set project $PROJECT_ID\n",
|
||||
"\n",
|
||||
"import vertexai\n",
|
||||
"\n",
|
||||
"vertexai.init(\n",
|
||||
" project=PROJECT_ID,\n",
|
||||
" location=REGION,\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "3zJJDmldn7rw"
|
||||
},
|
||||
"source": [
|
||||
"## Deploy Qwen3 with SGLang"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "_3Swj3pxn7rw"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Select the model variants\n",
|
||||
"\n",
|
||||
"# @markdown Set the model to deploy.\n",
|
||||
"\n",
|
||||
"base_model_name = \"Qwen3-235B-A22B\" # @param [\"Qwen3-235B-A22B\", \"Qwen3-235B-A22B-FP8\", \"Qwen3-30B-A3B\", \"Qwen3-30B-A3B-Base\", \"Qwen/Qwen3-30B-A3B-FP8\", \"Qwen3-32B\", \"Qwen/Qwen3-32B-FP8\", \"Qwen3-14B\", \"Qwen3-14B-Base\", \"Qwen/Qwen3-14B-FP8\", \"Qwen3-8B\", \"Qwen3-8B-Base\", \"Qwen/Qwen3-8B-FP8\", \"Qwen3-4B\", \"Qwen3-4B-Base\", \"Qwen/Qwen3-4B-FP8\", \"Qwen3-1.7B\", \"Qwen3-1.7B-Base\", \"Qwen/Qwen3-1.7B-FP8\", \"Qwen3-0.6B\", \"Qwen3-0.6B-Base\", \"Qwen/Qwen3-0.6B-FP8\"] {isTemplate:true}\n",
|
||||
"model_id = \"Qwen/\" + base_model_name\n",
|
||||
"hf_model_id = model_id\n",
|
||||
"\n",
|
||||
"# The pre-built serving docker images.\n",
|
||||
"SGLANG_DOCKER_URI = \"us-docker.pkg.dev/deeplearning-platform-release/vertex-model-garden/sglang-serve.cu124.0-4.ubuntu2204.py310:20250428-1803-rc0\"\n",
|
||||
"\n",
|
||||
"# @markdown Choose whether to use a [Spot VM](https://cloud.google.com/compute/docs/instances/spot) for the deployment.\n",
|
||||
"is_spot = False # @param {type:\"boolean\"}\n",
|
||||
"\n",
|
||||
"# @markdown Set use_dedicated_endpoint to False if you don't want to use [dedicated endpoint](https://cloud.google.com/vertex-ai/docs/general/deployment#create-dedicated-endpoint). Note that [dedicated endpoint does not support VPC Service Controls](https://cloud.google.com/vertex-ai/docs/predictions/choose-endpoint-type), uncheck the box if you are using VPC-SC.\n",
|
||||
"use_dedicated_endpoint = True # @param {type:\"boolean\"}\n",
|
||||
"\n",
|
||||
"# @markdown Find Vertex AI prediction supported accelerators and regions at https://cloud.google.com/vertex-ai/docs/predictions/configure-compute.\n",
|
||||
"accelerator_type = \"NVIDIA_H100_80GB\" # @param [\"NVIDIA_H100_80GB\", \"NVIDIA_L4\"] {isTemplate:true}\n",
|
||||
"\n",
|
||||
"PUBLISHER_MODEL_NAME = f\"publishers/qwen/models/qwen3@{base_model_name.lower()}\"\n",
|
||||
"\n",
|
||||
"if accelerator_type == \"NVIDIA_H100_80GB\":\n",
|
||||
" if is_spot:\n",
|
||||
" resource_id = \"custom_model_serving_preemptible_nvidia_h100_gpus\"\n",
|
||||
" else:\n",
|
||||
" resource_id = \"custom_model_serving_nvidia_h100_gpus\"\n",
|
||||
" if base_model_name in [\"Qwen3-235B-A22B\"]:\n",
|
||||
" machine_type = \"a3-highgpu-8g\"\n",
|
||||
" accelerator_count = 8\n",
|
||||
" model_id = \"gs://vertex-model-garden-restricted-us/qwen3/Qwen3-235B-A22B\"\n",
|
||||
" elif base_model_name in [\"Qwen3-235B-A22B-FP8\"]:\n",
|
||||
" machine_type = \"a3-highgpu-4g\"\n",
|
||||
" accelerator_count = 4\n",
|
||||
" else:\n",
|
||||
" machine_type = \"a3-highgpu-1g\"\n",
|
||||
" accelerator_count = 1\n",
|
||||
"elif accelerator_type == \"NVIDIA_L4\":\n",
|
||||
" if is_spot:\n",
|
||||
" resource_id = \"custom_model_serving_preemptible_nvidia_l4_gpus\"\n",
|
||||
" else:\n",
|
||||
" resource_id = \"custom_model_serving_nvidia_l4_gpus\"\n",
|
||||
" if base_model_name in [\"Qwen3-235B-A22B\", \"Qwen3-235B-A22B-FP8\"]:\n",
|
||||
" raise ValueError(\"L4s are insufficient to server Qwen3-235B-A22B.\")\n",
|
||||
" elif base_model_name in [\"Qwen3-30B-A3B\", \"Qwen3-30B-A3B-Base\", \"Qwen3-32B\"]:\n",
|
||||
" machine_type = \"g2-standard-48\"\n",
|
||||
" accelerator_count = 4\n",
|
||||
" elif base_model_name in [\n",
|
||||
" \"Qwen3-14B\",\n",
|
||||
" \"Qwen3-14B-Base\",\n",
|
||||
" \"Qwen3-30B-A3B-FP8\",\n",
|
||||
" \"Qwen3-32B-FP8\",\n",
|
||||
" ]:\n",
|
||||
" machine_type = \"g2-standard-24\"\n",
|
||||
" accelerator_count = 2\n",
|
||||
" elif base_model_name in [\n",
|
||||
" \"Qwen3-8B\",\n",
|
||||
" \"Qwen3-8B-Base\",\n",
|
||||
" \"Qwen3-4B\",\n",
|
||||
" \"Qwen3-4B-Base\",\n",
|
||||
" \"Qwen3-1.7B\",\n",
|
||||
" \"Qwen3-1.7B-Base\",\n",
|
||||
" \"Qwen3-0.6B\",\n",
|
||||
" \"Qwen3-0.6B-Base\",\n",
|
||||
" \"Qwen/Qwen3-14B-FP8\",\n",
|
||||
" \"Qwen/Qwen3-8B-FP8\",\n",
|
||||
" \"Qwen/Qwen3-4B-FP8\",\n",
|
||||
" \"Qwen/Qwen3-1.7B-FP8\",\n",
|
||||
" \"Qwen/Qwen3-0.6B-FP8\",\n",
|
||||
" ]:\n",
|
||||
" machine_type = \"g2-standard-12\"\n",
|
||||
" accelerator_count = 1\n",
|
||||
" else:\n",
|
||||
" raise ValueError(f\"Recommended GPU setting not found for: {base_model_name}.\")\n",
|
||||
"else:\n",
|
||||
" raise ValueError(f\"Recommended GPU setting not found for: {base_model_name}.\")\n",
|
||||
"\n",
|
||||
"check_quota(\n",
|
||||
" project_id=PROJECT_ID,\n",
|
||||
" region=REGION,\n",
|
||||
" resource_id=resource_id,\n",
|
||||
" accelerator_count=accelerator_count,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# @markdown Click \"Show Code\" to see more details."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "omW0LaC8wWz5"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title [Option 1] Deploy with Model Garden SDK\n",
|
||||
"# @markdown Deploy with Gen AI model-centric SDK. This section uploads the prebuilt model to Model Registry and deploys it to a Vertex AI Endpoint. It takes 15 minutes to 1 hour to finish depending on the size of the model. See [use open models with Vertex AI](https://cloud.google.com/vertex-ai/generative-ai/docs/open-models/use-open-models) for documentation on other use cases.\n",
|
||||
"deploy_request_timeout = 1800 # 30 minutes\n",
|
||||
"from vertexai.preview import model_garden\n",
|
||||
"\n",
|
||||
"model = model_garden.OpenModel(PUBLISHER_MODEL_NAME)\n",
|
||||
"endpoints[LABEL] = model.deploy(\n",
|
||||
" machine_type=machine_type,\n",
|
||||
" accelerator_type=accelerator_type,\n",
|
||||
" accelerator_count=accelerator_count,\n",
|
||||
" use_dedicated_endpoint=use_dedicated_endpoint,\n",
|
||||
" spot=is_spot,\n",
|
||||
" deploy_request_timeout=deploy_request_timeout,\n",
|
||||
" accept_eula=False,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# @markdown Click \"Show Code\" to see more details."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "3m-tDxgawYhU"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title [Option 2] Deploy with customized configs\n",
|
||||
"\n",
|
||||
"# @markdown This section uploads Qwen3 models to Model Registry and deploys them to a Vertex Prediction Endpoint. It takes ~1 hour to finish.\n",
|
||||
"\n",
|
||||
"# @markdown It's recommended to use the region selected by the deployment button on the model card. If the deployment button is not available, it's recommended to stay with the default region of the notebook.\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def poll_operation(op_name: str) -> bool: # noqa: F811\n",
|
||||
" creds, _ = auth.default()\n",
|
||||
" auth_req = auth.transport.requests.Request()\n",
|
||||
" creds.refresh(auth_req)\n",
|
||||
" headers = {\n",
|
||||
" \"Authorization\": f\"Bearer {creds.token}\",\n",
|
||||
" }\n",
|
||||
" get_resp = requests.get(\n",
|
||||
" f\"https://{REGION}-aiplatform.googleapis.com/ui/{op_name}\",\n",
|
||||
" headers=headers,\n",
|
||||
" )\n",
|
||||
" opjs = get_resp.json()\n",
|
||||
" if \"error\" in opjs:\n",
|
||||
" raise ValueError(f\"Operation failed: {opjs['error']}\")\n",
|
||||
" return opjs.get(\"done\", False)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def poll_and_wait(op_name: str, total_wait: int, interval: int = 60): # noqa: F811\n",
|
||||
" waited = 0\n",
|
||||
" while not poll_operation(op_name):\n",
|
||||
" if waited > total_wait:\n",
|
||||
" raise TimeoutError(\"Operation timed out\")\n",
|
||||
" print(\n",
|
||||
" f\"\\rStill waiting for operation... Waited time in second: {waited:<6}\",\n",
|
||||
" end=\"\",\n",
|
||||
" flush=True,\n",
|
||||
" )\n",
|
||||
" waited += interval\n",
|
||||
" time.sleep(interval)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def deploy_model_sglang_multihost(\n",
|
||||
" model_name: str,\n",
|
||||
" model_id: str,\n",
|
||||
" publisher: str,\n",
|
||||
" publisher_model_id: str,\n",
|
||||
" service_account: str = \"\",\n",
|
||||
" base_model_id: str = \"\",\n",
|
||||
" machine_type: str = \"g2-standard-8\",\n",
|
||||
" accelerator_type: str = \"NVIDIA_L4\",\n",
|
||||
" accelerator_count: int = 1,\n",
|
||||
" multihost_gpu_node_count: int = 1,\n",
|
||||
" gpu_memory_utilization: float | None = None,\n",
|
||||
" context_length: int | None = None,\n",
|
||||
" dtype: str | None = None,\n",
|
||||
" enable_trust_remote_code: bool = False,\n",
|
||||
" enable_torch_compile: bool = False,\n",
|
||||
" torch_compile_max_bs: int | None = None,\n",
|
||||
" attention_backend: str = \"\",\n",
|
||||
" enable_flashinfer_mla: bool = False,\n",
|
||||
" disable_cuda_graph: bool = False,\n",
|
||||
" speculative_algorithm: str | None = None,\n",
|
||||
" speculative_draft_model_path: str = \"\",\n",
|
||||
" speculative_num_steps: int = 3,\n",
|
||||
" speculative_eagle_topk: int = 1,\n",
|
||||
" speculative_num_draft_tokens: int = 4,\n",
|
||||
" enable_jit_deepgemm: bool = False,\n",
|
||||
" enable_dp_attention: bool = False,\n",
|
||||
" dp_size: int = 1,\n",
|
||||
" use_dedicated_endpoint: bool = False,\n",
|
||||
" max_num_seqs: int | None = None,\n",
|
||||
" is_spot: bool = True,\n",
|
||||
") -> Tuple[aiplatform.Model, aiplatform.Endpoint]:\n",
|
||||
" \"\"\"Deploys trained models with SGLang into Vertex AI.\"\"\"\n",
|
||||
" endpoint = aiplatform.Endpoint.create(\n",
|
||||
" display_name=f\"{model_name}-endpoint\",\n",
|
||||
" dedicated_endpoint_enabled=use_dedicated_endpoint,\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" if not base_model_id:\n",
|
||||
" base_model_id = model_id\n",
|
||||
"\n",
|
||||
" # See https://docs.sglang.ai/backend/server_arguments.html for a list of possible arguments with descriptions.\n",
|
||||
" sglang_args = [\n",
|
||||
" f\"--model={model_id}\",\n",
|
||||
" f\"--tp={accelerator_count * multihost_gpu_node_count}\",\n",
|
||||
" f\"--dp={dp_size}\",\n",
|
||||
" ]\n",
|
||||
"\n",
|
||||
" if context_length:\n",
|
||||
" sglang_args.append(f\"--context-length={context_length}\")\n",
|
||||
"\n",
|
||||
" if gpu_memory_utilization:\n",
|
||||
" sglang_args.append(f\"--mem-fraction-static={gpu_memory_utilization}\")\n",
|
||||
"\n",
|
||||
" if max_num_seqs:\n",
|
||||
" sglang_args.append(f\"--max-running-requests={max_num_seqs}\")\n",
|
||||
"\n",
|
||||
" if dtype:\n",
|
||||
" sglang_args.append(f\"--dtype={dtype}\")\n",
|
||||
"\n",
|
||||
" if enable_trust_remote_code:\n",
|
||||
" sglang_args.append(\"--trust-remote-code\")\n",
|
||||
"\n",
|
||||
" if enable_torch_compile:\n",
|
||||
" sglang_args.append(\"--enable-torch-compile\")\n",
|
||||
" if torch_compile_max_bs:\n",
|
||||
" sglang_args.append(f\"--torch-compile-max-bs={torch_compile_max_bs}\")\n",
|
||||
"\n",
|
||||
" if attention_backend:\n",
|
||||
" sglang_args.append(f\"--attention-backend={attention_backend}\")\n",
|
||||
"\n",
|
||||
" if enable_flashinfer_mla:\n",
|
||||
" sglang_args.append(\"--enable-flashinfer-mla\")\n",
|
||||
"\n",
|
||||
" if disable_cuda_graph:\n",
|
||||
" sglang_args.append(\"--disable-cuda-graph\")\n",
|
||||
"\n",
|
||||
" if speculative_algorithm:\n",
|
||||
" sglang_args.append(f\"--speculative-algorithm={speculative_algorithm}\")\n",
|
||||
" sglang_args.append(\n",
|
||||
" f\"--speculative-draft-model-path={speculative_draft_model_path}\"\n",
|
||||
" )\n",
|
||||
" sglang_args.append(f\"--speculative-num-steps={speculative_num_steps}\")\n",
|
||||
" sglang_args.append(f\"--speculative-eagle-topk={speculative_eagle_topk}\")\n",
|
||||
" sglang_args.append(\n",
|
||||
" f\"--speculative-num-draft-tokens={speculative_num_draft_tokens}\"\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" if enable_dp_attention:\n",
|
||||
" sglang_args.append(\"--enable-dp-attention\")\n",
|
||||
"\n",
|
||||
" env_vars = {\n",
|
||||
" \"MODEL_ID\": base_model_id,\n",
|
||||
" \"DEPLOY_SOURCE\": \"notebook\",\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" if enable_jit_deepgemm:\n",
|
||||
" env_vars[\"SGL_ENABLE_JIT_DEEPGEMM\"] = \"1\"\n",
|
||||
"\n",
|
||||
" # HF_TOKEN is not a compulsory field and may not be defined.\n",
|
||||
" try:\n",
|
||||
" if HF_TOKEN:\n",
|
||||
" env_vars[\"HF_TOKEN\"] = HF_TOKEN\n",
|
||||
" except NameError:\n",
|
||||
" pass\n",
|
||||
"\n",
|
||||
" model = aiplatform.Model.upload(\n",
|
||||
" display_name=model_name,\n",
|
||||
" serving_container_image_uri=SGLANG_DOCKER_URI,\n",
|
||||
" serving_container_args=sglang_args,\n",
|
||||
" serving_container_ports=[30000],\n",
|
||||
" serving_container_predict_route=\"/vertex_generate\",\n",
|
||||
" serving_container_health_route=\"/health\",\n",
|
||||
" serving_container_environment_variables=env_vars,\n",
|
||||
" serving_container_shared_memory_size_mb=(16 * 1024), # 16 GB\n",
|
||||
" serving_container_deployment_timeout=7200,\n",
|
||||
" model_garden_source_model_name=(\n",
|
||||
" f\"publishers/{publisher}/models/{publisher_model_id}\"\n",
|
||||
" ),\n",
|
||||
" )\n",
|
||||
" print(\n",
|
||||
" f\"Deploying {model_name} on {machine_type} with {int(accelerator_count * multihost_gpu_node_count)} {accelerator_type} GPU(s).\"\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" creds, _ = auth.default()\n",
|
||||
" auth_req = auth.transport.requests.Request()\n",
|
||||
" creds.refresh(auth_req)\n",
|
||||
"\n",
|
||||
" url = f\"https://{REGION}-aiplatform.googleapis.com/ui/projects/{PROJECT_ID}/locations/{REGION}/endpoints/{endpoint.name}:deployModel\"\n",
|
||||
" headers = {\n",
|
||||
" \"Content-Type\": \"application/json\",\n",
|
||||
" \"Authorization\": f\"Bearer {creds.token}\",\n",
|
||||
" }\n",
|
||||
" data = {\n",
|
||||
" \"deployedModel\": {\n",
|
||||
" \"model\": model.resource_name,\n",
|
||||
" \"displayName\": model_name,\n",
|
||||
" \"dedicatedResources\": {\n",
|
||||
" \"machineSpec\": {\n",
|
||||
" \"machineType\": machine_type,\n",
|
||||
" \"multihostGpuNodeCount\": multihost_gpu_node_count,\n",
|
||||
" \"acceleratorType\": accelerator_type,\n",
|
||||
" \"acceleratorCount\": accelerator_count,\n",
|
||||
" },\n",
|
||||
" \"minReplicaCount\": 1,\n",
|
||||
" \"maxReplicaCount\": 1,\n",
|
||||
" },\n",
|
||||
" \"system_labels\": {\n",
|
||||
" \"NOTEBOOK_NAME\": \"model_garden_pytorch_qwen3_deployment.ipynb\",\n",
|
||||
" \"NOTEBOOK_ENVIRONMENT\": common_util.get_deploy_source(),\n",
|
||||
" },\n",
|
||||
" },\n",
|
||||
" }\n",
|
||||
" if service_account:\n",
|
||||
" data[\"deployedModel\"][\"serviceAccount\"] = service_account\n",
|
||||
" if is_spot:\n",
|
||||
" data[\"deployedModel\"][\"dedicatedResources\"][\"spot\"] = True\n",
|
||||
" response = requests.post(url, headers=headers, json=data)\n",
|
||||
" print(f\"Deploy Model response: {response.json()}\")\n",
|
||||
" if response.status_code != 200 or \"name\" not in response.json():\n",
|
||||
" raise ValueError(f\"Failed to deploy model: {response.text}\")\n",
|
||||
" poll_and_wait(response.json()[\"name\"], 7200)\n",
|
||||
" print(\"endpoint_name:\", endpoint.name)\n",
|
||||
"\n",
|
||||
" return model, endpoint\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"models[LABEL], endpoints[LABEL] = deploy_model_sglang_multihost(\n",
|
||||
" model_name=common_util.get_job_name_with_datetime(prefix=\"qwen3-serve\"),\n",
|
||||
" model_id=model_id,\n",
|
||||
" publisher=\"qwen\",\n",
|
||||
" publisher_model_id=\"qwen3\",\n",
|
||||
" base_model_id=hf_model_id,\n",
|
||||
" machine_type=machine_type,\n",
|
||||
" accelerator_type=accelerator_type,\n",
|
||||
" accelerator_count=accelerator_count,\n",
|
||||
" use_dedicated_endpoint=use_dedicated_endpoint,\n",
|
||||
" is_spot=is_spot,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# @markdown Click \"Show Code\" to see more details."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "AGVPzwHkn7rw"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Raw predict\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# @markdown Once deployment succeeds, you can send requests to the endpoint with text prompts. Sampling parameters supported by SGLang can be found [here](https://docs.sglang.ai/backend/sampling_params.html).\n",
|
||||
"\n",
|
||||
"# @markdown Example:\n",
|
||||
"\n",
|
||||
"# @markdown ```\n",
|
||||
"# @markdown User: What is the best way to diagnose and fix a flickering light in my house?\n",
|
||||
"# @markdown Assistant: Okay, the user is asking about the best way to diagnose and fix a flickering light in their house. Let me start by breaking down the steps. First, I need to consider the possible causes of flickering. Common issues could be a faulty light bulb, a blown fuse, a short circuit, or a problem with the circuit itself.\n",
|
||||
"# @markdown ```\n",
|
||||
"# @markdown Additionally, you can moderate the generated text with Vertex AI. See [Moderate text documentation](https://cloud.google.com/natural-language/docs/moderating-text) for more details.\n",
|
||||
"\n",
|
||||
"# Loads an existing endpoint instance using the endpoint name:\n",
|
||||
"# - Using `endpoint_name = endpoint.name` allows us to get the\n",
|
||||
"# endpoint name of the endpoint `endpoint` created in the cell\n",
|
||||
"# above.\n",
|
||||
"# - Alternatively, you can set `endpoint_name = \"1234567890123456789\"` to load\n",
|
||||
"# an existing endpoint with the ID 1234567890123456789.\n",
|
||||
"# You may uncomment the code below to load an existing endpoint.\n",
|
||||
"\n",
|
||||
"# endpoint_name = \"\" # @param {type:\"string\"}\n",
|
||||
"# aip_endpoint_name = (\n",
|
||||
"# f\"projects/{PROJECT_ID}/locations/{REGION}/endpoints/{endpoint_name}\"\n",
|
||||
"# )\n",
|
||||
"# endpoint = aiplatform.Endpoint(aip_endpoint_name)\n",
|
||||
"\n",
|
||||
"# @markdown A chat template formatted prompt for Qwen3 models are shown below as an example.\n",
|
||||
"prompt = \"<|im_start|>user What is the best way to diagnose and fix a flickering light in my house?<|im_end|><|im_start|>assistant\" # @param {type: \"string\"}\n",
|
||||
"\n",
|
||||
"# @markdown By default, Qwen3 has thinking capabilities enabled, similar to QwQ-32B. This means the model will use its reasoning abilities to enhance the quality of generated responses.\n",
|
||||
"# @markdown The model will generate think content wrapped in a \\<think>...\\</think> block, followed by the final response.\n",
|
||||
"# @markdown `max_new_tokens` may need to be increased to accommodate the additional think content.\n",
|
||||
"enable_thinking = True # @param {type:\"boolean\"}\n",
|
||||
"if not enable_thinking:\n",
|
||||
" prompt += \"<think></think>\"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"max_new_tokens = 1024 # @param {type:\"integer\"}\n",
|
||||
"\n",
|
||||
"# @markdown Sampling parameters such as `temperature` and `top_p` are automatically set according to [Qwen3's best practices](https://huggingface.co/Qwen/Qwen3-30B-A3B#best-practices).\n",
|
||||
"if enable_thinking:\n",
|
||||
" temperature = 0.6\n",
|
||||
" top_p = 0.95\n",
|
||||
" top_k = 20\n",
|
||||
" min_p = 0\n",
|
||||
"else:\n",
|
||||
" temperature = 0.7\n",
|
||||
" top_p = 0.8\n",
|
||||
" top_k = 20\n",
|
||||
" min_p = 0\n",
|
||||
"\n",
|
||||
"# @markdown If you encounter an issue like `ServiceUnavailable: 503 Took too long to respond when processing`, you can reduce the maximum number of output tokens, by lowering `max_new_tokens`.\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Overrides parameters for inferences.\n",
|
||||
"instances = [{\"text\": prompt}]\n",
|
||||
"parameters = {\n",
|
||||
" \"sampling_params\": {\n",
|
||||
" \"max_new_tokens\": max_new_tokens,\n",
|
||||
" \"temperature\": temperature,\n",
|
||||
" \"top_p\": top_p,\n",
|
||||
" \"top_k\": top_k,\n",
|
||||
" \"min_p\": min_p,\n",
|
||||
" }\n",
|
||||
"}\n",
|
||||
"response = endpoints[\"sglang_gpu\"].predict(\n",
|
||||
" instances=instances,\n",
|
||||
" parameters=parameters,\n",
|
||||
" use_dedicated_endpoint=use_dedicated_endpoint,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"for prediction in response.predictions:\n",
|
||||
" print(prediction)\n",
|
||||
"\n",
|
||||
"# @markdown Click \"Show Code\" to see more details."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "ZauMzfXJzAKZ"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Chat completion\n",
|
||||
"\n",
|
||||
"if use_dedicated_endpoint:\n",
|
||||
" DEDICATED_ENDPOINT_DNS = endpoints[\"sglang_gpu\"].gca_resource.dedicated_endpoint_dns\n",
|
||||
"ENDPOINT_RESOURCE_NAME = endpoints[\"sglang_gpu\"].resource_name\n",
|
||||
"\n",
|
||||
"# @markdown Because the Qwen3 models generate detailed reasoning steps, the output is expected to be long. We recommend using streaming for a better generation experience.\n",
|
||||
"# @title Chat Completions Inference\n",
|
||||
"\n",
|
||||
"# @markdown Once deployment succeeds, you can send requests to the endpoint using the OpenAI SDK.\n",
|
||||
"\n",
|
||||
"# @markdown First you will need to install the SDK and some auth-related dependencies.\n",
|
||||
"\n",
|
||||
"! pip install -qU openai google-auth requests\n",
|
||||
"\n",
|
||||
"# @markdown Next fill out some request parameters:\n",
|
||||
"\n",
|
||||
"user_message = \"How is your day going?\" # @param {type: \"string\"}\n",
|
||||
"# @markdown If you encounter the issue like `ServiceUnavailable: 503 Took too long to respond when processing`, you can reduce the maximum number of output tokens, such as set `max_tokens` as 20.\n",
|
||||
"max_tokens = 50 # @param {type: \"integer\"}\n",
|
||||
"temperature = 1.0 # @param {type: \"number\"}\n",
|
||||
"stream = False # @param {type: \"boolean\"}\n",
|
||||
"\n",
|
||||
"# @markdown Now we can send a request.\n",
|
||||
"\n",
|
||||
"import google.auth\n",
|
||||
"import openai\n",
|
||||
"\n",
|
||||
"creds, project = google.auth.default()\n",
|
||||
"auth_req = google.auth.transport.requests.Request()\n",
|
||||
"creds.refresh(auth_req)\n",
|
||||
"\n",
|
||||
"BASE_URL = (\n",
|
||||
" f\"https://{REGION}-aiplatform.googleapis.com/v1beta1/{ENDPOINT_RESOURCE_NAME}\"\n",
|
||||
")\n",
|
||||
"try:\n",
|
||||
" if use_dedicated_endpoint:\n",
|
||||
" BASE_URL = f\"https://{DEDICATED_ENDPOINT_DNS}/v1beta1/{ENDPOINT_RESOURCE_NAME}\"\n",
|
||||
"except NameError:\n",
|
||||
" pass\n",
|
||||
"\n",
|
||||
"client = openai.OpenAI(base_url=BASE_URL, api_key=creds.token)\n",
|
||||
"\n",
|
||||
"model_response = client.chat.completions.create(\n",
|
||||
" model=\"\",\n",
|
||||
" messages=[{\"role\": \"user\", \"content\": user_message}],\n",
|
||||
" temperature=temperature,\n",
|
||||
" max_tokens=max_tokens,\n",
|
||||
" stream=stream,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"if stream:\n",
|
||||
" usage = None\n",
|
||||
" contents = []\n",
|
||||
" for chunk in model_response:\n",
|
||||
" if chunk.usage is not None:\n",
|
||||
" usage = chunk.usage\n",
|
||||
" continue\n",
|
||||
" print(chunk.choices[0].delta.content, end=\"\")\n",
|
||||
" contents.append(chunk.choices[0].delta.content)\n",
|
||||
" print(f\"\\n\\n{usage}\")\n",
|
||||
"else:\n",
|
||||
" print(model_response)\n",
|
||||
"\n",
|
||||
"# @markdown Click \"Show Code\" to see more details."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "JETd33jIDcjm"
|
||||
},
|
||||
"source": [
|
||||
"## Clean up resources"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "911406c1561e"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Delete the models and endpoints\n",
|
||||
"\n",
|
||||
"# @markdown Delete the experiment models and endpoints to recycle the resources\n",
|
||||
"# @markdown and avoid unnecessary continuous charges that may incur.\n",
|
||||
"\n",
|
||||
"# Undeploy model and delete endpoint.\n",
|
||||
"for endpoint in endpoints.values():\n",
|
||||
" endpoint.delete(force=True)\n",
|
||||
"\n",
|
||||
"# Delete models.\n",
|
||||
"for model in models.values():\n",
|
||||
" model.delete()"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"name": "model_garden_pytorch_qwen3_deployment.ipynb",
|
||||
"toc_visible": true
|
||||
},
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"name": "python3"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 0
|
||||
}
|
||||
@@ -34,6 +34,11 @@
|
||||
"\n",
|
||||
"<table><tbody><tr>\n",
|
||||
" <td style=\"text-align: center\">\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/workbench/instances\">\n",
|
||||
" <img alt=\"Workbench logo\" src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" width=\"32px\"><br> Run in Workbench\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td style=\"text-align: center\">\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/colab/import/https:%2F%2Fraw.githubusercontent.com%2FGoogleCloudPlatform%2Fvertex-ai-samples%2Fmain%2Fnotebooks%2Fcommunity%2Fmodel_garden%2Fmodel_garden_pytorch_sam.ipynb\">\n",
|
||||
" <img alt=\"Google Cloud Colab Enterprise logo\" src=\"https://lh3.googleusercontent.com/JmcxdQi-qOpctIvWKgPtrzZdJJK-J3sWE1RsfjZNwshCFgE_9fULcNpuXYTilIR2hjwN\" width=\"32px\"><br> Run in Colab Enterprise\n",
|
||||
" </a>\n",
|
||||
@@ -67,6 +72,10 @@
|
||||
"- Deploy the model on [Endpoint](https://cloud.google.com/vertex-ai/docs/predictions/using-private-endpoints).\n",
|
||||
"- Run online predictions for image captioning.\n",
|
||||
"\n",
|
||||
"### File a bug\n",
|
||||
"\n",
|
||||
"File a bug on [GitHub](https://github.com/GoogleCloudPlatform/vertex-ai-samples/issues/new) if you encounter any issue with the notebook.\n",
|
||||
"\n",
|
||||
"### Costs\n",
|
||||
"\n",
|
||||
"This tutorial uses billable components of Google Cloud:\n",
|
||||
@@ -99,42 +108,32 @@
|
||||
"\n",
|
||||
"# @markdown 1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
|
||||
"\n",
|
||||
"# @markdown 2. For finetuning, **[click here](https://console.cloud.google.com/iam-admin/quotas?location=us-central1&metric=aiplatform.googleapis.com%2Frestricted_image_training_nvidia_a100_80gb_gpus)** to check if your project already has the required 8 Nvidia A100 80 GB GPUs in the us-central1 region. If yes, then run this notebook in the us-central1 region. If you do not have 8 Nvidia A100 80 GPUs or have more GPU requirements than this, then schedule your job with Nvidia H100 GPUs via Dynamic Workload Scheduler using [these instructions](https://cloud.google.com/vertex-ai/docs/training/schedule-jobs-dws). For Dynamic Workload Scheduler, check the [us-central1](https://console.cloud.google.com/iam-admin/quotas?location=us-central1&metric=aiplatform.googleapis.com%2Fcustom_model_training_preemptible_nvidia_h100_gpus) or [europe-west4](https://console.cloud.google.com/iam-admin/quotas?location=europe-west4&metric=aiplatform.googleapis.com%2Fcustom_model_training_preemptible_nvidia_h100_gpus) quota for Nvidia H100 GPUs. If you do not have enough GPUs, then you can follow [these instructions](https://cloud.google.com/docs/quotas/view-manage#viewing_your_quota_console) to request quota.\n",
|
||||
"\n",
|
||||
"# @markdown 3. For serving, **[click here](https://console.cloud.google.com/iam-admin/quotas?location=us-central1&metric=aiplatform.googleapis.com%2Fcustom_model_serving_nvidia_l4_gpus)** to check if your project already has the required 1 L4 GPU in the us-central1 region. If yes, then run this notebook in the us-central1 region. If you need more L4 GPUs for your project, then you can follow [these instructions](https://cloud.google.com/docs/quotas/view-manage#viewing_your_quota_console) to request more. Alternatively, if you want to run predictions with A100 80GB or H100 GPUs, we recommend using the regions listed below. **NOTE:** Make sure you have associated quota in selected regions. Click the links to see your current quota for each GPU type: [Nvidia A100 80GB](https://console.cloud.google.com/iam-admin/quotas?metric=aiplatform.googleapis.com%2Fcustom_model_serving_nvidia_a100_80gb_gpus), [Nvidia H100 80GB](https://console.cloud.google.com/iam-admin/quotas?metric=aiplatform.googleapis.com%2Fcustom_model_serving_nvidia_h100_gpus).\n",
|
||||
"\n",
|
||||
"# @markdown > | Machine Type | Accelerator Type | Recommended Regions |\n",
|
||||
"# @markdown | ----------- | ----------- | ----------- |\n",
|
||||
"# @markdown | a2-ultragpu-1g | 1 NVIDIA_A100_80GB | us-central1, us-east4, europe-west4, asia-southeast1, us-east4 |\n",
|
||||
"# @markdown | a3-highgpu-2g | 2 NVIDIA_H100_80GB | us-west1, asia-southeast1, europe-west4 |\n",
|
||||
"# @markdown | a3-highgpu-4g | 4 NVIDIA_H100_80GB | us-west1, asia-southeast1, europe-west4 |\n",
|
||||
"# @markdown | a3-highgpu-8g | 8 NVIDIA_H100_80GB | us-central1, europe-west4, us-west1, asia-southeast1 |\n",
|
||||
"\n",
|
||||
"# @markdown 4. **[Optional]** [Create a Cloud Storage bucket](https://cloud.google.com/storage/docs/creating-buckets) for storing experiment outputs. Set the BUCKET_URI for the experiment environment. The specified Cloud Storage bucket (`BUCKET_URI`) should be located in the same region as where the notebook was launched. Note that a multi-region bucket (eg. \"us\") is not considered a match for a single region covered by the multi-region range (eg. \"us-central1\"). If not set, a unique GCS bucket will be created instead.\n",
|
||||
"\n",
|
||||
"BUCKET_URI = \"gs://\" # @param {type:\"string\"}\n",
|
||||
"\n",
|
||||
"# @markdown 5. **[Optional]** Set region. If not set, the region will be set automatically according to Colab Enterprise environment.\n",
|
||||
"# @markdown 2. **[Optional]** Set region. If not set, the region will be set automatically according to Colab Enterprise environment.\n",
|
||||
"\n",
|
||||
"REGION = \"\" # @param {type:\"string\"}\n",
|
||||
"\n",
|
||||
"# Upgrade Vertex AI SDK.\n",
|
||||
"! pip3 install --upgrade --quiet 'google-cloud-aiplatform>=1.84.0'\n",
|
||||
"! git clone https://github.com/GoogleCloudPlatform/vertex-ai-samples.git\n",
|
||||
"\n",
|
||||
"import datetime\n",
|
||||
"import importlib\n",
|
||||
"import os\n",
|
||||
"import uuid\n",
|
||||
"\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"import numpy as np\n",
|
||||
"import pycocotools.mask as mask_util\n",
|
||||
"from google.cloud import aiplatform\n",
|
||||
"\n",
|
||||
"if os.environ.get(\"VERTEX_PRODUCT\") != \"COLAB_ENTERPRISE\":\n",
|
||||
" ! pip install --upgrade tensorflow\n",
|
||||
"! git clone https://github.com/GoogleCloudPlatform/vertex-ai-samples.git\n",
|
||||
"\n",
|
||||
"common_util = importlib.import_module(\n",
|
||||
" \"vertex-ai-samples.community-content.vertex_model_garden.model_oss.notebook_util.common_util\"\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"models, endpoints = {}, {}\n",
|
||||
"LABEL = \"sam_model\"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Get the default cloud project id.\n",
|
||||
@@ -142,60 +141,23 @@
|
||||
"\n",
|
||||
"# Get the default region for launching jobs.\n",
|
||||
"if not REGION:\n",
|
||||
" if not os.environ.get(\"GOOGLE_CLOUD_REGION\"):\n",
|
||||
" raise ValueError(\n",
|
||||
" \"REGION must be set. See\"\n",
|
||||
" \" https://cloud.google.com/vertex-ai/docs/general/locations for\"\n",
|
||||
" \" available cloud locations.\"\n",
|
||||
" )\n",
|
||||
" REGION = os.environ[\"GOOGLE_CLOUD_REGION\"]\n",
|
||||
"\n",
|
||||
"# Enable the Vertex AI API and Compute Engine API, if not already.\n",
|
||||
"print(\"Enabling Vertex AI API and Compute Engine API.\")\n",
|
||||
"! gcloud services enable aiplatform.googleapis.com compute.googleapis.com\n",
|
||||
"\n",
|
||||
"# Cloud Storage bucket for storing the experiment artifacts.\n",
|
||||
"# A unique GCS bucket will be created for the purpose of this notebook. If you\n",
|
||||
"# prefer using your own GCS bucket, change the value yourself below.\n",
|
||||
"now = datetime.datetime.now().strftime(\"%Y%m%d%H%M%S\")\n",
|
||||
"BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n",
|
||||
"\n",
|
||||
"if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n",
|
||||
" BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n",
|
||||
" BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n",
|
||||
" ! gsutil mb -l {REGION} {BUCKET_URI}\n",
|
||||
"else:\n",
|
||||
" assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n",
|
||||
" shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n",
|
||||
" bucket_region = shell_output[0].strip().lower()\n",
|
||||
" if bucket_region != REGION:\n",
|
||||
" raise ValueError(\n",
|
||||
" \"Bucket region %s is different from notebook region %s\"\n",
|
||||
" % (bucket_region, REGION)\n",
|
||||
" )\n",
|
||||
"print(f\"Using this GCS Bucket: {BUCKET_URI}\")\n",
|
||||
"\n",
|
||||
"STAGING_BUCKET = os.path.join(BUCKET_URI, \"temporal\")\n",
|
||||
"MODEL_BUCKET = os.path.join(BUCKET_URI, \"sam\")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Initialize Vertex AI API.\n",
|
||||
"print(\"Initializing Vertex AI API.\")\n",
|
||||
"aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=STAGING_BUCKET)\n",
|
||||
"\n",
|
||||
"# Gets the default SERVICE_ACCOUNT.\n",
|
||||
"shell_output = ! gcloud projects describe $PROJECT_ID\n",
|
||||
"project_number = shell_output[-1].split(\":\")[1].strip().replace(\"'\", \"\")\n",
|
||||
"SERVICE_ACCOUNT = f\"{project_number}-compute@developer.gserviceaccount.com\"\n",
|
||||
"print(\"Using this default Service Account:\", SERVICE_ACCOUNT)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n",
|
||||
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n",
|
||||
"aiplatform.init(project=PROJECT_ID, location=REGION)\n",
|
||||
"\n",
|
||||
"! gcloud config set project $PROJECT_ID\n",
|
||||
"! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n",
|
||||
"! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\""
|
||||
"\n",
|
||||
"import vertexai\n",
|
||||
"\n",
|
||||
"vertexai.init(\n",
|
||||
" project=PROJECT_ID,\n",
|
||||
" location=REGION,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"base_model_name = \"sam-vit-large\"\n",
|
||||
"PUBLISHER_MODEL_NAME = f\"publishers/meta/models/segment-anything@{base_model_name}\""
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -209,29 +171,28 @@
|
||||
"source": [
|
||||
"# @title Deploy\n",
|
||||
"\n",
|
||||
"# @markdown This section deploys a pre-trained `sam-vit-large` model on Model Registry by using 1 L4 Machine.\n",
|
||||
"\n",
|
||||
"# @markdown The model deploy step will take around 20 minutes to complete.\n",
|
||||
"\n",
|
||||
"# The pre-built serving docker image.\n",
|
||||
"# The model artifacts are embedded within the container, except for model weights which will be downloaded during deployment.\n",
|
||||
"SERVE_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/sam-serve:public-image-20240121\"\n",
|
||||
"\n",
|
||||
"# @markdown Set the accelerator type.\n",
|
||||
"serve_accelerator_type = \"NVIDIA_L4\" # @param[\"NVIDIA_TESLA_V100\", \"NVIDIA_L4\"]\n",
|
||||
"accelerator_type = \"NVIDIA_L4\" # @param[\"NVIDIA_TESLA_V100\", \"NVIDIA_L4\"]\n",
|
||||
"\n",
|
||||
"if serve_accelerator_type == \"NVIDIA_TESLA_V100\":\n",
|
||||
" serve_machine_type = \"n1-standard-8\"\n",
|
||||
" serve_accelerator_count = 1\n",
|
||||
"elif serve_accelerator_type == \"NVIDIA_L4\":\n",
|
||||
" serve_machine_type = \"g2-standard-12\"\n",
|
||||
" serve_accelerator_count = 1\n",
|
||||
"if accelerator_type == \"NVIDIA_TESLA_V100\":\n",
|
||||
" machine_type = \"n1-standard-8\"\n",
|
||||
" accelerator_count = 1\n",
|
||||
"elif accelerator_type == \"NVIDIA_L4\":\n",
|
||||
" machine_type = \"g2-standard-12\"\n",
|
||||
" accelerator_count = 1\n",
|
||||
"else:\n",
|
||||
" print(f\"Unsupported accelerator type: {accelerator_type}\")\n",
|
||||
"\n",
|
||||
"MODEL_ID = \"facebook/sam-vit-large\"\n",
|
||||
"task = \"mask-generation\"\n",
|
||||
"\n",
|
||||
"# @markdown Set use_dedicated_endpoint to False if you don't want to use [dedicated endpoint](https://cloud.google.com/vertex-ai/docs/general/deployment#create-dedicated-endpoint). Note that [dedicated endpoint does not support VPC Service Controls](https://cloud.google.com/vertex-ai/docs/predictions/choose-endpoint-type), uncheck the box if you are using VPC-SC.\n",
|
||||
"use_dedicated_endpoint = True # @param {type:\"boolean\"}\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def deploy_model(\n",
|
||||
" task, display_name, model_id, machine_type, accelerator_type, accelerator_count\n",
|
||||
@@ -255,20 +216,62 @@
|
||||
" )\n",
|
||||
" model.deploy(\n",
|
||||
" endpoint=endpoint,\n",
|
||||
" machine_type=serve_machine_type,\n",
|
||||
" accelerator_type=serve_accelerator_type,\n",
|
||||
" machine_type=machine_type,\n",
|
||||
" accelerator_type=accelerator_type,\n",
|
||||
" accelerator_count=1,\n",
|
||||
" deploy_request_timeout=1800,\n",
|
||||
" system_labels={\"NOTEBOOK_NAME\": \"model_garden_pytorch_sam.ipynb\"},\n",
|
||||
" system_labels={\n",
|
||||
" \"NOTEBOOK_NAME\": \"model_garden_pytorch_sam.ipynb\",\n",
|
||||
" \"NOTEBOOK_ENVIRONMENT\": common_util.get_deploy_source(),\n",
|
||||
" },\n",
|
||||
" )\n",
|
||||
" return model, endpoint\n",
|
||||
" return model, endpoint"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "HF8tcQUFG7jB"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title [Option 1] Deploy with Model Garden SDK\n",
|
||||
"\n",
|
||||
"# @markdown Deploy with Gen AI model-centric SDK. This section uploads the prebuilt model to Model Registry and deploys it to a Vertex AI Endpoint. It takes 15 minutes to 1 hour to finish depending on the size of the model. See [use open models with Vertex AI](https://cloud.google.com/vertex-ai/generative-ai/docs/open-models/use-open-models) for documentation on other use cases.\n",
|
||||
"from vertexai.preview import model_garden\n",
|
||||
"\n",
|
||||
"model = model_garden.OpenModel(PUBLISHER_MODEL_NAME)\n",
|
||||
"endpoints[LABEL] = model.deploy(\n",
|
||||
" machine_type=machine_type,\n",
|
||||
" accelerator_type=accelerator_type,\n",
|
||||
" accelerator_count=accelerator_count,\n",
|
||||
" use_dedicated_endpoint=use_dedicated_endpoint,\n",
|
||||
" accept_eula=True, # Accept the End User License Agreement (EULA) on the model card before deploy. Otherwise, the deployment will be forbidden.\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "lyZSkLIwG7jB"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title [Option 2] Deploy with customized configs\n",
|
||||
"\n",
|
||||
"# @markdown This section deploys a pre-trained `sam-vit-large` model on Model Registry by using 1 L4 Machine.\n",
|
||||
"\n",
|
||||
"# @markdown The model deploy step will take around 20 minutes to complete.\n",
|
||||
"\n",
|
||||
"common_util.check_quota(\n",
|
||||
" project_id=PROJECT_ID,\n",
|
||||
" region=REGION,\n",
|
||||
" accelerator_type=serve_accelerator_type,\n",
|
||||
" accelerator_count=serve_accelerator_count,\n",
|
||||
" accelerator_type=accelerator_type,\n",
|
||||
" accelerator_count=accelerator_count,\n",
|
||||
" is_for_training=False,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
@@ -276,8 +279,8 @@
|
||||
" task=task,\n",
|
||||
" display_name=common_util.get_job_name_with_datetime(prefix=task),\n",
|
||||
" model_id=\"facebook/sam-vit-large\",\n",
|
||||
" machine_type=serve_machine_type,\n",
|
||||
" accelerator_type=serve_accelerator_type,\n",
|
||||
" machine_type=machine_type,\n",
|
||||
" accelerator_type=accelerator_type,\n",
|
||||
" accelerator_count=1,\n",
|
||||
")"
|
||||
]
|
||||
@@ -373,11 +376,7 @@
|
||||
"\n",
|
||||
"# Delete models.\n",
|
||||
"for model in models.values():\n",
|
||||
" model.delete()\n",
|
||||
"\n",
|
||||
"delete_bucket = False # @param {type:\"boolean\"}\n",
|
||||
"if delete_bucket:\n",
|
||||
" ! gsutil -m rm -r $BUCKET_NAME"
|
||||
" model.delete()"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
+18
-18
@@ -29,28 +29,28 @@
|
||||
"id": "2bd716bf3e39"
|
||||
},
|
||||
"source": [
|
||||
"# Vertex AI Model Garden - ViT-GPT2 Image Captioning\n",
|
||||
"# Vertex AI Model Garden - ViT-GPT2 Image Captioning [Deprecated]\n",
|
||||
"\n",
|
||||
"<table align=\"left\">\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_vit_gpt2_image_captioning.ipynb\">\n",
|
||||
" <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_vit_gpt2_image_captioning.ipynb\">\n",
|
||||
" <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
|
||||
"\u003ctable align=\"left\"\u003e\n",
|
||||
" \u003ctd\u003e\n",
|
||||
" \u003ca href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_vit_gpt2_image_captioning.ipynb\"\u003e\n",
|
||||
" \u003cimg src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"\u003e Run in Colab\n",
|
||||
" \u003c/a\u003e\n",
|
||||
" \u003c/td\u003e\n",
|
||||
" \u003ctd\u003e\n",
|
||||
" \u003ca href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_pytorch_vit_gpt2_image_captioning.ipynb\"\u003e\n",
|
||||
" \u003cimg src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\"\u003e\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/notebooks/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/model_garden/model_garden_pytorch_vit_gpt2_image_captioning.ipynb\">\n",
|
||||
" <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
|
||||
" \u003c/a\u003e\n",
|
||||
" \u003c/td\u003e\n",
|
||||
" \u003ctd\u003e\n",
|
||||
" \u003ca href=\"https://console.cloud.google.com/vertex-ai/notebooks/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/model_garden/model_garden_pytorch_vit_gpt2_image_captioning.ipynb\"\u003e\n",
|
||||
" \u003cimg src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\"\u003e\n",
|
||||
"Open in Vertex AI Workbench\n",
|
||||
" </a>\n",
|
||||
" \u003c/a\u003e\n",
|
||||
" (a Python-3 CPU notebook is recommended)\n",
|
||||
" </td>\n",
|
||||
"</table>"
|
||||
" \u003c/td\u003e\n",
|
||||
"\u003c/table\u003e"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
@@ -31,18 +31,18 @@
|
||||
},
|
||||
"source": [
|
||||
"# Vertex AI Model Garden - WeatherNext Forecasting\n",
|
||||
"<table><tbody><tr>\n",
|
||||
" <td style=\"text-align: center\">\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/colab/import/https:%2F%2Fraw.githubusercontent.com%2FGoogleCloudPlatform%2Fvertex-ai-samples%2Fmain%2Fnotebooks%2Fcommunity%2Fmodel_garden%2Fmodel_garden_weather_prediction_on_vertex.ipynb\">\n",
|
||||
" <img alt=\"Google Cloud Colab Enterprise logo\" src=\"https://lh3.googleusercontent.com/JmcxdQi-qOpctIvWKgPtrzZdJJK-J3sWE1RsfjZNwshCFgE_9fULcNpuXYTilIR2hjwN\" width=\"32px\"><br> Run in Colab Enterprise\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td style=\"text-align: center\">\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_weather_prediction_on_vertex.ipynb\">\n",
|
||||
" <img alt=\"GitHub logo\" src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" width=\"32px\"><br> View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
"</tr></tbody></table>"
|
||||
"\u003ctable\u003e\u003ctbody\u003e\u003ctr\u003e\n",
|
||||
" \u003ctd style=\"text-align: center\"\u003e\n",
|
||||
" \u003ca href=\"https://console.cloud.google.com/vertex-ai/colab/import/https:%2F%2Fraw.githubusercontent.com%2FGoogleCloudPlatform%2Fvertex-ai-samples%2Fmain%2Fnotebooks%2Fcommunity%2Fmodel_garden%2Fmodel_garden_weather_prediction_on_vertex.ipynb\"\u003e\n",
|
||||
" \u003cimg alt=\"Google Cloud Colab Enterprise logo\" src=\"https://lh3.googleusercontent.com/JmcxdQi-qOpctIvWKgPtrzZdJJK-J3sWE1RsfjZNwshCFgE_9fULcNpuXYTilIR2hjwN\" width=\"32px\"\u003e\u003cbr\u003e Run in Colab Enterprise\n",
|
||||
" \u003c/a\u003e\n",
|
||||
" \u003c/td\u003e\n",
|
||||
" \u003ctd style=\"text-align: center\"\u003e\n",
|
||||
" \u003ca href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_weather_prediction_on_vertex.ipynb\"\u003e\n",
|
||||
" \u003cimg alt=\"GitHub logo\" src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" width=\"32px\"\u003e\u003cbr\u003e View on GitHub\n",
|
||||
" \u003c/a\u003e\n",
|
||||
" \u003c/td\u003e\n",
|
||||
"\u003c/tr\u003e\u003c/tbody\u003e\u003c/table\u003e"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -81,7 +81,7 @@
|
||||
"\n",
|
||||
"### Request For TPU Quota\n",
|
||||
"\n",
|
||||
"By default, the quota for TPU training [Custom model training TPU v5e cores per region](https://console.cloud.google.com/iam-admin/quotas?location=us-central1&metric=aiplatform.googleapis.com%2Fcustom_model_training_tpu_v5e) is 0. TPU quota is only available in `us-west1`, `us-west4`, `us-central1`. You can request for higher TPU quota following the instructions at [\"Request a higher quota\"](https://cloud.google.com/docs/quota/view-manage#requesting_higher_quota). It is suggested to request at least 4 v5e to run this notebook."
|
||||
"By default, the quota for TPU training [Custom model training TPU v5e cores per region](https://console.cloud.google.com/iam-admin/quotas?location=us-central1\u0026metric=aiplatform.googleapis.com%2Fcustom_model_training_tpu_v5e) is 0. TPU quota is only available in `us-west1`, `us-west4`, `us-central1`. You can request for higher TPU quota following the instructions at [\"Request a higher quota\"](https://cloud.google.com/docs/quota/view-manage#requesting_higher_quota). It is suggested to request at least 4 v5e to run this notebook."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -112,12 +112,12 @@
|
||||
"import importlib\n",
|
||||
"import os\n",
|
||||
"import uuid\n",
|
||||
"from typing import Tuple\n",
|
||||
"from typing import Tuple, List\n",
|
||||
"import glob\n",
|
||||
"from google.cloud import aiplatform, storage\n",
|
||||
"\n",
|
||||
"# Upgrade Vertex AI SDK.\n",
|
||||
"! pip3 install --upgrade --quiet 'google-cloud-aiplatform>=1.64.0'\n",
|
||||
"! pip3 install --upgrade --quiet 'google-cloud-aiplatform\u003e=1.64.0'\n",
|
||||
"if not os.path.exists(\"./vertex-ai-samples\"):\n",
|
||||
" ! git clone https://github.com/GoogleCloudPlatform/vertex-ai-samples.git\n",
|
||||
"! pip3 uninstall --quiet -y xarray\n",
|
||||
@@ -181,7 +181,7 @@
|
||||
"\n",
|
||||
"\n",
|
||||
"# Utility functions for vertex jobs.\n",
|
||||
"def get_job_name_with_datetime(prefix: str) -> str:\n",
|
||||
"def get_job_name_with_datetime(prefix: str) -\u003e str:\n",
|
||||
" \"\"\"Gets the job name with date time when triggering training or deployment\n",
|
||||
" jobs in Vertex AI.\n",
|
||||
" \"\"\"\n",
|
||||
@@ -229,16 +229,67 @@
|
||||
"\n",
|
||||
"print(xarray.backends.list_engines())\n",
|
||||
"\n",
|
||||
"def get_existing_demo_step(\n",
|
||||
" num_forecast_steps: int,\n",
|
||||
" model_type: str = \"gen_small\",\n",
|
||||
" ) -\u003e str:\n",
|
||||
" # The demo data only supports some steps, and we can only run the predictions\n",
|
||||
" # if num_forecast_steps is smaller than or equal to the maximum supported setps.\n",
|
||||
" if model_type == \"gen_small\":\n",
|
||||
" # The max supported steps are obtained from gs://dm_graphcast/gencast/dataset.\n",
|
||||
" supported_demo_steps = [1, 4, 12, 20, 30]\n",
|
||||
" elif model_type == \"graph_small\":\n",
|
||||
" # The max supported steps are obtained from gs://dm_graphcast/graphcast/dataset.\n",
|
||||
" supported_demo_steps = [1, 4, 12, 20, 40]\n",
|
||||
" elif model_type == \"graph_operational\":\n",
|
||||
" # The max supported steps are obtained from gs://dm_graphcast/graphcast/dataset.\n",
|
||||
" supported_demo_steps = [1, 4, 12]\n",
|
||||
" else:\n",
|
||||
" raise ValueError(\"Invalid model_type.\")\n",
|
||||
"\n",
|
||||
" # Find the proper demo data for forecasting.\n",
|
||||
" found_supported_step = supported_demo_steps[-1]\n",
|
||||
" for i, supported_step in enumerate(supported_demo_steps):\n",
|
||||
" if num_forecast_steps \u003c= supported_step:\n",
|
||||
" found_supported_step = supported_step\n",
|
||||
" break\n",
|
||||
" if num_forecast_steps \u003e found_supported_step:\n",
|
||||
" raise ValueError(f\"Supported demo steps for {model_type} in gs://dm_graphcast are {supported_demo_steps}. {num_forecast_steps} is too large, and could not find proper demo data.\")\n",
|
||||
" return found_supported_step\n",
|
||||
"\n",
|
||||
"def get_suggested_machines(\n",
|
||||
" num_forecast_steps: int,\n",
|
||||
" model_type: str = \"gen_small\",) -\u003e Tuple[str, str, int]:\n",
|
||||
" if model_type == \"gen_small\":\n",
|
||||
" if num_forecast_steps \u003c= 16:\n",
|
||||
" machine_type = \"ct5lp-hightpu-4t\"\n",
|
||||
" tpu_topology = \"2x2\"\n",
|
||||
" accelerator_count = 4\n",
|
||||
" else:\n",
|
||||
" machine_type = \"ct5lp-hightpu-8t\"\n",
|
||||
" tpu_topology = \"2x4\"\n",
|
||||
" accelerator_count = 8\n",
|
||||
" else:\n",
|
||||
" if num_forecast_steps \u003c= 16:\n",
|
||||
" machine_type = \"ct5lp-hightpu-1t\"\n",
|
||||
" tpu_topology = \"1x1\"\n",
|
||||
" accelerator_count = 1\n",
|
||||
" else:\n",
|
||||
" machine_type = \"ct5lp-hightpu-4t\"\n",
|
||||
" tpu_topology = \"2x2\"\n",
|
||||
" accelerator_count = 4\n",
|
||||
" return machine_type, tpu_topology, accelerator_count\n",
|
||||
"\n",
|
||||
"def select(\n",
|
||||
" data: xarray.Dataset,\n",
|
||||
" variable: str,\n",
|
||||
" level: Optional[int] = None,\n",
|
||||
" max_steps: Optional[int] = None\n",
|
||||
" ) -> xarray.Dataset:\n",
|
||||
" ) -\u003e xarray.Dataset:\n",
|
||||
" data = data[variable]\n",
|
||||
" if \"batch\" in data.dims:\n",
|
||||
" data = data.isel(batch=0)\n",
|
||||
" if max_steps is not None and \"time\" in data.sizes and max_steps < data.sizes[\"time\"]:\n",
|
||||
" if max_steps is not None and \"time\" in data.sizes and max_steps \u003c data.sizes[\"time\"]:\n",
|
||||
" data = data.isel(time=range(0, max_steps))\n",
|
||||
" if level is not None and \"level\" in data.coords:\n",
|
||||
" data = data.sel(level=level)\n",
|
||||
@@ -248,7 +299,7 @@
|
||||
" data: xarray.Dataset,\n",
|
||||
" center: Optional[float] = None,\n",
|
||||
" robust: bool = False,\n",
|
||||
" ) -> tuple[xarray.Dataset, matplotlib.colors.Normalize, str]:\n",
|
||||
" ) -\u003e tuple[xarray.Dataset, matplotlib.colors.Normalize, str]:\n",
|
||||
" vmin = np.nanpercentile(data, (2 if robust else 0))\n",
|
||||
" vmax = np.nanpercentile(data, (98 if robust else 100))\n",
|
||||
" if center is not None:\n",
|
||||
@@ -264,7 +315,7 @@
|
||||
" plot_size: float = 5,\n",
|
||||
" robust: bool = False,\n",
|
||||
" cols: int = 4\n",
|
||||
" ) -> tuple[xarray.Dataset, matplotlib.colors.Normalize, str]:\n",
|
||||
" ) -\u003e tuple[xarray.Dataset, matplotlib.colors.Normalize, str]:\n",
|
||||
"\n",
|
||||
" first_data = next(iter(data.values()))[0]\n",
|
||||
" max_steps = first_data.sizes.get(\"time\", 1)\n",
|
||||
@@ -326,45 +377,40 @@
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"# @title Configure Models\n",
|
||||
"# @markdown You can config WeatherNext models with *data_type*, *model_type* and *num_forecast_steps*.\n",
|
||||
"\n",
|
||||
"output_dir = f\"{BUCKET_URI}/science\"\n",
|
||||
"accelerator_type = \"TPU_V5e\"\n",
|
||||
"SCIENCE_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai-restricted/vertex-vision-model-garden-dockers/science-serve.tpu.0-1.debian12.py310:20250331_0715_RC03\"\n",
|
||||
"\n",
|
||||
"# @markdown All demo data are from the public gcs bucket gs://dm_graphcast.\n",
|
||||
"# @markdown You can prepare your own data similarly for forecasting.\n",
|
||||
"# @markdown - The demo data for gen_small is from the date 2019-03-29 with resolution 1.0.\n",
|
||||
"# @markdown - The demo data for graph_small is from the date 2022-01-01 with resolution 1.0.\n",
|
||||
"# @markdown - The demo data for graph_operational is from the date 2022-01-01 with resolution 0.25.\n",
|
||||
"model_type = \"gen_small\" # @param [\"gen_small\", \"graph_small\", \"graph_operational\"]\n",
|
||||
"# @markdown All demo data are from the public gcs bucket `gs://dm_graphcast`.\n",
|
||||
"# @markdown You can prepare your own data similarly to demo data for forecasting.\n",
|
||||
"# @markdown The demo data for gen_small is from the date 2019-03-29 with resolution 1.0.\n",
|
||||
"# @markdown The demo data for graph_small is from the date 2022-01-01 with resolution 1.0.\n",
|
||||
"# @markdown The demo data for graph_operational is from the date 2022-01-01 with resolution 0.25.\n",
|
||||
"\n",
|
||||
"model_type = \"graph_operational\" # @param [\"gen_small\", \"graph_small\", \"graph_operational\"]\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"num_forecast_steps = 10 # @param {type:\"integer\"}\n",
|
||||
"# @markdown *num_forecast_steps* will specific the number of forecast steps, which will indicate the forcasting time combined with model leading time.\n",
|
||||
"# @markdown Assuming num_forecast_steps=4, and the leading time is 6 hours, then the results will contain forecasts with 6 hours, 12 hours, 18 hours and 24 hours.\n",
|
||||
"# @markdown WeatherNext Gen and Graph models support leading time as 12 hours and 6 hours separately.\n",
|
||||
"# @markdown num_forecast_steps will be truncated to the maximum of allowed values if it is beyond.\n",
|
||||
"num_forecast_steps = 4 # @param {type:\"integer\"}\n",
|
||||
"# @markdown num_forecast_steps will be truncated to the maximum of allowed values if it is beyond. Maximum of num_forecast_steps for Weather Gen and Graph models are 30 and 40 separately.\n",
|
||||
"machine_type, tpu_topology, accelerator_count = get_suggested_machines(num_forecast_steps, model_type)\n",
|
||||
"\n",
|
||||
"SCIENCE_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai-restricted/vertex-vision-model-garden-dockers/science-serve.tpu.0-1.debian12.py310:20250331_0715_RC03\"\n",
|
||||
"data_storage_dir = \"gs://dm_graphcast\"\n",
|
||||
"\n",
|
||||
"existing_demo_step = get_existing_demo_step(num_forecast_steps, model_type)\n",
|
||||
"if model_type == \"gen_small\":\n",
|
||||
" input_file = f\"{data_storage_dir}/gencast/dataset/source-era5_date-2019-03-29_res-1.0_levels-13_steps-04.nc\"\n",
|
||||
" machine_type = \"ct5lp-hightpu-4t\"\n",
|
||||
" tpu_topology = \"2x2\"\n",
|
||||
" accelerator_count = 4\n",
|
||||
" input_file = f\"{data_storage_dir}/gencast/dataset/source-era5_date-2019-03-29_res-1.0_levels-13_steps-{existing_demo_step:02d}.nc\"\n",
|
||||
" # num_ensemble_samples (WeatherNext Gen models only) specified the number of ensembling samples per step.\n",
|
||||
" num_ensemble_samples = 8\n",
|
||||
" parameters = {\"num_forecast_steps\": num_forecast_steps, \"num_ensemble_samples\": num_ensemble_samples}\n",
|
||||
"elif model_type == \"graph_small\":\n",
|
||||
" input_file = f\"{data_storage_dir}/graphcast/dataset/source-era5_date-2022-01-01_res-1.0_levels-13_steps-12.nc\"\n",
|
||||
" machine_type = \"ct5lp-hightpu-1t\"\n",
|
||||
" tpu_topology = \"1x1\"\n",
|
||||
" accelerator_count = 1\n",
|
||||
" input_file = f\"{data_storage_dir}/graphcast/dataset/source-era5_date-2022-01-01_res-1.0_levels-13_steps-{existing_demo_step:02d}.nc\"\n",
|
||||
" parameters = {\"num_forecast_steps\": num_forecast_steps}\n",
|
||||
"elif model_type == \"graph_operational\":\n",
|
||||
" input_file = f\"{data_storage_dir}/graphcast/dataset/source-hres_date-2022-01-01_res-0.25_levels-13_steps-12.nc\"\n",
|
||||
" machine_type = \"ct5lp-hightpu-1t\"\n",
|
||||
" tpu_topology = \"1x1\"\n",
|
||||
" accelerator_count = 1\n",
|
||||
" input_file = f\"{data_storage_dir}/graphcast/dataset/source-hres_date-2022-01-01_res-0.25_levels-13_steps-{existing_demo_step:02d}.nc\"\n",
|
||||
" parameters = {\"num_forecast_steps\": num_forecast_steps}\n",
|
||||
"else:\n",
|
||||
" raise ValueError(\"Invalid example_type.\")\n",
|
||||
@@ -381,6 +427,7 @@
|
||||
"\n",
|
||||
"print(f\"machine_type is {machine_type}.\")\n",
|
||||
"print(f\"tpu_topology is {tpu_topology}.\")\n",
|
||||
"print(f\"SCIENCE_DOCKER_URI is {SCIENCE_DOCKER_URI}.\")\n",
|
||||
"print(f\"The prediction instances: {instances}\")\n"
|
||||
],
|
||||
"metadata": {
|
||||
@@ -398,7 +445,6 @@
|
||||
"# @markdown It usually takes a couple of minutes to finish.\n",
|
||||
"# @markdown Click on the generated link in the output to see your run in the Cloud Console.\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"print(\"Check if there are enough quota.\")\n",
|
||||
"common_util.check_quota(\n",
|
||||
" project_id=PROJECT_ID,\n",
|
||||
@@ -529,8 +575,7 @@
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "911406c1561e",
|
||||
"cellView": "form"
|
||||
"id": "911406c1561e"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -538,7 +583,8 @@
|
||||
"\n",
|
||||
"delete_bucket = True # @param {type:\"boolean\"}\n",
|
||||
"if delete_bucket:\n",
|
||||
" ! gsutil -m rm -r $BUCKET_URI\n"
|
||||
" ! gsutil -m rm -r $BUCKET_URI\n",
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
@@ -57,6 +57,7 @@
|
||||
/training/tpuv5e_gemma_peft_finetuning_and_serving.ipynb @brianchunkang
|
||||
/training/tpuv5e_llama2_pytorch_finetuning_and_serving.ipynb @brianchunkang @chiefkarlin
|
||||
/prediction/get_started_with_psc_private_endpoint.ipynb @tianjiaoliu
|
||||
/prediction/get_started_with_dedicated_endpoint.ipynb @tianjiaoliu
|
||||
/ray_on_vertex_ai/spark_on_ray_on_vertex_ai.ipynb @ravi-dalal
|
||||
/generative_ai/mistralai_intro.ipynb @sujituk
|
||||
/generative_ai/ai21labs_intro.ipynb @miri-bar
|
||||
|
||||
@@ -409,7 +409,7 @@
|
||||
"df = bq_client.list_rows(table).to_dataframe()\n",
|
||||
"\n",
|
||||
"# Drop unusable rows\n",
|
||||
"df = df.replace(to_replace=NA_VALUES, value=np.NaN).dropna()\n",
|
||||
"df = df.replace(to_replace=NA_VALUES, value=np.nan).dropna()\n",
|
||||
"\n",
|
||||
"# Convert categorical columns to numeric\n",
|
||||
"df[\"island\"], _ = pd.factorize(df[\"island\"])\n",
|
||||
|
||||
@@ -153,53 +153,13 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! pip3 install --upgrade --quiet google-cloud-aiplatform \\\n",
|
||||
"%pip install --upgrade --quiet google-cloud-aiplatform \\\n",
|
||||
" google-cloud \\\n",
|
||||
" google-cloud-storage \\\n",
|
||||
" pillow \\\n",
|
||||
" numpy"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "ff555b32bab8"
|
||||
},
|
||||
"source": [
|
||||
"### Restart runtime (Colab only)\n",
|
||||
"\n",
|
||||
"To use the newly installed packages, you must restart the runtime on Google Colab."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "f09b4dff629a"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import sys\n",
|
||||
"\n",
|
||||
"if \"google.colab\" in sys.modules:\n",
|
||||
"\n",
|
||||
" import IPython\n",
|
||||
"\n",
|
||||
" app = IPython.Application.instance()\n",
|
||||
" app.kernel.do_shutdown(True)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "54c5ef8a8f43"
|
||||
},
|
||||
"source": [
|
||||
"<div class=\"alert alert-block alert-warning\">\n",
|
||||
"<b>⚠️ The kernel is going to restart. Wait until it's finished before continuing to the next step. ⚠️</b>\n",
|
||||
"</div>\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
|
||||
@@ -72,6 +72,12 @@
|
||||
"\n",
|
||||
"### Available Anthropic Claude models\n",
|
||||
"\n",
|
||||
"#### Claude Sonnet 4\n",
|
||||
"Anthropic's mid-size model with superior intelligence for high-volume uses in coding, in-depth research, agents, & more.\n",
|
||||
"\n",
|
||||
"#### Claude Opus 4\n",
|
||||
"Anthropic’s most powerful model yet and the state-of-the-art coding model. It delivers sustained performance on long-running tasks that require focused effort and thousands of steps, significantly expanding what AI agents can solve. Claude Opus 4 is ideal for powering frontier agent products and features.\n",
|
||||
"\n",
|
||||
"#### Claude 3.7 Sonnet\n",
|
||||
"Industry-leading model for coding and powering AI agents—and the first Claude model to offer extended thinking.\n",
|
||||
"\n",
|
||||
@@ -137,47 +143,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! pip3 install -U -q httpx"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "b9f4c57a43f6"
|
||||
},
|
||||
"source": [
|
||||
"### Restart runtime (Colab only)\n",
|
||||
"\n",
|
||||
"To use the newly installed packages, you must restart the runtime on Google Colab."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "3b9119a60525"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import sys\n",
|
||||
"\n",
|
||||
"if \"google.colab\" in sys.modules:\n",
|
||||
"\n",
|
||||
" import IPython\n",
|
||||
"\n",
|
||||
" app = IPython.Application.instance()\n",
|
||||
" app.kernel.do_shutdown(True)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "e767418763cd"
|
||||
},
|
||||
"source": [
|
||||
"<div class=\"alert alert-block alert-warning\">\n",
|
||||
"<b>⚠️ The kernel is going to restart. Wait until it's finished before continuing to the next step. ⚠️</b>\n",
|
||||
"</div>\n"
|
||||
"%pip install -U -q httpx"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -225,11 +191,15 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"MODEL = \"claude-3-7-sonnet@20250219\" # @param [\"claude-3-7-sonnet@20250219\", \"claude-3-5-sonnet-v2@20241022\", \"claude-3-5-haiku@20241022\", \"claude-3-5-sonnet@20240620\", \"claude-3-opus@20240229\", \"claude-3-haiku@20240307\", \"claude-3-sonnet@20240229\" ]\n",
|
||||
"if MODEL == \"claude-3-7-sonnet@20250219\":\n",
|
||||
" available_regions = [\"us-east5\", \"europe-west1\"]\n",
|
||||
"MODEL = \"claude-sonnet-4@20250514\" # @param [\"claude-sonnet-4@20250514\",\"claude-opus-4@20250514\",\"claude-3-7-sonnet@20250219\", \"claude-3-5-sonnet-v2@20241022\", \"claude-3-5-haiku@20241022\", \"claude-3-5-sonnet@20240620\", \"claude-3-opus@20240229\", \"claude-3-haiku@20240307\", \"claude-3-sonnet@20240229\" ]\n",
|
||||
"if MODEL == \"claude-sonnet-4@20250514\":\n",
|
||||
" available_regions = [\"us-east5\", \"europe-west4\", \"GLOBAL\"]\n",
|
||||
"elif MODEL == \"claude-opus-4@20250514\":\n",
|
||||
" available_regions = [\"us-east5\", \"europe-west4\"]\n",
|
||||
"elif MODEL == \"claude-3-7-sonnet@20250219\":\n",
|
||||
" available_regions = [\"us-east5\", \"europe-west1\", \"europe-west4\", \"GLOBAL\"]\n",
|
||||
"elif MODEL == \"claude-3-5-sonnet-v2@20241022\":\n",
|
||||
" available_regions = [\"us-east5\", \"europe-west1\"]\n",
|
||||
" available_regions = [\"us-east5\", \"europe-west1\", \"GLOBAL\"]\n",
|
||||
"elif MODEL == \"claude-3-5-haiku@20241022\":\n",
|
||||
" available_regions = [\"us-east5\"]\n",
|
||||
"elif MODEL == \"claude-3-5-sonnet@20240620\":\n",
|
||||
@@ -302,6 +272,11 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"PROJECT_ID = \"[your-project-id]\" # @param {type:\"string\"}\n",
|
||||
"\n",
|
||||
"if LOCATION == \"GLOBAL\":\n",
|
||||
" ENDPOINT = \"https://aiplatform.googleapis.com\"\n",
|
||||
"else:\n",
|
||||
" ENDPOINT = f\"https://{LOCATION}-aiplatform.googleapis.com\"\n",
|
||||
"ENDPOINT = f\"https://{LOCATION}-aiplatform.googleapis.com\"\n",
|
||||
"\n",
|
||||
"if not PROJECT_ID or PROJECT_ID == \"[your-project-id]\":\n",
|
||||
@@ -653,11 +628,15 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"MODEL = \"claude-3-7-sonnet@20250219\" # @param [\"claude-3-7-sonnet@20250219\", \"claude-3-5-sonnet-v2@20241022\", \"claude-3-5-haiku@20241022\", \"claude-3-5-sonnet@20240620\", \"claude-3-opus@20240229\", \"claude-3-haiku@20240307\", \"claude-3-sonnet@20240229\" ]\n",
|
||||
"if MODEL == \"claude-3-7-sonnet@20250219\":\n",
|
||||
" available_regions = [\"us-east5\", \"europe-west1\"]\n",
|
||||
"MODEL = \"claude-sonnet-4@20250514\" # @param [\"claude-sonnet-4@20250514\",\"claude-opus-4@20250514\",\"claude-3-7-sonnet@20250219\", \"claude-3-5-sonnet-v2@20241022\", \"claude-3-5-haiku@20241022\", \"claude-3-5-sonnet@20240620\", \"claude-3-opus@20240229\", \"claude-3-haiku@20240307\", \"claude-3-sonnet@20240229\" ]\n",
|
||||
"if MODEL == \"claude-sonnet-4@20250514\":\n",
|
||||
" available_regions = [\"us-east5\", \"europe-west4\", \"GLOBAL\"]\n",
|
||||
"elif MODEL == \"claude-opus-4@20250514\":\n",
|
||||
" available_regions = [\"us-east5\", \"europe-west4\"]\n",
|
||||
"elif MODEL == \"claude-3-7-sonnet@20250219\":\n",
|
||||
" available_regions = [\"us-east5\", \"europe-west1\", \"europe-west4\", \"GLOBAL\"]\n",
|
||||
"elif MODEL == \"claude-3-5-sonnet-v2@20241022\":\n",
|
||||
" available_regions = [\"us-east5\", \"europe-west1\"]\n",
|
||||
" available_regions = [\"us-east5\", \"europe-west1\", \"GLOBAL\"]\n",
|
||||
"elif MODEL == \"claude-3-5-haiku@20241022\":\n",
|
||||
" available_regions = [\"us-east5\"]\n",
|
||||
"elif MODEL == \"claude-3-5-sonnet@20240620\":\n",
|
||||
|
||||
@@ -56,6 +56,17 @@
|
||||
"</table>"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "dFkhRcsaqToV"
|
||||
},
|
||||
"source": [
|
||||
"NOTE: This notebook has been tested in the following environment:\n",
|
||||
"\n",
|
||||
"Python version = 3.10"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
@@ -74,6 +85,9 @@
|
||||
"\n",
|
||||
"### Available Mistral AI models\n",
|
||||
"\n",
|
||||
"* ### Mistral OCR (25.05)\n",
|
||||
"Mistral OCR (25.05) is a model specialized in extracting text and images from documents. It is specifically built to preserve the structure of the document pages and automatically formats the extracted text in Markdown.\n",
|
||||
"\n",
|
||||
"* ### Mistral Small 3.1 (25.03)\n",
|
||||
"Mistral Small 3.1 (25.03) is the enhanced version of Mistral Small 3, featuring multimodal capabilities and an extended context length of up to 128k.\n",
|
||||
"\n",
|
||||
@@ -89,7 +103,7 @@
|
||||
"\n",
|
||||
"## Objective\n",
|
||||
"\n",
|
||||
"This notebook shows how to use **Vertex AI API** to call the Mistral AI models on Vertex AI API with the Large, Nemo, and Codestral models.\n",
|
||||
"This notebook shows how to use **Vertex AI API** to call the Mistral AI models.\n",
|
||||
"\n",
|
||||
"For more information, see the [Use Mistral's](https://docs.mistral.ai/) documentation and [Mistral's models](https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/mistral) on Google Cloud.\n",
|
||||
"\n",
|
||||
@@ -160,22 +174,19 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"MODEL = \"mistral-small-2503\" # @param [\"mistral-small-2503\", \"codestral-2501\", \"mistral-large-2411\", \"mistral-large\", \"mistral-nemo\", \"codestral\"]\n",
|
||||
"MODEL = \"mistral-small-2503\" # @param [\"mistral-small-2503\", \"codestral-2501\", \"mistral-large-2411\", \"mistral-nemo\", \"mistral-ocr-2505\"]\n",
|
||||
"if MODEL == \"mistral-small-2503\":\n",
|
||||
" available_regions = [\"europe-west4\", \"us-central1\"]\n",
|
||||
" available_versions = [\"latest\"]\n",
|
||||
"elif MODEL == \"mistral-large-2411\":\n",
|
||||
" available_regions = [\"europe-west4\", \"us-central1\"]\n",
|
||||
" available_versions = [\"latest\"]\n",
|
||||
"elif MODEL == \"mistral-large\":\n",
|
||||
"elif MODEL == \"mistral-ocr-2505\":\n",
|
||||
" available_regions = [\"europe-west4\", \"us-central1\"]\n",
|
||||
" available_versions = [\"2407\"]\n",
|
||||
" available_versions = [\"latest\"]\n",
|
||||
"elif MODEL == \"mistral-nemo\":\n",
|
||||
" available_regions = [\"europe-west4\", \"us-central1\"]\n",
|
||||
" available_versions = [\"2407\"]\n",
|
||||
"elif MODEL == \"codestral\":\n",
|
||||
" available_regions = [\"europe-west4\", \"us-central1\"]\n",
|
||||
" available_versions = [\"2405\"]\n",
|
||||
"elif MODEL == \"codestral-2501\":\n",
|
||||
" available_regions = [\"europe-west4\", \"us-central1\"]\n",
|
||||
" available_versions = [\"latest\"]"
|
||||
@@ -289,6 +300,15 @@
|
||||
"import requests"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "Y_8xeTfWqToW"
|
||||
},
|
||||
"source": [
|
||||
"To try the OCR model, please refer to the section \"Performing OCR\"."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
@@ -693,20 +713,20 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"MODEL = \"mistral-small-2503\" # @param [\"mistral-small-2503\", \"mistral-large\", \"mistral-nemo\", \"codestral\"]\n",
|
||||
"MODEL = \"mistral-small-2503\" # @param [\"mistral-small-2503\", \"mistral-large\", \"mistral-nemo\", \"codestral-2501\"]\n",
|
||||
"\n",
|
||||
"if MODEL == \"mistral-small\":\n",
|
||||
"if MODEL == \"mistral-small-2503\":\n",
|
||||
" available_regions = [\"europe-west4\", \"us-central1\"]\n",
|
||||
" available_versions = [\"latest\", \"2503\"]\n",
|
||||
" available_versions = [\"latest\"]\n",
|
||||
"elif MODEL == \"mistral-large\":\n",
|
||||
" available_regions = [\"europe-west4\", \"us-central1\"]\n",
|
||||
" available_versions = [\"latest\", \"2411\", \"2407\"]\n",
|
||||
" available_versions = [\"latest\", \"2411\"]\n",
|
||||
"elif MODEL == \"mistral-nemo\":\n",
|
||||
" available_regions = [\"europe-west4\", \"us-central1\"]\n",
|
||||
" available_versions = [\"latest\", \"2407\"]\n",
|
||||
"elif MODEL == \"codestral\":\n",
|
||||
"elif MODEL == \"codestral-2501\":\n",
|
||||
" available_regions = [\"europe-west4\", \"us-central1\"]\n",
|
||||
" available_versions = [\"latest\", \"2501\"]"
|
||||
" available_versions = [\"latest\"]"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -963,12 +983,12 @@
|
||||
" access_token=access_token, region=LOCATION, project_id=PROJECT_ID\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"MODEL = \"codestral\"\n",
|
||||
"MODEL = \"codestral-2501\"\n",
|
||||
"MODEL_VERSION = \"2501\"\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" resp = client.fim.complete(\n",
|
||||
" model=f\"{MODEL}-{MODEL_VERSION}\",\n",
|
||||
" model=f\"{MODEL}\",\n",
|
||||
" prompt=\"def count_words_in_file(file_path: str) -> int\",\n",
|
||||
" suffix=\"return n_words\",\n",
|
||||
" )\n",
|
||||
@@ -1336,6 +1356,279 @@
|
||||
"else:\n",
|
||||
" print(f\"Request failed with status code: {response.status_code}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "JEbcSlVTqToe"
|
||||
},
|
||||
"source": [
|
||||
"### Leveraging multimodal inputs (images and text)\n",
|
||||
"\n",
|
||||
"When using a multimodal model like `mistral-small-2503` it is possible to pass images as input in addition to text. To do so, you will need to encode the image in base64 and pass it inside an `image_url` content block:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "SRckVQXkqToe"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"MM_MODEL = \"mistral-small-2503\"\n",
|
||||
"url = f\"{ENDPOINT}/v1/projects/{PROJECT_ID}/locations/{LOCATION}/publishers/mistralai/models/{MM_MODEL}:rawPredict\"\n",
|
||||
"data = {\n",
|
||||
" \"model\": MM_MODEL,\n",
|
||||
" \"messages\": [\n",
|
||||
" {\n",
|
||||
" \"role\": \"user\",\n",
|
||||
" \"content\": [\n",
|
||||
" {\"type\": \"text\", \"text\": \"What is in this image ?\"},\n",
|
||||
" {\n",
|
||||
" \"type\": \"image_url\",\n",
|
||||
" \"image_url\": \"data:image/jpeg;base64,<your_base64_image>\",\n",
|
||||
" },\n",
|
||||
" ],\n",
|
||||
" }\n",
|
||||
" ],\n",
|
||||
"}\n",
|
||||
"headers = {\n",
|
||||
" \"Authorization\": f\"Bearer {access_token}\",\n",
|
||||
" \"Content-Type\": \"application/json\",\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"# Make the POST request\n",
|
||||
"response = requests.post(url, headers=headers, json=data)\n",
|
||||
"\n",
|
||||
"# Check status code and try to parse the response as JSON\n",
|
||||
"if response.status_code == 200:\n",
|
||||
" try:\n",
|
||||
" response_dict = response.json()\n",
|
||||
" print(response_dict[\"choices\"][0][\"message\"][\"content\"])\n",
|
||||
" except json.JSONDecodeError as e:\n",
|
||||
" print(\"Error decoding JSON:\", e)\n",
|
||||
" print(\"Raw response:\", response.text) # Print raw response if parsing fails\n",
|
||||
"else:\n",
|
||||
" print(f\"Request failed with status code: {response.status_code}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "znQtFeXUqToe"
|
||||
},
|
||||
"source": [
|
||||
"### Performing OCR\n",
|
||||
"\n",
|
||||
"Mistral OCR (`mistral-ocr-2505`) is an Optical Character Recognition API that sets a new standard in document understanding. Unlike other models, Mistral OCR comprehends each element of documents: images, text, tables, equations, etc. It takes images and PDFs as input and extracts content in an ordered interleaved text and images.\n",
|
||||
"\n",
|
||||
"The following example showcases the application of the OCR API on a PDF file."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "KRaYEGJ1qToe"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import base64\n",
|
||||
"\n",
|
||||
"# The URL you provided\n",
|
||||
"pdf_url = \"https://arxiv.org/pdf/2501.12948\"\n",
|
||||
"\n",
|
||||
"# 1. Download the file\n",
|
||||
"print(f\"Attempting to download from: {pdf_url}\")\n",
|
||||
"response = requests.get(pdf_url)\n",
|
||||
"pdf_content_bytes = response.content\n",
|
||||
"\n",
|
||||
"# 2. Base64 encode the downloaded content\n",
|
||||
"encoded_content_bytes = base64.b64encode(pdf_content_bytes)\n",
|
||||
"encoded_string = encoded_content_bytes.decode(\"utf-8\")\n",
|
||||
"\n",
|
||||
"print(\"Download and encoding complete.\")\n",
|
||||
"\n",
|
||||
"pdf_doc_base64_uri = f\"data:application/pdf;base64,{encoded_string}\"\n",
|
||||
"\n",
|
||||
"# print(pdf_doc_base64_uri)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "Tn3sECpRqToj"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Get the access token\n",
|
||||
"process = subprocess.Popen(\n",
|
||||
" \"gcloud auth print-access-token\", stdout=subprocess.PIPE, shell=True\n",
|
||||
")\n",
|
||||
"(access_token_bytes, err) = process.communicate()\n",
|
||||
"access_token = access_token_bytes.decode(\"utf-8\").strip() # Strip newline\n",
|
||||
"\n",
|
||||
"OCR_MODEL = \"mistral-ocr-2505\"\n",
|
||||
"url = f\"{ENDPOINT}/v1/projects/{PROJECT_ID}/locations/{LOCATION}/publishers/mistralai/models/{OCR_MODEL}:rawPredict\"\n",
|
||||
"data = {\n",
|
||||
" \"model\": OCR_MODEL,\n",
|
||||
" \"document\": {\"type\": \"document_url\", \"document_url\": pdf_doc_base64_uri},\n",
|
||||
"}\n",
|
||||
"headers = {\n",
|
||||
" \"Authorization\": f\"Bearer {access_token}\",\n",
|
||||
" \"Content-Type\": \"application/json\",\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"# Make the POST request\n",
|
||||
"response = requests.post(url, headers=headers, json=data)\n",
|
||||
"\n",
|
||||
"# Check status code and try to parse the response as JSON\n",
|
||||
"if response.status_code == 200:\n",
|
||||
" try:\n",
|
||||
" response_dict = response.json()\n",
|
||||
" print(response_dict)\n",
|
||||
" except json.JSONDecodeError as e:\n",
|
||||
" print(\"Error decoding JSON:\", e)\n",
|
||||
" print(\"Raw response:\", response.text) # Print raw response if parsing fails\n",
|
||||
"else:\n",
|
||||
" print(f\"Request failed with status code: {response.status_code}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "CmpLW-IEqToj"
|
||||
},
|
||||
"source": [
|
||||
"To get more details on the options available when calling the OCR API, please refer to the [Mistral API documentation](https://docs.mistral.ai/api/#tag/ocr)."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "da51562946ab"
|
||||
},
|
||||
"source": [
|
||||
"## Using Mistral AI models in self-deploy mode\n",
|
||||
"\n",
|
||||
"The following models can be self-deployed on your own Google Cloud infrastructure:\n",
|
||||
"\n",
|
||||
"- `codestral-2501`\n",
|
||||
"\n",
|
||||
"Once the model has been procured and deployed, you can run inference on it by going through the following steps.\n",
|
||||
"\n",
|
||||
"- First, gather the required information element to build the URL of your deployed endpoint, namely:\n",
|
||||
" - Your Google Cloud Project ID\n",
|
||||
" - The name of the deployed model\n",
|
||||
" - The region where your model endpoint is deployed\n",
|
||||
" - Your model endpoint's ID\n",
|
||||
"\n",
|
||||
" Most of these elements are referenced as environment variables in the cell below, which contains two functions:\n",
|
||||
"\n",
|
||||
" - `get_access_token()` returns a credential object and extracts an access token to authenticate your API calls to the model endpoint,\n",
|
||||
" - `build_endpoint_url()` return your model endpoint's URL."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "82df0a0edf76"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import google.auth\n",
|
||||
"import google.auth.credentials\n",
|
||||
"import httpx\n",
|
||||
"from google.auth.transport.requests import Request\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def get_access_token() -> str:\n",
|
||||
" credentials, _ = google.auth.default(\n",
|
||||
" scopes=[\"https://www.googleapis.com/auth/cloud-platform\"]\n",
|
||||
" )\n",
|
||||
" credentials.refresh(Request())\n",
|
||||
" return credentials.token\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def build_endpoint_url(\n",
|
||||
" project_id: str, region: str, endpoint_id: str, is_streamed: bool = False\n",
|
||||
") -> str:\n",
|
||||
" base_url = f\"https://{region}-aiplatform.googleapis.com/v1/projects/{project_id}/locations/{region}\"\n",
|
||||
" suffix = \"streamRawPredict\" if is_streamed else \"rawPredict\"\n",
|
||||
" endpoint_url = f\"{base_url}/endpoints/{endpoint_id}:{suffix}\"\n",
|
||||
" return endpoint_url"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "812ef74c57df"
|
||||
},
|
||||
"source": [
|
||||
"- Next, run these functions to retrieve your model endpoint URL and the access token:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "ba2c1a073d76"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"project_id = \"\" # Add your Google Cloud Project ID here\n",
|
||||
"region = \"\" # Add your Google Cloud Region here\n",
|
||||
"endpoint_id = \"\" # Add the Vertex endpoint ID of your deployed model here\n",
|
||||
"model = \"codestral-2501-self-deploy\"\n",
|
||||
"\n",
|
||||
"url = build_endpoint_url(project_id=project_id, region=region, endpoint_id=endpoint_id)\n",
|
||||
"token = get_access_token()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "a0bb2730f5e3"
|
||||
},
|
||||
"source": [
|
||||
"- Finally, make the HTTP call to your model endpoint URL with the input of your choice. If your request runs successfully, you should retrieve a response containing the model's answer."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "b104599c4d25"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"is_streamed = False\n",
|
||||
"user_message_content = \"Who is the best French painter? Answer in one short sentence\"\n",
|
||||
"\n",
|
||||
"headers = {\"Accept\": \"application/json\", \"Authorization\": f\"Bearer {token}\"}\n",
|
||||
"payload = {\n",
|
||||
" \"model\": model,\n",
|
||||
" \"messages\": [{\"role\": \"user\", \"content\": user_message_content}],\n",
|
||||
" \"stream\": is_streamed,\n",
|
||||
"}\n",
|
||||
"url = build_endpoint_url(project_id=project_id, region=region, endpoint_id=endpoint_id)\n",
|
||||
"\n",
|
||||
"with httpx.Client() as client:\n",
|
||||
" resp = client.post(url=url, json=payload, headers=headers, timeout=None)\n",
|
||||
" print(resp.text)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "89e4f9bf981f"
|
||||
},
|
||||
"source": [
|
||||
"From there you can tune your requests to match the capabilities of the regular serverless endpoints by streaming output tokens, leveraging tool use or performing FIM completion (for Codestral only), etc."
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
|
||||
+1
-1
@@ -971,7 +971,7 @@
|
||||
],
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"name": "Qodo_intro.ipynb",
|
||||
"name": "qodo_intro.ipynb",
|
||||
"toc_visible": true
|
||||
},
|
||||
"kernelspec": {
|
||||
+4
-4
@@ -8,7 +8,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Copyright 2023 Google LLC\n",
|
||||
"# Copyright 2025 Google LLC\n",
|
||||
"#\n",
|
||||
"# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
|
||||
"# you may not use this file except in compliance with the License.\n",
|
||||
@@ -99,7 +99,7 @@
|
||||
"source": [
|
||||
"### Dataset\n",
|
||||
"\n",
|
||||
"In this tutorial, you will use the sample dataset called \"wide_and_deep_trainer_container_tests_input.jsonl\" developed by Google Brain.\n",
|
||||
"In this tutorial, you will use the sample dataset called \"wide_and_deep_trainer_container_tests_input.jsonl\" developed by Google DeepMind.\n",
|
||||
"\n",
|
||||
"You will import the file from Google Cloud Samples Data Bucket in the next few steps.\n",
|
||||
"\n",
|
||||
@@ -286,7 +286,7 @@
|
||||
"import numpy as np\n",
|
||||
"import pandas as pd\n",
|
||||
"import scann\n",
|
||||
"from vertexai.preview.language_models import TextEmbeddingModel"
|
||||
"from vertexai.language_models import TextEmbeddingModel"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -306,7 +306,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"model = TextEmbeddingModel.from_pretrained(\"google/textembedding-gecko@001\")"
|
||||
"model = TextEmbeddingModel.from_pretrained(\"gemini-embedding-001\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
@@ -4,11 +4,12 @@
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "ur8xi4C7S06n"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Copyright 2023 Google LLC\n",
|
||||
"# @title Copyright 2025 Google LLC\n",
|
||||
"#\n",
|
||||
"# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
|
||||
"# you may not use this file except in compliance with the License.\n",
|
||||
@@ -29,7 +30,7 @@
|
||||
"id": "JAPoU8Sm5E6e"
|
||||
},
|
||||
"source": [
|
||||
"# Text Embedding New API\n",
|
||||
"# Getting Text Embeddings on Vertex AI\n",
|
||||
"\n",
|
||||
"<table align=\"left\">\n",
|
||||
" <td style=\"text-align: center\">\n",
|
||||
@@ -63,9 +64,7 @@
|
||||
"source": [
|
||||
"## Overview\n",
|
||||
"\n",
|
||||
"This notebook is a code example for how to call our newly released text emebedding models (text-embedding-004, text-multilingual-embedding-002, text-embedding-preview-0815).\n",
|
||||
"\n",
|
||||
"Learn more about [text embedding api](https://cloud.google.com/vertex-ai/docs/generative-ai/embeddings/get-text-embeddings#api_changes_to_models_released_in_or_after_august_2023)."
|
||||
"This notebook will walk you through the process of getting text embeddings given a text-embedding model such as **gemini-embedding-001**. See also [getting text embeddings](https://cloud.google.com/vertex-ai/docs/generative-ai/embeddings/get-text-embeddings#api_changes_to_models_released_in_or_after_august_2023)."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -76,17 +75,17 @@
|
||||
"source": [
|
||||
"### Objective\n",
|
||||
"\n",
|
||||
"In this tutorial, you learn how to call text embedding latest APIs on two new GA models text-embedding-004, text-multilingual-embedding-002 and one preview model text-embedding-preview-0815.\n",
|
||||
"In this tutorial, you learn how to get a text embedding given a text-embedding model and a text.\n",
|
||||
"\n",
|
||||
"This tutorial uses the following Google Cloud ML services and resources:\n",
|
||||
"\n",
|
||||
"- Vertex LLM SDK\n",
|
||||
"* Vertex AI\n",
|
||||
"\n",
|
||||
"The steps include:\n",
|
||||
"\n",
|
||||
"The steps performed include:\n",
|
||||
"\n",
|
||||
"- Installation and imports\n",
|
||||
"- Generate embeddings"
|
||||
"* Installing and initializing Vertex AI SDK.\n",
|
||||
"* Authenticating your Google Cloud account.\n",
|
||||
"* Getting an embedding given a model and a text."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -115,24 +114,44 @@
|
||||
"## Get Started"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "OvuvT7KHT89K"
|
||||
},
|
||||
"source": [
|
||||
"### Set up your Google Cloud project\n",
|
||||
"\n",
|
||||
"**The following steps are required, regardless of your notebook environment.**\n",
|
||||
"\n",
|
||||
"1. [Select or create a Google Cloud project](https://console.cloud.google.com/projectselector2/). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
|
||||
"\n",
|
||||
"2. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
|
||||
"\n",
|
||||
"3. [Enable APIs](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,documentai.googleapis.com).\n",
|
||||
"\n",
|
||||
"4. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk)."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "6dac73aa3f75"
|
||||
},
|
||||
"source": [
|
||||
"### Install Vertex AI SDK for python and other required packages"
|
||||
"### Installation\n",
|
||||
"\n",
|
||||
"This tutorial requires you to install the latest `google-cloud-aiplatform` package."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "6c944350c059"
|
||||
"id": "PC0H5W7KO4AI"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Install Vertex AI SDK for Python\n",
|
||||
"!pip3 install --quiet --upgrade google-cloud-aiplatform"
|
||||
]
|
||||
},
|
||||
@@ -142,7 +161,7 @@
|
||||
"id": "6e0c0cdffff3"
|
||||
},
|
||||
"source": [
|
||||
"### Restart runtime (Colab only)\n",
|
||||
"#### Restart runtime (Colab only)\n",
|
||||
"\n",
|
||||
"To use the newly installed packages, you must restart the runtime on Google Colab."
|
||||
]
|
||||
@@ -165,91 +184,117 @@
|
||||
" app.kernel.do_shutdown(True)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "7b49231643e4"
|
||||
},
|
||||
"source": [
|
||||
"<div class=\"alert alert-block alert-warning\">\n",
|
||||
"<b>⚠️ The kernel is going to restart. Wait until it's finished before continuing to the next step. ⚠️</b>\n",
|
||||
"</div>\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "23b421f88a9b"
|
||||
},
|
||||
"source": [
|
||||
"### Authenticate your notebook environment (Colab only)\n",
|
||||
"\n",
|
||||
"Authenticate your environment on Google Colab."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "7de6ef0fac42"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import sys\n",
|
||||
"\n",
|
||||
"if \"google.colab\" in sys.modules:\n",
|
||||
"\n",
|
||||
" from google.colab import auth\n",
|
||||
"\n",
|
||||
" auth.authenticate_user()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "e487ec618b5e"
|
||||
},
|
||||
"source": [
|
||||
"### Set Google Cloud project information and initialize Vertex AI SDK for python\n",
|
||||
"### Initialize Vertex AI SDK\n",
|
||||
"\n",
|
||||
"To get started using Vertex AI, you must have an existing Google Cloud project and [enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com). Learn more about [setting up a project and a development environment](https://cloud.google.com/vertex-ai/docs/start/cloud-environment)."
|
||||
"Import and Initialize the SDK for your project and region.\n",
|
||||
"\n",
|
||||
"**If you don't know your project ID**, try the following:\n",
|
||||
"* Run `gcloud config list`.\n",
|
||||
"* Run `gcloud projects list`.\n",
|
||||
"* See also [locating the project ID](https://support.google.com/googleapi/answer/7014113).\n",
|
||||
"\n",
|
||||
"See also [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "48d81914dc54"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title (Required) Set PROJECT_ID and REGION\n",
|
||||
"PROJECT_ID = \"[your-project-id]\" # @param {type:\"string\"}\n",
|
||||
"LOCATION = \"us-central1\" # @param {type:\"string\"}\n",
|
||||
"REGION = \"us-central1\" # @param {type:\"string\"}\n",
|
||||
"if not PROJECT_ID.strip():\n",
|
||||
" raise ValueError(\"'PROJECT_ID' is required.\")\n",
|
||||
"if not REGION.strip():\n",
|
||||
" raise ValueError(\"'REGION' is required.\")\n",
|
||||
"!gcloud config set project {PROJECT_ID}\n",
|
||||
"\n",
|
||||
"import vertexai\n",
|
||||
"from vertexai.language_models import TextEmbeddingInput, TextEmbeddingModel\n",
|
||||
"\n",
|
||||
"from google.cloud import aiplatform\n",
|
||||
"\n",
|
||||
"aiplatform.init(project=PROJECT_ID, location=LOCATION)"
|
||||
"vertexai.init(project=PROJECT_ID, location=REGION)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "09720c707f1c"
|
||||
"id": "2nDeSBMbAEZq"
|
||||
},
|
||||
"source": [
|
||||
"### Import libraries"
|
||||
"### Authenticate your Google Cloud account\n",
|
||||
"\n",
|
||||
"Depending on your Jupyter environment, you may have to manually authenticate. Follow the relevant instructions below."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "AYT3vFsIAKGQ"
|
||||
},
|
||||
"source": [
|
||||
"**1. Colab:**"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "a38ef56b684a"
|
||||
"id": "IfLvy4izAMJq"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from vertexai.language_models import TextEmbeddingInput, TextEmbeddingModel"
|
||||
"# @title (Required on Colab) `authenticate_user()`\n",
|
||||
"import builtins\n",
|
||||
"import os\n",
|
||||
"import sys\n",
|
||||
"\n",
|
||||
"running_in_colab = \"google.colab\" in sys.modules and hasattr(builtins, \"get_ipython\")\n",
|
||||
"if running_in_colab and not os.getenv(\"IS_TESTING\"):\n",
|
||||
" from google.colab import auth\n",
|
||||
"\n",
|
||||
" auth.authenticate_user(project_id=PROJECT_ID)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "A69ooMk5ATkl"
|
||||
},
|
||||
"source": [
|
||||
"**2. Vertex AI Workbench**\n",
|
||||
"\n",
|
||||
"Make sure that [the Compute Engine default service account](https://cloud.google.com/compute/docs/access/service-accounts#default_service_account) running a Vertex AI workbench instance has the permission iam.serviceAccounts.actAs (most likely through [roles/iam.serviceAccountUser](https://cloud.google.com/iam/docs/understanding-roles#iam.serviceAccountUser)) at [the IAM & Admin page of the Cloud Console](https://console.cloud.google.com/iam-admin). This permission allows a workbench instance to act as the service account when interacting with other Google Cloud services."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "EMekZw5mAXG4"
|
||||
},
|
||||
"source": [
|
||||
"**3. Local JupyterLab instance, uncomment and run:**"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "weGVw4YSAgjT"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# !gcloud auth login"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -258,26 +303,21 @@
|
||||
"id": "3xRjU9DGQzh6"
|
||||
},
|
||||
"source": [
|
||||
"### Generate embeddings\n",
|
||||
"### Get Text Embeddings\n",
|
||||
"\n",
|
||||
"1. Set the model name. The latest models are\n",
|
||||
" * \"text-embedding-004\" for English.\n",
|
||||
" * \"text-multilingual-embedding-002\" for i18n.\n",
|
||||
" * \"text-embedding-preview-0815\" for English with text and code embeddings for Python and Java.\n",
|
||||
"1. Specify the model. The latest model is **gemini-embedding-001**, a state-of-the-art model built on the Gemini backbone that supports 100+ languages as well as code.\n",
|
||||
" \n",
|
||||
" See the [language coverage](https://cloud.google.com/vertex-ai/generative-ai/docs/embeddings/get-text-embeddings#language_coverage_for_textembedding-gecko-multilingual_models) for the list of supported languages.\n",
|
||||
" \n",
|
||||
" See [the public doc](https://cloud.google.com/vertex-ai/generative-ai/docs/embeddings/get-text-embeddings) for the full list of supported models.\n",
|
||||
"2. Set the task type, text, and title (*optional and valid only for task type \"RETRIEVAL_DOCUMENT\"*). The valid task types are:\n",
|
||||
" * \"RETRIEVAL_QUERY\"\n",
|
||||
" * \"RETRIEVAL_DOCUMENT\"\n",
|
||||
" * \"SEMANTIC_SIMILARITY\"\n",
|
||||
" * \"CLASSIFICATION\"\n",
|
||||
" * \"CLUSTERING\"\n",
|
||||
" * \"QUESTION_ANSWERING\" (*valid only for the latest models*)\n",
|
||||
" * \"FACT_VERIFICATION\" (*valid only for the latest models*)\n",
|
||||
" * \"CODE_RETRIEVAL_QUERY\" (*valid only for the preview-0815 model*)\n",
|
||||
"3. Set the output dimensionality (*optional and valid only for the latest models*)."
|
||||
" See the [supported text languages](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/text-embeddings-api#supported_text_languages) and the [supported language models](https://cloud.google.com/vertex-ai/generative-ai/docs/embeddings/get-text-embeddings#supported-models).\n",
|
||||
"2. Specify the task, text, and optional title (valid only for task **RETRIEVAL_DOCUMENT**). The valid task types are:\n",
|
||||
" * **RETRIEVAL_QUERY**\n",
|
||||
" * **RETRIEVAL_DOCUMENT**\n",
|
||||
" * **SEMANTIC_SIMILARITY**\n",
|
||||
" * **CLASSIFICATION**\n",
|
||||
" * **CLUSTERING**\n",
|
||||
" * **QUESTION_ANSWERING** (valid only for the latest models)\n",
|
||||
" * **FACT_VERIFICATION** (valid only for the latest models)\n",
|
||||
" * **CODE_RETRIEVAL_QUERY** (valid only for the latest models)\n",
|
||||
"3. Optionally specify the output dimensionality (valid only for the latest models)."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -290,11 +330,11 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title { run: \"auto\" }\n",
|
||||
"MODEL = \"text-embedding-004\" # @param [\"text-embedding-004\", \"text-multilingual-embedding-002\",\"text-embedding-preview-0815\",\"text-embedding-preview-0409\", \"text-multilingual-embedding-preview-0409\", \"textembedding-gecko@003\", \"textembedding-gecko-multilingual@001\"]\n",
|
||||
"TASK = \"RETRIEVAL_DOCUMENT\" # @param [\"RETRIEVAL_QUERY\", \"RETRIEVAL_DOCUMENT\", \"SEMANTIC_SIMILARITY\", \"CLASSIFICATION\", \"CLUSTERING\", \"QUESTION_ANSWERING\", \"FACT_VERIFICATION\", \"CODE_RETRIEVAL_QUERY\"]\n",
|
||||
"TEXT = \"Banana Muffin?\" # @param {type:\"string\"}\n",
|
||||
"MODEL = \"gemini-embedding-001\" # @param [\"gemini-embedding-001\", \"text-embedding-005\", \"text-embedding-004\", \"text-multilingual-embedding-002\"]\n",
|
||||
"TASK = \"RETRIEVAL_QUERY\" # @param [\"RETRIEVAL_QUERY\", \"RETRIEVAL_DOCUMENT\", \"SEMANTIC_SIMILARITY\", \"CLASSIFICATION\", \"CLUSTERING\", \"QUESTION_ANSWERING\", \"FACT_VERIFICATION\", \"CODE_RETRIEVAL_QUERY\"]\n",
|
||||
"TEXT = \"Gemini Embedding is a state-of-the-art embedding model.\" # @param {type:\"string\"}\n",
|
||||
"TITLE = \"\" # @param {type:\"string\"}\n",
|
||||
"OUTPUT_DIMENSIONALITY = 256 # @param [1, 768, \"None\"] {type:\"raw\", allow-input:true}\n",
|
||||
"OUTPUT_DIMENSIONALITY = 768 # @param [1, 3072, \"None\"] {type:\"raw\", allow-input:true}\n",
|
||||
"\n",
|
||||
"if not MODEL:\n",
|
||||
" raise ValueError(\"MODEL must be specified.\")\n",
|
||||
@@ -303,23 +343,22 @@
|
||||
"if TITLE and TASK != \"RETRIEVAL_DOCUMENT\":\n",
|
||||
" raise ValueError(\"TITLE can only be specified for TASK 'RETRIEVAL_DOCUMENT'\")\n",
|
||||
"if OUTPUT_DIMENSIONALITY is not None and MODEL not in [\n",
|
||||
" \"text-embedding-005\",\n",
|
||||
" \"text-embedding-004\",\n",
|
||||
" \"gemini-embedding-001\",\n",
|
||||
" \"text-multilingual-embedding-002\",\n",
|
||||
" \"text-embedding-preview-0815\",\n",
|
||||
" \"text-embedding-preview-0409\",\n",
|
||||
" \"text-multilingual-embedding-preview-0409\",\n",
|
||||
"]:\n",
|
||||
" raise ValueError(f\"OUTPUT_DIMENTIONALITY cannot be specified for model '{MODEL}'.\")\n",
|
||||
"if TASK in [\"QUESTION_ANSWERING\", \"FACT_VERIFICATION\"] and MODEL not in [\n",
|
||||
" \"text-embedding-005\",\n",
|
||||
" \"text-embedding-004\",\n",
|
||||
" \"gemini-embedding-001\",\n",
|
||||
" \"text-multilingual-embedding-002\",\n",
|
||||
" \"text-embedding-preview-0815\",\n",
|
||||
" \"text-embedding-preview-0409\",\n",
|
||||
" \"text-multilingual-embedding-preview-0409\",\n",
|
||||
"]:\n",
|
||||
" raise ValueError(f\"TASK '{TASK}' is not valid for model '{MODEL}'.\")\n",
|
||||
"if TASK in [\"CODE_RETRIEVAL_QUERY\"] and MODEL not in [\n",
|
||||
" \"text-embedding-preview-0815\",\n",
|
||||
" \"text-embedding-005\",\n",
|
||||
" \"gemini-embedding-001\"\n",
|
||||
"]:\n",
|
||||
" raise ValueError(f\"TASK '{TASK}' is not valid for model '{MODEL}'.\")"
|
||||
]
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Copyright 2024 Google LLC\n",
|
||||
"# @title Copyright 2025 Google LLC\n",
|
||||
"#\n",
|
||||
"# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
|
||||
"# you may not use this file except in compliance with the License.\n",
|
||||
@@ -34,22 +34,22 @@
|
||||
"\n",
|
||||
"<table align=\"left\">\n",
|
||||
" <td style=\"text-align: center\">\n",
|
||||
" <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/generative_ai/text_embeddings.ipynb\">\n",
|
||||
" <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/generative_ai/text_embedding_new_api.ipynb\">\n",
|
||||
" <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Google Colaboratory logo\"><br> Open in Colab\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td style=\"text-align: center\">\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/colab/import/https:%2F%2Fraw.githubusercontent.com%2FGoogleCloudPlatform%2Fvertex-ai-samples%2Fmain%2Fnotebooks%2Fofficial%2Fgenerative_ai%2Ftext_embeddings.ipynb\">\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/colab/import/https:%2F%2Fraw.githubusercontent.com%2FGoogleCloudPlatform%2Fvertex-ai-samples%2Fmain%2Fnotebooks%2Fofficial%2Fgenerative_ai%2Ftext_embedding_new_api.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://cloud.google.com/ml-engine/images/colab-enterprise-logo-32px.png\" alt=\"Google Cloud Colab Enterprise logo\"><br> Open in Colab Enterprise\n",
|
||||
" </a>\n",
|
||||
" </td> \n",
|
||||
" <td style=\"text-align: center\">\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/generative_ai/text_embeddings.ipynb\">\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/generative_ai/text_embedding_new_api.ipynb\">\n",
|
||||
" <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\"><br> Open in Workbench\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td style=\"text-align: center\">\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/generative_ai/text_embeddings.ipynb\">\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/generative_ai/text_embedding_new_api.ipynb\">\n",
|
||||
" <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\"><br> View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
@@ -64,7 +64,7 @@
|
||||
"source": [
|
||||
"## Overview\n",
|
||||
"\n",
|
||||
"This notebook will walk you through the process of getting text embeddings given a text-embedding model such as **text-embedding-005** and **text-multilingual-embedding-002**. See also [getting text embeddings](https://cloud.google.com/vertex-ai/docs/generative-ai/embeddings/get-text-embeddings#api_changes_to_models_released_in_or_after_august_2023)."
|
||||
"This notebook will walk you through the process of getting text embeddings given a text-embedding model such as **gemini-embedding-001**. See also [getting text embeddings](https://cloud.google.com/vertex-ai/docs/generative-ai/embeddings/get-text-embeddings#api_changes_to_models_released_in_or_after_august_2023)."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -305,9 +305,7 @@
|
||||
"source": [
|
||||
"### Get Text Embeddings\n",
|
||||
"\n",
|
||||
"1. Specify the model. The latest models are\n",
|
||||
" * **text-embedding-005** for English.\n",
|
||||
" * **text-multilingual-embedding-002** for supported, international languages.\n",
|
||||
"1. Specify the model. The latest model is **gemini-embedding-001**, a state-of-the-art model built on the Gemini backbone that supports 100+ languages as well as code.\n",
|
||||
" \n",
|
||||
" See the [supported text languages](https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/text-embeddings-api#supported_text_languages) and the [supported language models](https://cloud.google.com/vertex-ai/generative-ai/docs/embeddings/get-text-embeddings#supported-models).\n",
|
||||
"2. Specify the task, text, and optional title (valid only for task **RETRIEVAL_DOCUMENT**). The valid task types are:\n",
|
||||
@@ -332,11 +330,11 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title { run: \"auto\" }\n",
|
||||
"MODEL = \"text-embedding-005\" # @param [\"text-embedding-005\", \"text-embedding-004\", \"text-multilingual-embedding-002\"]\n",
|
||||
"TASK = \"CODE_RETRIEVAL_QUERY\" # @param [\"RETRIEVAL_QUERY\", \"RETRIEVAL_DOCUMENT\", \"SEMANTIC_SIMILARITY\", \"CLASSIFICATION\", \"CLUSTERING\", \"QUESTION_ANSWERING\", \"FACT_VERIFICATION\", \"CODE_RETRIEVAL_QUERY\"]\n",
|
||||
"TEXT = \"Banana Muffin?\" # @param {type:\"string\"}\n",
|
||||
"MODEL = \"gemini-embedding-001\" # @param [\"gemini-embedding-001\", \"text-embedding-005\", \"text-embedding-004\", \"text-multilingual-embedding-002\"]\n",
|
||||
"TASK = \"RETRIEVAL_QUERY\" # @param [\"RETRIEVAL_QUERY\", \"RETRIEVAL_DOCUMENT\", \"SEMANTIC_SIMILARITY\", \"CLASSIFICATION\", \"CLUSTERING\", \"QUESTION_ANSWERING\", \"FACT_VERIFICATION\", \"CODE_RETRIEVAL_QUERY\"]\n",
|
||||
"TEXT = \"Gemini Embedding is a state-of-the-art embedding model.\" # @param {type:\"string\"}\n",
|
||||
"TITLE = \"\" # @param {type:\"string\"}\n",
|
||||
"OUTPUT_DIMENSIONALITY = 256 # @param [1, 768, \"None\"] {type:\"raw\", allow-input:true}\n",
|
||||
"OUTPUT_DIMENSIONALITY = 768 # @param [1, 3072, \"None\"] {type:\"raw\", allow-input:true}\n",
|
||||
"\n",
|
||||
"if not MODEL:\n",
|
||||
" raise ValueError(\"MODEL must be specified.\")\n",
|
||||
@@ -347,17 +345,20 @@
|
||||
"if OUTPUT_DIMENSIONALITY is not None and MODEL not in [\n",
|
||||
" \"text-embedding-005\",\n",
|
||||
" \"text-embedding-004\",\n",
|
||||
" \"gemini-embedding-001\",\n",
|
||||
" \"text-multilingual-embedding-002\",\n",
|
||||
"]:\n",
|
||||
" raise ValueError(f\"OUTPUT_DIMENTIONALITY cannot be specified for model '{MODEL}'.\")\n",
|
||||
"if TASK in [\"QUESTION_ANSWERING\", \"FACT_VERIFICATION\"] and MODEL not in [\n",
|
||||
" \"text-embedding-005\",\n",
|
||||
" \"text-embedding-004\",\n",
|
||||
" \"gemini-embedding-001\",\n",
|
||||
" \"text-multilingual-embedding-002\",\n",
|
||||
"]:\n",
|
||||
" raise ValueError(f\"TASK '{TASK}' is not valid for model '{MODEL}'.\")\n",
|
||||
"if TASK in [\"CODE_RETRIEVAL_QUERY\"] and MODEL not in [\n",
|
||||
" \"text-embedding-005\",\n",
|
||||
" \"gemini-embedding-001\"\n",
|
||||
"]:\n",
|
||||
" raise ValueError(f\"TASK '{TASK}' is not valid for model '{MODEL}'.\")"
|
||||
]
|
||||
|
||||
@@ -181,51 +181,12 @@
|
||||
"# Don't bother installing tensorflow or explainable_ai_sdk on Colab\n",
|
||||
"\n",
|
||||
"# Install required packages.\n",
|
||||
"! pip3 install --upgrade --quiet \\\n",
|
||||
"%pip install --upgrade --quiet \\\n",
|
||||
" google-cloud-aiplatform[full] \\\n",
|
||||
" google-cloud-bigquery \\\n",
|
||||
" explainable_ai_sdk \\\n",
|
||||
" tensorflow"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "restart"
|
||||
},
|
||||
"source": [
|
||||
"### Restart runtime (Colab only)\n",
|
||||
"\n",
|
||||
"To use the newly installed packages, you must restart the runtime on Google Colab."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "D-ZBOjErv5mM"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import sys\n",
|
||||
"\n",
|
||||
"if \"google.colab\" in sys.modules:\n",
|
||||
"\n",
|
||||
" import IPython\n",
|
||||
"\n",
|
||||
" app = IPython.Application.instance()\n",
|
||||
" app.kernel.do_shutdown(True)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "ee775571c2b5"
|
||||
},
|
||||
"source": [
|
||||
"<div class=\"alert alert-block alert-warning\">\n",
|
||||
"<b>⚠️ The kernel is going to restart. Wait until it's finished before continuing to the next step. ⚠️</b>\n",
|
||||
"</div>\n"
|
||||
" tensorflow \\\n",
|
||||
" \"numpy<2.0\""
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -594,7 +555,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"MODEL_PATH = \"gs://mco-mm/churn\"\n",
|
||||
"MODEL_PATH = \"gs://cloud-samples-data/vertex-ai/model-deployment/models/churn\"\n",
|
||||
"\n",
|
||||
"params = {\"sampled_shapley_attribution\": {\"path_count\": 10}}\n",
|
||||
"EXPLAIN_PARAMS = aiplatform.explain.ExplanationParameters(params)\n",
|
||||
@@ -826,7 +787,7 @@
|
||||
"MONITOR_INTERVAL = 1 # @param {type:\"number\"}\n",
|
||||
"\n",
|
||||
"# URI to training dataset.\n",
|
||||
"DATASET_BQ_URI = \"bq://mco-mm.bqmlga4.train\" # @param {type:\"string\"}\n",
|
||||
"DATASET_BQ_URI = \"bq://storage-samples.bqmlga4.train\" # @param {type:\"string\"}\n",
|
||||
"# Prediction target column name in training dataset.\n",
|
||||
"TARGET = \"churned\"\n",
|
||||
"\n",
|
||||
|
||||
@@ -136,52 +136,12 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Install the packages\n",
|
||||
"! pip3 install --upgrade google-cloud-aiplatform \\\n",
|
||||
"%pip install --upgrade google-cloud-aiplatform \\\n",
|
||||
" google-cloud-storage \\\n",
|
||||
" kfp \\\n",
|
||||
" google-cloud-pipeline-components"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "blGlVGFYW9Pt"
|
||||
},
|
||||
"source": [
|
||||
"### Restart runtime (Colab only)\n",
|
||||
"\n",
|
||||
"To use the newly installed packages, you must restart the runtime on Google Colab."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "0JrvuK6LUYnQ"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import sys\n",
|
||||
"\n",
|
||||
"if \"google.colab\" in sys.modules:\n",
|
||||
"\n",
|
||||
" import IPython\n",
|
||||
"\n",
|
||||
" app = IPython.Application.instance()\n",
|
||||
" app.kernel.do_shutdown(True)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "ffcde4d56c00"
|
||||
},
|
||||
"source": [
|
||||
"<div class=\"alert alert-block alert-warning\">\n",
|
||||
"<b>⚠️ The kernel is going to restart. Wait until it's finished before continuing to the next step. ⚠️</b>\n",
|
||||
"</div>\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
|
||||
@@ -0,0 +1,827 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "UUTFwSh5u9OK"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Copyright 2024 Google LLC\n",
|
||||
"#\n",
|
||||
"# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
|
||||
"# you may not use this file except in compliance with the License.\n",
|
||||
"# You may obtain a copy of the License at\n",
|
||||
"#\n",
|
||||
"# https://www.apache.org/licenses/LICENSE-2.0\n",
|
||||
"#\n",
|
||||
"# Unless required by applicable law or agreed to in writing, software\n",
|
||||
"# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
|
||||
"# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
|
||||
"# See the License for the specific language governing permissions and\n",
|
||||
"# limitations under the License."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "qsLff0QbdE8D"
|
||||
},
|
||||
"source": [
|
||||
"# Online Prediction Dedicated Endpint\n",
|
||||
"\n",
|
||||
"<table align=\"left\">\n",
|
||||
" <td style=\"text-align: center\">\n",
|
||||
" <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/prediction/get_started_with_dedicated_endpoint.ipynb\">\n",
|
||||
" <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Google Colaboratory logo\"><br> Open in Colab\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td style=\"text-align: center\">\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/colab/import/https:%2F%2Fraw.githubusercontent.com%2FGoogleCloudPlatform%2Fvertex-ai-samples%2Fmain%2Fnotebooks%2Fofficial%2Fprediction%2Fget_started_with_dedicated_endpoint.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://cloud.google.com/ml-engine/images/colab-enterprise-logo-32px.png\" alt=\"Google Cloud Colab Enterprise logo\"><br> Open in Colab Enterprise\n",
|
||||
" </a>\n",
|
||||
" </td> \n",
|
||||
" <td style=\"text-align: center\">\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/prediction/get_started_with_dedicated_endpoint.ipynb\">\n",
|
||||
" <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\"><br> Open in Workbench\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td style=\"text-align: center\">\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/prediction/get_started_with_dedicated_endpoint.ipynb\">\n",
|
||||
" <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\"><br> View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
"</table>"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "0C7kTlrH1bO8"
|
||||
},
|
||||
"source": [
|
||||
"## Overview\n",
|
||||
"\n",
|
||||
"A dedicated public endpoint is a public endpoint for online prediction. It offers the following benefits:\n",
|
||||
"\n",
|
||||
"* Dedicated networking: When you send a prediction request to a dedicated public endpoint, it is isolated from other users' traffic.\n",
|
||||
"* Optimized network latency\n",
|
||||
"* Larger payload support: Up to 10 MB.\n",
|
||||
"* Longer request timeouts: Configurable up to 1 hour.\n",
|
||||
"* Generative AI-ready: Streaming and gRPC are supported."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "D2sWuTr81v9m"
|
||||
},
|
||||
"source": [
|
||||
"## Get started"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "RakMIliNYh8O"
|
||||
},
|
||||
"source": [
|
||||
"### Install Vertex AI SDK for Python and other required packages"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "nSHmJT9cTggu"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! pip3 install --upgrade --quiet google-cloud-aiplatform"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "936Zz5YI2NeA"
|
||||
},
|
||||
"source": [
|
||||
"### Restart runtime (Colab only)\n",
|
||||
"\n",
|
||||
"To use the newly installed packages, you must restart the runtime on Google Colab."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "66oJ55lG2Tiq"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import sys\n",
|
||||
"\n",
|
||||
"if \"google.colab\" in sys.modules:\n",
|
||||
"\n",
|
||||
" import IPython\n",
|
||||
"\n",
|
||||
" app = IPython.Application.instance()\n",
|
||||
" app.kernel.do_shutdown(True)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "Clr61ben2WwY"
|
||||
},
|
||||
"source": [
|
||||
"<div class=\"alert alert-block alert-warning\">\n",
|
||||
"<b>⚠️ The kernel is going to restart. Wait until it's finished before continuing to the next step. ⚠️</b>\n",
|
||||
"</div>\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "v848aGbn2acH"
|
||||
},
|
||||
"source": [
|
||||
"### Authenticate your notebook environment (Colab only)\n",
|
||||
"\n",
|
||||
"Authenticate your environment on Google Colab.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "IVeoyQPz2cfh"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import sys\n",
|
||||
"\n",
|
||||
"if \"google.colab\" in sys.modules:\n",
|
||||
"\n",
|
||||
" from google.colab import auth\n",
|
||||
"\n",
|
||||
" auth.authenticate_user()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "HId-ySlY2jlI"
|
||||
},
|
||||
"source": [
|
||||
"### Set Google Cloud project information and initialize Vertex AI SDK for Python\n",
|
||||
"\n",
|
||||
"To get started using Vertex AI, you must have an existing Google Cloud project and [enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com). Learn more about [setting up a project and a development environment](https://cloud.google.com/vertex-ai/docs/start/cloud-environment)."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "Y4gnZI9OX6VJ"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Set project id to be the current project ID.\n",
|
||||
"\n",
|
||||
"PROJECT_ID = \"[your-project-id]\" # @param {type:\"string\"}\n",
|
||||
"LOCATION = \"us-central1\" # @param {type:\"string\"}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "-NrpFROTjoVL"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from google.cloud import aiplatform\n",
|
||||
"\n",
|
||||
"aiplatform.init(project=PROJECT_ID, location=LOCATION)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "moS794OKaaCt"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Create GCS Bucket\n",
|
||||
"BUCKET_URI = \"gs://your-bucket-name-unique\" # @param {type:\"string\"}\n",
|
||||
"! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "SASm5tKB2JGF"
|
||||
},
|
||||
"source": [
|
||||
"## Create Dedicated Endpoint and Deploy Model"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "taUmNhwP2S3W"
|
||||
},
|
||||
"source": [
|
||||
"### Create Dedicated Endpoint\n",
|
||||
"When creating endpoint, set `dedicated_endpoint_enabled` to True."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "IZUfl8Q82crP"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"endpoint = aiplatform.Endpoint.create(\n",
|
||||
" display_name=\"test-dedicated-endpoint\",\n",
|
||||
" dedicated_endpoint_enabled=True,\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "swERjzZ-a_Nd"
|
||||
},
|
||||
"source": [
|
||||
"### Prepare Test Models\n",
|
||||
"\n",
|
||||
"We prepared a tensorflow test model, feel free to use your own models."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "0fea4d3018ab"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! pip freeze | grep google-cloud-aiplatform"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "bA2zhbzJ1R0k"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# List all your models\n",
|
||||
"for my_model in aiplatform.Model.list():\n",
|
||||
" print(my_model.display_name)\n",
|
||||
" print(my_model.gca_resource.name)\n",
|
||||
"\n",
|
||||
"# If you want to use an existing model, use the resource id\n",
|
||||
"# model = aiplatform.Model('projects/12345/locations/us-central1/models/456789')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "3PtFCQNHbloQ"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Upload a new tensorflow model\n",
|
||||
"# @markdown You can skip this if use an existing model.\n",
|
||||
"\n",
|
||||
"# TF Model\n",
|
||||
"DISPLAY_NAME = \"tensorflow model\" # @param {type:\"string\"}\n",
|
||||
"ARTIFACT_URI = BUCKET_URI + \"/tensorflow\"\n",
|
||||
"IMAGE_URI = \"us-docker.pkg.dev/vertex-ai/prediction/tf2-cpu.2-12:latest\"\n",
|
||||
"\n",
|
||||
"# Copy Test Models to the Bucket\n",
|
||||
"! gsutil -m cp -r \"gs://cloud-samples-data/vertex-ai/prediction/test-models-requests/tensorflow/*\" {ARTIFACT_URI}\n",
|
||||
"\n",
|
||||
"model = aiplatform.Model.upload(\n",
|
||||
" display_name=DISPLAY_NAME,\n",
|
||||
" artifact_uri=ARTIFACT_URI,\n",
|
||||
" serving_container_image_uri=IMAGE_URI,\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "USWCW-SNo-9M"
|
||||
},
|
||||
"source": [
|
||||
"#### Deploy Model\n",
|
||||
"\n",
|
||||
"Deploy the model to the endpoint. If it is your first model with dedicated endpoint, it will take ~30 min."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "S_eRJglhpVfL"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"endpoint.deploy(model=model, traffic_percentage=100, machine_type=\"e2-standard-8\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "civyNQaPr4QD"
|
||||
},
|
||||
"source": [
|
||||
"## Make Predictions\n",
|
||||
"\n",
|
||||
"Dedicated endpoint cannot be accessed with the shared `aiplatform.googleapis.com` DNS. \\\n",
|
||||
"Instead, it has its own dedicated DNS\n",
|
||||
"```\n",
|
||||
"DEDICATED_DNS = f\"https://{endpoint.gca_resource.dedicated_endpoint_dns}/v1beta1/{endpoint.resource_name}\"\n",
|
||||
"# or\n",
|
||||
"DEDICATED_DNS = f\"https://{ENDPOINT_ID}.{LOCATION}-{PROJECT_NUMBER}.prediction.vertexai.goog\"\n",
|
||||
"```\n",
|
||||
"\n",
|
||||
"Python SDK has been integrated to support dedicated endpoint using a flag `use_dedicated_endpoint`.\n",
|
||||
"\n",
|
||||
"Alternatively, you can send HTTP/GRPC request directly to this DNS in any language you prefer."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "I-O9U63juoWE"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Predict\n",
|
||||
"# @markdown You can use client library.\n",
|
||||
"\n",
|
||||
"use_python_sdk = True # @param {type:\"boolean\"}\n",
|
||||
"# @markdown response = my_endpoint.predict( \\\n",
|
||||
"# @markdown   instances=[{\"feat_1\":val_1, \"feat_2\":val_2}]}, \\\n",
|
||||
"# @markdown   headers = {'Content-Type':'application/json'}, \\\n",
|
||||
"# @markdown   dedicated_endpoint=True, \\\n",
|
||||
"# @markdown )\n",
|
||||
"\n",
|
||||
"if use_python_sdk:\n",
|
||||
" instances = [\n",
|
||||
" {\"dense_input\": [14.0, 7.0, 2545.461893666405, 54.2, 48.5, 0.0, 61.3, 0.0, 0.0, 0.0]},\n",
|
||||
" {\"dense_input\": [28.0, 14.0, 1234, 27.1, 90, 0.0, 61.3, 0.0, 0.0, 0.0]}\n",
|
||||
" ]\n",
|
||||
" response = endpoint.predict(instances=instances, use_dedicated_endpoint=True)\n",
|
||||
" print(response)\n",
|
||||
"\n",
|
||||
"# @markdown You can also make HTTP/GRPC request directly.\n",
|
||||
"\n",
|
||||
"use_http = True # @param {type:\"boolean\"}\n",
|
||||
"# @markdown POST request to the following URL:\\\n",
|
||||
"# @markdown `https://ENDPOINT_ID.us-central1-PROJECT_NUMBER.prediction.vertexai.goog/v1/projects/PROJECT_NUMBER/locations/LOCATION/endpoints/ENDPOINT_ID:predict`\n",
|
||||
"\n",
|
||||
"if use_http:\n",
|
||||
" request_path = f\"https://{endpoint.gca_resource.dedicated_endpoint_dns}/v1/{endpoint.resource_name}:predict\"\n",
|
||||
" ! curl {request_path} -X POST -H \"Content-Type: application/json\" -H \"Authorization: Bearer `gcloud auth print-access-token`\" -d \\\n",
|
||||
" '{{ \\\n",
|
||||
" \"instances\": [ \\\n",
|
||||
" {{\"dense_input\": [14.0, 7.0, 2545.461893666405, 54.2, 48.5, 0.0, 61.3, 0.0, 0.0, 0.0]}}, \\\n",
|
||||
" {{\"dense_input\": [28.0, 14.0, 1234, 27.1, 90, 0.0, 61.3, 0.0, 0.0, 0.0]}} \\\n",
|
||||
" ] \\\n",
|
||||
" }}'"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "RRoiziQ9FAw6"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Raw Predict\n",
|
||||
"# @markdown You can use client library.\n",
|
||||
"\n",
|
||||
"use_python_sdk = True # @param {type:\"boolean\"}\n",
|
||||
"# @markdown response = my_endpoint.raw_predict( \\\n",
|
||||
"# @markdown    body = b'{\"instances\":[{\"feat_1\":val_1, \"feat_2\":val_2}]}', \\\n",
|
||||
"# @markdown    headers = {'Content-Type':'application/json'}, \\\n",
|
||||
"# @markdown    dedicated_endpoint=True, \\\n",
|
||||
"# @markdown ) \\\n",
|
||||
"# @markdown status_code = response.status_code \\\n",
|
||||
"# @markdown results = json.dumps(response.text)\n",
|
||||
"\n",
|
||||
"if use_python_sdk:\n",
|
||||
" body = b'{ \\\n",
|
||||
" \"instances\": [ \\\n",
|
||||
" {\"dense_input\": [14.0, 7.0, 2545.461893666405, 54.2, 48.5, 0.0, 61.3, 0.0, 0.0, 0.0]}, \\\n",
|
||||
" {\"dense_input\": [28.0, 14.0, 1234, 27.1, 90, 0.0, 61.3, 0.0, 0.0, 0.0]} \\\n",
|
||||
" ]\\\n",
|
||||
" }'\n",
|
||||
" import os\n",
|
||||
" token = !gcloud auth print-access-token\n",
|
||||
" token = token[0]\n",
|
||||
" headers = {\"content-type\": \"application/json\", \"Authorization\": f\"Bearer {token}\"}\n",
|
||||
" response = endpoint.raw_predict(body=body, headers=headers, use_dedicated_endpoint=True)\n",
|
||||
" # print(response.status_code)\n",
|
||||
" print(response.text)\n",
|
||||
"\n",
|
||||
"# @markdown You can also make HTTP/GRPC request directly.\n",
|
||||
"\n",
|
||||
"use_http = True # @param {type:\"boolean\"}\n",
|
||||
"# @markdown POST request to the following URL:\\\n",
|
||||
"# @markdown `https://ENDPOINT_ID.us-central1-PROJECT_NUMBER.prediction.vertexai.goog/v1/projects/PROJECT_NUMBER/locations/LOCATION/endpoints/ENDPOINT_ID:rawPredict`\n",
|
||||
"\n",
|
||||
"if use_http:\n",
|
||||
" request_path = f\"https://{endpoint.gca_resource.dedicated_endpoint_dns}/v1/{endpoint.resource_name}:rawPredict\"\n",
|
||||
" ! curl {request_path} -X POST -H \"Content-Type: application/json\" -H \"Authorization: Bearer `gcloud auth print-access-token`\" -d \\\n",
|
||||
" '{{ \\\n",
|
||||
" \"instances\": [ \\\n",
|
||||
" {{\"dense_input\": [14.0, 7.0, 2545.461893666405, 54.2, 48.5, 0.0, 61.3, 0.0, 0.0, 0.0]}}, \\\n",
|
||||
" {{\"dense_input\": [28.0, 14.0, 1234, 27.1, 90, 0.0, 61.3, 0.0, 0.0, 0.0]}} \\\n",
|
||||
" ] \\\n",
|
||||
" }}'"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "IbZDIJlIRl1b"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# @title Stream Raw Predict\n",
|
||||
"# @markdown You can use client library.\n",
|
||||
"\n",
|
||||
"use_python_sdk = True # @param {type:\"boolean\"}\n",
|
||||
"# @markdown for stream_response in my_endpoint.stream_raw_predict( \\\n",
|
||||
"# @markdown    body = b'{\"instances\":[{\"feat_1\":val_1, \"feat_2\":val_2}]}', \\\n",
|
||||
"# @markdown    headers = {'Content-Type':'application/json'}, \\\n",
|
||||
"# @markdown    use_dedicated_endpoint=True, \\\n",
|
||||
"# @markdown ): \\\n",
|
||||
"# @markdown    status_code = response.status_code \\\n",
|
||||
"# @markdown    stream_result = json.dumps(response.text)\n",
|
||||
"\n",
|
||||
"if use_python_sdk:\n",
|
||||
" body = b'{ \\\n",
|
||||
" \"instances\": [ \\\n",
|
||||
" {\"dense_input\": [14.0, 7.0, 2545.461893666405, 54.2, 48.5, 0.0, 61.3, 0.0, 0.0, 0.0]}, \\\n",
|
||||
" {\"dense_input\": [28.0, 14.0, 1234, 27.1, 90, 0.0, 61.3, 0.0, 0.0, 0.0]} \\\n",
|
||||
" ]\\\n",
|
||||
" }'\n",
|
||||
" import os\n",
|
||||
" token = !gcloud auth print-access-token\n",
|
||||
" token = token[0]\n",
|
||||
" headers = {\"content-type\": \"application/json\", \"Authorization\": f\"Bearer {token}\"}\n",
|
||||
" for stream_response in endpoint.raw_predict(body=body, headers=headers, use_dedicated_endpoint=True):\n",
|
||||
" # print(response.status_code)\n",
|
||||
" print(response.text)\n",
|
||||
"\n",
|
||||
"# @markdown You can also make HTTP/GRPC request directly.\n",
|
||||
"\n",
|
||||
"use_http = True # @param {type:\"boolean\"}\n",
|
||||
"# @markdown POST request to the following URL:\\\n",
|
||||
"# @markdown `https://ENDPOINT_ID.us-central1-PROJECT_NUMBER.prediction.vertexai.goog/v1/projects/PROJECT_NUMBER/locations/LOCATION/endpoints/ENDPOINT_ID:streamRawPredict`\n",
|
||||
"\n",
|
||||
"if use_http:\n",
|
||||
" request_path = f\"https://{endpoint.gca_resource.dedicated_endpoint_dns}/v1/{endpoint.resource_name}:streamRawPredict\"\n",
|
||||
" ! curl {request_path} -X POST -H \"Content-Type: application/json\" -H \"Authorization: Bearer `gcloud auth print-access-token`\" -d \\\n",
|
||||
" '{{ \\\n",
|
||||
" \"instances\": [ \\\n",
|
||||
" {{\"dense_input\": [14.0, 7.0, 2545.461893666405, 54.2, 48.5, 0.0, 61.3, 0.0, 0.0, 0.0]}}, \\\n",
|
||||
" {{\"dense_input\": [28.0, 14.0, 1234, 27.1, 90, 0.0, 61.3, 0.0, 0.0, 0.0]}} \\\n",
|
||||
" ] \\\n",
|
||||
" }}'"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "35gPjIcSfIVY"
|
||||
},
|
||||
"source": [
|
||||
"### Chat Completion (Model Garden Only)\n",
|
||||
"**You can use OpenAI client library to do chat completion.**\n",
|
||||
"\n",
|
||||
"```\n",
|
||||
"client = openai.OpenAI(base_url=DEDICATED_DNS, api_key=creds.token)\n",
|
||||
"model_response = client.chat.completions.create(\n",
|
||||
" model=\"your model\",\n",
|
||||
" messages=[{\"role\": \"user\", \"content\": user_message}],\n",
|
||||
" temperature=temperature,\n",
|
||||
" max_tokens=max_tokens,\n",
|
||||
" stream=stream,\n",
|
||||
")\n",
|
||||
"```\n",
|
||||
"\n",
|
||||
"**You can also make HTTP/GRPC request directly.**\n",
|
||||
"\n",
|
||||
"POST request to the following URL:\n",
|
||||
"```\n",
|
||||
"curl -X POST -H \"Content-Type: application/json\" \\\n",
|
||||
" -H \"Authorization: Bearer `gcloud auth print-access-token`\" \\\n",
|
||||
" https://DEDICATED_DNS/v1beta1/projects/PROJECT_NUMBER/locations/LOCATION/endpoints/ENDPOINT_ID/chat/completions -d PAYLOAD\n",
|
||||
"```\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "Ka-pVhA1Xf-g"
|
||||
},
|
||||
"source": [
|
||||
"## Supported Features"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "_TCktMxbA4mb"
|
||||
},
|
||||
"source": [
|
||||
"### Traffic split\n",
|
||||
"\n",
|
||||
"Deploy another model, and update the traffic split to be 50:50, this should take less than 5 min using the test model. After the deployment is done, you can rerun the prediction again for multiple times, you should be able to see the deployed_model_id are different."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "bVLgpRiRBEL7"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"endpoint.deploy(model=model, traffic_percentage=50, machine_type=\"e2-standard-8\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "2dx975IkCv7v"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"instances = [\n",
|
||||
" {\n",
|
||||
" \"dense_input\": [\n",
|
||||
" 14.0,\n",
|
||||
" 7.0,\n",
|
||||
" 2545.461893666405,\n",
|
||||
" 54.2,\n",
|
||||
" 48.5,\n",
|
||||
" 0.0,\n",
|
||||
" 61.3,\n",
|
||||
" 0.0,\n",
|
||||
" 0.0,\n",
|
||||
" 0.0,\n",
|
||||
" ]\n",
|
||||
" },\n",
|
||||
" {\"dense_input\": [28.0, 14.0, 1234, 27.1, 90, 0.0, 61.3, 0.0, 0.0, 0.0]},\n",
|
||||
"]\n",
|
||||
"counter = {}\n",
|
||||
"for i in range(1000):\n",
|
||||
" response = endpoint.predict(instances=instances, use_dedicated_endpoint=True)\n",
|
||||
" if response.deployed_model_id in counter.keys():\n",
|
||||
" counter[response.deployed_model_id] += 1\n",
|
||||
" else:\n",
|
||||
" counter[response.deployed_model_id] = 1\n",
|
||||
"print(counter)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "2Hld7iDmEyiF"
|
||||
},
|
||||
"source": [
|
||||
"You can update the traffic split with the following command and run the code above again."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "HXPI-2q9Eh6X"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"deployed_model_id_0 = list(counter)[0]\n",
|
||||
"deployed_model_id_1 = list(counter)[1]\n",
|
||||
"\n",
|
||||
"endpoint.update(traffic_split={deployed_model_id_0: 20, deployed_model_id_1: 80})"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "oXG3G-7D4hUJ"
|
||||
},
|
||||
"source": [
|
||||
"### Custom Timeout\n",
|
||||
"\n",
|
||||
"Default 600s(10 min) timeout will be applied if the inference timeout is 0. Max timeout allowed is 1h. \\\n",
|
||||
"\n",
|
||||
"Use `EndpointService.UpdateEndpointLongRunning` to change the setting \\\n",
|
||||
"`EndpointService.UpdateEndpoint` doesn't support request/response logging change.\n",
|
||||
"\n",
|
||||
"```\n",
|
||||
"timeout_endpoint = aiplatform.Endpoint.create(\n",
|
||||
" display_name=\"test-dedicated-endpoint-with-timeout\",\n",
|
||||
" dedicated_endpoint_enabled=True,\n",
|
||||
" inference_timeout=1800, # Unit: Second.\n",
|
||||
")\n",
|
||||
"```\n",
|
||||
"Alternatively, you can create the endpoint using HTTP.\n",
|
||||
"\n",
|
||||
"Create endpoint with timeout sample:\n",
|
||||
"```\n",
|
||||
"curl -X POST -H \"Content-Type: application/json\" \\\n",
|
||||
" -H \"Authorization: Bearer `gcloud auth print-access-token`\" \\\n",
|
||||
" https://${DEDICATED_DNS}/v1beta1/projects/${PROJECT_NUMBER}/locations/us-central1/endpoints -d \\\n",
|
||||
" '{ \\\n",
|
||||
" displayName: \"test-dedicated-endpoint-with-timeout\", \\\n",
|
||||
" dedicatedEndpointEnabled: true, \\\n",
|
||||
" client_connection_config: \\\n",
|
||||
" { \\\n",
|
||||
" \"inferenceTimeout\": \\\n",
|
||||
" {\"seconds\": 20, \"nanos\": 0} \\\n",
|
||||
" } \\\n",
|
||||
" }'\n",
|
||||
"```\n",
|
||||
"Update endpoint long running sample:\n",
|
||||
"```\n",
|
||||
"curl -X POST \\\n",
|
||||
" -H \"Authorization: Bearer $(gcloud auth print-access-token)\" \\\n",
|
||||
" -H \"Content-Type: application/json\" \\\n",
|
||||
" https://${DEDICATED_DNS}/v1beta1/projects/${PROJECT_ID}/locations/${REGION}/endpoints/${ENDPOINT_ID}:update \\\n",
|
||||
" -d '{ \\\n",
|
||||
" \"endpoint\": { \\\n",
|
||||
" \"clientConnectionConfig\": { \\\n",
|
||||
" \"inferenceTimeout\": {\"seconds\": 5, \"nanos\": 0} \\\n",
|
||||
" }, \\\n",
|
||||
" } \\\n",
|
||||
" }'\n",
|
||||
"```"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "yDNTq4Zh39MS"
|
||||
},
|
||||
"source": [
|
||||
"### Request&Response Logging\n",
|
||||
"\n",
|
||||
"Request/Response logging will skip payload that exceed 10M which is the big query limit.\n",
|
||||
"\n",
|
||||
"Use `EndpointService.UpdateEndpointLongRunning` to change the setting \\\n",
|
||||
"`EndpointService.UpdateEndpoint` doesn't support request/response logging change.\n",
|
||||
"\n",
|
||||
"```\n",
|
||||
"logging_endpoint = aiplatform.Endpoint.create(\n",
|
||||
" display_name=\"test-dedicated-endpoint-with-logging\",\n",
|
||||
" dedicated_endpoint_enabled=True,\n",
|
||||
" enable_request_response_logging=True,\n",
|
||||
" request_response_logging_sampling_rate=1.0, # default 0.0\n",
|
||||
" request_response_logging_bq_destination_table=\"bq://test_logging\",\n",
|
||||
" # if not set, a new table will be created with name `bq://{project_id}.logging_{endpoint_display_name}_{endpoint_id}.request_response_logging`\n",
|
||||
")\n",
|
||||
"```\n",
|
||||
"\n",
|
||||
"Alternatively, use HTTP directly.\n",
|
||||
"\n",
|
||||
"Create endpoint sample\n",
|
||||
"```\n",
|
||||
"curl -X POST \\\n",
|
||||
" -H \"Authorization: Bearer $(gcloud auth print-access-token)\" -H \"Content-Type: application/json\" \\\n",
|
||||
" https://${DEDICATED_DNS}/v1beta1/projects/${PROJECT_ID}/locations/us-central1/endpoints \\\n",
|
||||
" -d '{ \\\n",
|
||||
" \"display_name\": \"test-dedicated-endpoint-with-logging\", \\\n",
|
||||
" \"dedicatedEndpointEnabled\": true, \\\n",
|
||||
" \"predict_request_response_logging_config\": { \\\n",
|
||||
" \"enabled\": true, \\\n",
|
||||
" \"sampling_rate\": 0.5, \\\n",
|
||||
" \"bigquery_destination\": { \\\n",
|
||||
" \"output_uri\": \"bq://my-project\" \\\n",
|
||||
" } \\\n",
|
||||
" } \\\n",
|
||||
" }'\n",
|
||||
"```\n",
|
||||
"\n",
|
||||
"Update endpoint long running request sample\n",
|
||||
"```\n",
|
||||
"curl -X POST \\\n",
|
||||
" -H \"Authorization: Bearer $(gcloud auth print-access-token)\" \\\n",
|
||||
" -H \"Content-Type: application/json\" \\\n",
|
||||
" https://${DEDICATED_DNS}/v1beta1/projects/${PROJECT_ID}/locations/${REGION}/endpoints/${ENDPOINT_ID}:update \\\n",
|
||||
" -d '{ \\\n",
|
||||
" \"endpoint\": { \\\n",
|
||||
" \"predict_request_response_logging_config\": { \\\n",
|
||||
" \"enabled\": true, \\\n",
|
||||
" \"sampling_rate\": 0.5, \\\n",
|
||||
" \"bigquery_destination\": { \\\n",
|
||||
" \"output_uri\": \"bq://my-project\" \\\n",
|
||||
" } \\\n",
|
||||
" }, \\\n",
|
||||
" } \\\n",
|
||||
" }'\n",
|
||||
"```\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "XW_BtPnEFPp4"
|
||||
},
|
||||
"source": [
|
||||
"## Cleanup"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "E4Ik3eKqdI_2"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"endpoint.undeploy_all()\n",
|
||||
"endpoint.delete()\n",
|
||||
"model.delete()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "BTz-2N1XunXB"
|
||||
},
|
||||
"source": [
|
||||
"Delete the bucket if needed."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "NPQT5Wv9lC3O"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil rm -r {BUCKET_URI}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "2iR_Q5K_ksWi"
|
||||
},
|
||||
"source": [
|
||||
"Optionally, you can use the following command to clean up all endpoint and models if needed."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "vkyvlwTgky0J"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# for e in aiplatform.Endpoint.list():\n",
|
||||
"# e.undeploy_all()\n",
|
||||
"# e.delete()"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"name": "get_started_with_dedicated_endpoint.ipynb",
|
||||
"toc_visible": true
|
||||
},
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"name": "python3"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 0
|
||||
}
|
||||
@@ -164,49 +164,9 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! pip3 install google-cloud-aiplatform[ray]==1.56.0 ray[data,train,tune,serve] google-cloud-bigquery-storage pyarrow gcsfs setuptools==69.5.1 \"numpy<2\" -q --no-warn-conflicts\n",
|
||||
"! pip3 install torch==2.1.2 torchvision==0.16.2 torchmetrics==1.2.1 torchserve==0.9.0 torch-model-archiver==0.9.0 -q --no-warn-conflicts\n",
|
||||
"! pip3 install google-auth==2.27.0 etils==1.5.2 -q --no-warn-conflicts"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "16220914acc5"
|
||||
},
|
||||
"source": [
|
||||
"### Restart runtime (Colab only)\n",
|
||||
"\n",
|
||||
"To use the newly installed packages, you must restart the runtime on Google Colab."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "157953ab28f0"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import sys\n",
|
||||
"\n",
|
||||
"if \"google.colab\" in sys.modules:\n",
|
||||
"\n",
|
||||
" import IPython\n",
|
||||
"\n",
|
||||
" app = IPython.Application.instance()\n",
|
||||
" app.kernel.do_shutdown(True)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "c87a2a5d7e35"
|
||||
},
|
||||
"source": [
|
||||
"<div class=\"alert alert-block alert-warning\">\n",
|
||||
"<b>⚠️ The kernel is going to restart. Wait until it's finished before continuing to the next step. ⚠️</b>\n",
|
||||
"</div>\n"
|
||||
"%pip install google-cloud-aiplatform[ray]==1.56.0 ray[data,train,tune,serve] google-cloud-bigquery-storage pyarrow gcsfs setuptools==69.5.1 \"numpy<2\" -q --no-warn-conflicts\n",
|
||||
"%pip install torch==2.1.2 torchvision==0.16.2 torchmetrics==1.2.1 torchserve==0.9.0 torch-model-archiver==0.9.0 -q --no-warn-conflicts\n",
|
||||
"%pip install google-auth==2.27.0 etils==1.5.2 -q --no-warn-conflicts"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Copyright 2024 Google LLC\n",
|
||||
"# Copyright 2025 Google LLC\n",
|
||||
"#\n",
|
||||
"# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
|
||||
"# you may not use this file except in compliance with the License.\n",
|
||||
@@ -156,9 +156,9 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! pip3 install --upgrade --quiet google-cloud-aiplatform[ray]==1.59.0\n",
|
||||
"! pip3 install --upgrade --quiet google-cloud-aiplatform[ray]\n",
|
||||
"! pip3 install --upgrade --quiet pyspark\n",
|
||||
"! pip3 install --upgrade --quiet ray[all]==2.9.3\n",
|
||||
"! pip3 install --upgrade --quiet ray[all]==2.42.0\n",
|
||||
"! pip3 install --upgrade --quiet raydp"
|
||||
]
|
||||
},
|
||||
@@ -317,11 +317,11 @@
|
||||
"source": [
|
||||
"%%writefile {DOCKER_DIR}/Dockerfile\n",
|
||||
"\n",
|
||||
"FROM us-docker.pkg.dev/vertex-ai/training/ray-cpu.2-9.py310:latest\n",
|
||||
"FROM us-docker.pkg.dev/vertex-ai/training/ray-cpu.2-42.py310:latest\n",
|
||||
"\n",
|
||||
"RUN apt-get update -y \\\n",
|
||||
" && apt-get install openjdk-21-jdk -y \\\n",
|
||||
" && pip install --no-cache-dir raydp pyarrow==14.0 pandas==2.1.0"
|
||||
" && apt-get install openjdk-17-jdk -y \\\n",
|
||||
" && pip install --no-cache-dir raydp pyarrow pandas"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -591,7 +591,8 @@
|
||||
" time.sleep(30)\n",
|
||||
" func()\n",
|
||||
" break\n",
|
||||
" except Exception:\n",
|
||||
" except Exception as e:\n",
|
||||
" print(f\"An error occurred: {e}\")\n",
|
||||
" continue\n",
|
||||
"\n",
|
||||
"\n",
|
||||
|
||||
Reference in New Issue
Block a user