xiaohei/model_garden
1
0
Fork 0
mirror of https://github.com/GoogleCloudPlatform/vertex-ai-samples.git synced 2026-09-26 14:42:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update Llama 3.1 MaaS naming.

Browse Source 
PiperOrigin-RevId: 687362552
This commit is contained in:
Vertex MG Team
2024-10-18 11:09:18 -07:00
committed by Copybara-Service
parent a7dd5aa5a9
commit fb1871d100
5 changed files with 183 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files
notebooks/community/model_garden/model_garden_llama_guard_deployment.ipynb
+4 -4
View File
@@ -429,7 +429,7 @@
"source": [
"## Use the Llama Guard models to safeguard LLM inputs and outputs with the Vertex Llama 3.1 API service\n",
"\n",
"We use [meta-llama/Llama-Guard-3-8B](https://huggingface.co/meta-llama/Llama-Guard-3-8B) to safeguard input and output conversations with the [Llama 3.1 405B Instruct model API service on Vertex](https://console.cloud.google.com/vertex-ai/publishers/meta/model-garden/llama3-405b-instruct-maas).\n",
"We use [meta-llama/Llama-Guard-3-8B](https://huggingface.co/meta-llama/Llama-Guard-3-8B) to safeguard input and output conversations with the [Llama 3.1 405B Instruct model API service on Vertex](https://console.cloud.google.com/vertex-ai/publishers/meta/model-garden/llama-3.1-405b-instruct-maas).\n",
"\n",
"Llama Guard 3 builds on the capabilities introduced with Llama Guard 2, adding three new categories, Defamation, Elections and Code Interpreter Abuse. Additionally this model is multilingual and a new prompt format is introduced, making Llama Guard 3’s prompt format consistent with Llama 3+ Instruct models.\n",
"\n",
@@ -471,10 +471,10 @@
"# Note: the credential lives for 1 hour by default (https://cloud.google.com/docs/authentication/token-types#at-lifetime); after expiration, it must be refreshed.\n",
"\n",
"client = openai.OpenAI(\n",
" base_url=f\"https://us-central1-aiplatform.googleapis.com/v1beta1/projects/{PROJECT_ID}/locations/{REGION}/endpoints/openapi\",\n",
" base_url=f\"https://us-central1-aiplatform.googleapis.com/v1/projects/{PROJECT_ID}/locations/{REGION}/endpoints/openapi\",\n",
" api_key=creds.token,\n",
")\n",
"LLAMA3_405B_INSTRUCT = \"meta/llama3-405b-instruct-maas\""
"LLAMA3_405B_INSTRUCT = \"meta/llama-3.1-405b-instruct-maas\""
]
},
{
@@ -587,7 +587,7 @@
"# Note: the credential lives for 1 hour by default (https://cloud.google.com/docs/authentication/token-types#at-lifetime); after expiration, it must be refreshed.\n",
"\n",
"client = openai.OpenAI(\n",
" base_url=f\"https://us-central1-aiplatform.googleapis.com/v1beta1/projects/{PROJECT_ID}/locations/{REGION}/endpoints/openapi\",\n",
" base_url=f\"https://us-central1-aiplatform.googleapis.com/v1/projects/{PROJECT_ID}/locations/{REGION}/endpoints/openapi\",\n",
" api_key=creds.token,\n",
")\n",
"LLAMA3_90B_VISION_INSTRUCT = \"meta/llama-3.2-90b-vision-instruct-maas\""
notebooks/community/model_garden/model_garden_openai_api_llama3_1.ipynb
+3 -3
View File
@@ -367,7 +367,7 @@
"MODEL_LOCATION = \"us-central1\"\n",
"\n",
"client = openai.OpenAI(\n",
" base_url=f\"https://{MODEL_LOCATION}-aiplatform.googleapis.com/v1beta1/projects/{PROJECT_ID}/locations/{MODEL_LOCATION}/endpoints/openapi/chat/completions?\",\n",
" base_url=f\"https://{MODEL_LOCATION}-aiplatform.googleapis.com/v1/projects/{PROJECT_ID}/locations/{MODEL_LOCATION}/endpoints/openapi/chat/completions?\",\n",
" api_key=credentials.token,\n",
")"
]
@@ -396,7 +396,7 @@
},
"outputs": [],
"source": [
"MODEL_ID = \"meta/llama3-405b-instruct-maas\" # @param {type:\"string\"} [\"meta/llama3-8b-instruct-maas\", \"meta/llama3-70b-instruct-maas\", \"meta/llama3-405b-instruct-maas\"]"
"MODEL_ID = \"meta/llama-3.1-405b-instruct-maas\" # @param {type:\"string\"} [\"meta/llama-3.1-8b-instruct-maas\", \"meta/llama-3.1-70b-instruct-maas\", \"meta/llama-3.1-405b-instruct-maas\"]"
]
},
{
@@ -821,7 +821,7 @@
"source": [
"llm = ChatOpenAI(\n",
" model=MODEL_ID,\n",
" base_url=f\"https://{MODEL_LOCATION}-aiplatform.googleapis.com/v1beta1/projects/{PROJECT_ID}/locations/{MODEL_LOCATION}/endpoints/openapi/chat/completions?\",\n",
" base_url=f\"https://{MODEL_LOCATION}-aiplatform.googleapis.com/v1/projects/{PROJECT_ID}/locations/{MODEL_LOCATION}/endpoints/openapi/chat/completions?\",\n",
" api_key=credentials.token,\n",
")\n",
"\n",
notebooks/community/model_garden/model_garden_rag.ipynb
+119 -79
View File
@@ -4,6 +4,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "Rdr9qXnG1HaN"
},
"outputs": [],
@@ -31,34 +32,34 @@
"source": [
"## Model Garden RAG API\n",
"\n",
"\u003ctable align=\"left\"\u003e\n",
" \u003ctd style=\"text-align: center\"\u003e\n",
" \u003ca href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_rag.ipynb\"\u003e\n",
" \u003cimg src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Google Colaboratory logo\"\u003e\u003cbr\u003e Open in Colab\n",
" \u003c/a\u003e\n",
" \u003c/td\u003e\n",
" \u003ctd style=\"text-align: center\"\u003e\n",
" \u003ca href=\"https://console.cloud.google.com/vertex-ai/colab/import/https:%2F%2Fraw.githubusercontent.com%2FGoogleCloudPlatform%2Fvertex-ai-samples%2Fmain%2Fnotebooks%2Fcommunity%2Fmodel_garden%2Fmodel_garden_rag.ipynb\"\"\u003e\n",
" \u003cimg width=\"32px\" src=\"https://cloud.google.com/ml-engine/images/colab-enterprise-logo-32px.png\" alt=\"Google Cloud Colab Enterprise logo\"\u003e\u003cbr\u003e Open in Colab Enterprise\n",
" \u003c/a\u003e\n",
" \u003c/td\u003e \n",
" \u003ctd style=\"text-align: center\"\u003e\n",
" \u003ca href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_rag.ipynb\"\u003e\n",
" \u003cimg src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\"\u003e\u003cbr\u003e Open in Workbench\n",
" \u003c/a\u003e\n",
" \u003c/td\u003e\n",
" \u003ctd style=\"text-align: center\"\u003e\n",
" \u003ca href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_rag.ipynb\"\u003e\n",
" \u003cimg src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\"\u003e\u003cbr\u003e View on GitHub\n",
" \u003c/a\u003e\n",
" \u003c/td\u003e"
"<table align=\"left\">\n",
" <td style=\"text-align: center\">\n",
" <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_rag.ipynb\">\n",
" <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Google Colaboratory logo\"><br> Open in Colab\n",
" </a>\n",
" </td>\n",
" <td style=\"text-align: center\">\n",
" <a href=\"https://console.cloud.google.com/vertex-ai/colab/import/https:%2F%2Fraw.githubusercontent.com%2FGoogleCloudPlatform%2Fvertex-ai-samples%2Fmain%2Fnotebooks%2Fcommunity%2Fmodel_garden%2Fmodel_garden_rag.ipynb\"\">\n",
" <img width=\"32px\" src=\"https://cloud.google.com/ml-engine/images/colab-enterprise-logo-32px.png\" alt=\"Google Cloud Colab Enterprise logo\"><br> Open in Colab Enterprise\n",
" </a>\n",
" </td> \n",
" <td style=\"text-align: center\">\n",
" <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_rag.ipynb\">\n",
" <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\"><br> Open in Workbench\n",
" </a>\n",
" </td>\n",
" <td style=\"text-align: center\">\n",
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_rag.ipynb\">\n",
" <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\"><br> View on GitHub\n",
" </a>\n",
" </td>"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "a1xYf51YT_56"
},
"cell_type": "markdown",
"source": [
"## 0. Set up the Environment and Test Project"
]
@@ -67,11 +68,12 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "u9mTxNC41S_1"
},
"outputs": [],
"source": [
"!pip3 install --force-reinstall google-cloud-aiplatform \"numpy\u003c2.0.0\" --user\n",
"!pip3 install --force-reinstall google-cloud-aiplatform \"numpy<2.0.0\" --user\n",
"!pip install --upgrade --quiet openai"
]
},
@@ -79,11 +81,13 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "ILNZ8_hw1WaC"
},
"outputs": [],
"source": [
"from google.colab import auth\n",
"\n",
"auth.authenticate_user()\n",
"\n",
"# Install gcloud\n",
@@ -103,6 +107,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "yj8SdgZM1cOP"
},
"outputs": [],
@@ -130,6 +135,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "ldoxVA24qnAF"
},
"outputs": [],
@@ -143,18 +149,20 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "tB-fhAybq0T2"
},
"outputs": [],
"source": [
"# Set Project\n",
"PROJECT_ID = \"\" # @param {type:\"string\", \"placeholder\": \"your-project-id\"}"
"PROJECT_ID = \"\" # @param {type:\"string\", \"placeholder\": \"your-project-id\"}"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "O2Rr1Ymlq3Uq"
},
"outputs": [],
@@ -175,6 +183,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "5E1tVMx3rAXF"
},
"outputs": [],
@@ -185,7 +194,7 @@
")\n",
"\n",
"# Name your corpus\n",
"DISPLAY_NAME = \"\" # @param {type:\"string\", \"placeholder\": \"your-corpus-name\"}\n",
"DISPLAY_NAME = \"\" # @param {type:\"string\", \"placeholder\": \"your-corpus-name\"}\n",
"\n",
"rag_corpus = rag.create_corpus(\n",
" display_name=DISPLAY_NAME, embedding_model_config=embedding_model_config\n",
@@ -254,6 +263,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "16i1ZInQrFnL"
},
"outputs": [],
@@ -275,6 +285,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "4G5uyvbdraMY"
},
"outputs": [],
@@ -288,6 +299,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "Z2vnvVO9rtDF"
},
"outputs": [],
@@ -314,11 +326,12 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "y64Hdd_9r5H9"
},
"outputs": [],
"source": [
"GS_BUCKET = \"\" # @param {type:\"string\", \"placeholder\": \"your-gs-bucket\"}\n",
"GS_BUCKET = \"\" # @param {type:\"string\", \"placeholder\": \"your-gs-bucket\"}\n",
"\n",
"response = await rag.import_files_async( # noqa: F704\n",
" corpus_name=rag_corpus.name,\n",
@@ -332,6 +345,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "YiTAFiEasHLX"
},
"outputs": [],
@@ -356,11 +370,12 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "u16-LvjT2Thi"
},
"outputs": [],
"source": [
"FILE_ID = \"\" # @param {type:\"string\", \"placeholder\": \"your-file-id\"}\n",
"FILE_ID = \"\" # @param {type:\"string\", \"placeholder\": \"your-file-id\"}\n",
"FILE_PATH = f\"https://drive.google.com/file/d/{FILE_ID}\""
]
},
@@ -368,6 +383,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "iY4_6tshsPSA"
},
"outputs": [],
@@ -384,6 +400,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "Dl8gPm9l4DQ3"
},
"outputs": [],
@@ -405,18 +422,22 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "b280caeab721"
},
"outputs": [],
"source": [
"CHANNEL_ID = \"\" # @param {type:\"string\", \"placeholder\": \"your-slack-channel-id\"}\n",
"API_KEY_SECRET_VERSION = \"\" # @param {type:\"string\", \"placeholder\": \"your-secret-manager-resource-name\"}"
"CHANNEL_ID = \"\" # @param {type:\"string\", \"placeholder\": \"your-slack-channel-id\"}\n",
"# fmt: off\n",
"API_KEY_SECRET_VERSION = \"\" # @param {type:\"string\", \"placeholder\": \"your-secret-manager-resource-name\"}\n",
"# fmt: on"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "69d731dc8bd6"
},
"outputs": [],
@@ -430,6 +451,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "c54695d94783"
},
"outputs": [],
@@ -446,6 +468,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "19bd9fe1537b"
},
"outputs": [],
@@ -467,21 +490,25 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "97716299c38f"
},
"outputs": [],
"source": [
"EMAIL = \"\" # @param {type:\"string\", \"placeholder\": \"your-email\"}\n",
"SERVER_URI = \"\" # @param {type:\"string\", \"placeholder\": \"your-server.atlassian.net\"}\n",
"PROJECT = \"\" # @param {type:\"string\", \"placeholder\": \"your-project-name\"}\n",
"CUSTOM_QUERY = \"\" # @param {type:\"string\", \"placeholder\": \"your-custom-jql-query\"}\n",
"API_KEY_SECRET_VERSION = \"\" # @param {type:\"string\", \"placeholder\": \"your-secret-manager-resource-name\"}"
"EMAIL = \"\" # @param {type:\"string\", \"placeholder\": \"your-email\"}\n",
"SERVER_URI = \"\" # @param {type:\"string\", \"placeholder\": \"your-server.atlassian.net\"}\n",
"PROJECT = \"\" # @param {type:\"string\", \"placeholder\": \"your-project-name\"}\n",
"CUSTOM_QUERY = \"\" # @param {type:\"string\", \"placeholder\": \"your-custom-jql-query\"}\n",
"# fmt: off\n",
"API_KEY_SECRET_VERSION = \"\" # @param {type:\"string\", \"placeholder\": \"your-secret-manager-resource-name\"}\n",
"# fmt: on"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "768705a29d61"
},
"outputs": [],
@@ -503,6 +530,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "05b54b5973ae"
},
"outputs": [],
@@ -519,6 +547,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "dbb764934c18"
},
"outputs": [],
@@ -528,21 +557,24 @@
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "f2iw2eOjT_56"
},
"cell_type": "markdown",
"source": [
"## Using GenerateContent API with Google-operated Llama3 model endpoint\n",
"\n",
"When retrieval query similarity distance \u003c vector_distance_threshold, generate content will cite the retrieved context (from RagStore)."
"When retrieval query similarity distance < vector_distance_threshold, generate content will cite the retrieved context (from RagStore)."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "Ad6Uev02T_56"
},
"cell_type": "code",
"outputs": [],
"source": [
"rag_resource = rag.RagResource(\n",
" rag_corpus=rag_corpus.name,\n",
@@ -558,46 +590,47 @@
" ),\n",
" )\n",
")"
],
"outputs": [],
"execution_count": null
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "urlpq0YiT_56"
},
"cell_type": "code",
"outputs": [],
"source": [
"ENDPOINT = f\"projects/{PROJECT_ID}/locations/us-central1/publishers/meta/models/llama3-405b-instruct-maas\"\n",
"ENDPOINT = f\"projects/{PROJECT_ID}/locations/us-central1/publishers/meta/models/llama-3.1-405b-instruct-maas\"\n",
"\n",
"rag_model = GenerativeModel(ENDPOINT, tools=[rag_retrieval_tool])"
],
"outputs": [],
"execution_count": null
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "N5B2BSPdT_56"
},
"cell_type": "code",
"outputs": [],
"source": [
"GENERATE_CONTENT_PROMPT = 'What is RAG and why it is helpful?' #@param {type:\"string\"}\n",
"GENERATE_CONTENT_PROMPT = \"What is RAG and why it is helpful?\" # @param {type:\"string\"}\n",
"\n",
"response = rag_model.generate_content(GENERATE_CONTENT_PROMPT)"
],
"outputs": [],
"execution_count": null
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "sTNBJlPET_56"
},
"cell_type": "code",
"outputs": [],
"source": [
"response"
],
"outputs": [],
"execution_count": null
]
},
{
"cell_type": "markdown",
@@ -607,13 +640,14 @@
"source": [
"## Using GenerateContent API with self-deployed Llama3 model endpoint\n",
"\n",
"When retrieval query similarity distance \u003c vector_distance_threshold, generate content will cite the retrieved context (from RagStore).\n"
"When retrieval query similarity distance < vector_distance_threshold, generate content will cite the retrieved context (from RagStore).\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "LvTPfijxtAQO"
},
"outputs": [],
@@ -637,11 +671,12 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "T3IffRu42nRp"
},
"outputs": [],
"source": [
"ENDPOINT_ID = \"\" # @param {type:\"string\", \"placeholder\": \"your-endpoint-id\"}\n",
"ENDPOINT_ID = \"\" # @param {type:\"string\", \"placeholder\": \"your-endpoint-id\"}\n",
"ENDPOINT = f\"projects/{PROJECT_ID}/locations/us-central1/endpoints/{ENDPOINT_ID}\"\n",
"\n",
"rag_model = GenerativeModel(ENDPOINT, tools=[rag_retrieval_tool])"
@@ -651,6 +686,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "SFEEv2u0tVNz"
},
"outputs": [],
@@ -664,6 +700,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "0I2EniAZtiUt"
},
"outputs": [],
@@ -672,10 +709,10 @@
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "2dmb7XHET_56"
},
"cell_type": "markdown",
"source": [
"## Using ChatCompletions API with Google-operated Llama3 model endpoint\n",
"\n",
@@ -683,10 +720,13 @@
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "3xSVLXL3T_56"
},
"cell_type": "code",
"outputs": [],
"source": [
"import openai\n",
"from google.auth import default, transport\n",
@@ -696,51 +736,49 @@
"credentials.refresh(auth_request)\n",
"\n",
"client = openai.OpenAI(\n",
" base_url=f\"https://us-central1-aiplatform.googleapis.com/v1beta1/projects/{PROJECT_ID}/locations/us-central1/endpoints/openapi/chat/completions?\",\n",
" base_url=f\"https://us-central1-aiplatform.googleapis.com/v1/projects/{PROJECT_ID}/locations/us-central1/endpoints/openapi/chat/completions?\",\n",
" api_key=credentials.token,\n",
")"
],
"outputs": [],
"execution_count": null
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "7VfgTTF7T_56"
},
"cell_type": "code",
"outputs": [],
"source": [
"CHAT_COMPLETIONS_PROMPT = 'What is RAG and why it is helpful?' #@param {type:\"string\"}\n",
"CHAT_COMPLETIONS_PROMPT = \"What is RAG and why it is helpful?\" # @param {type:\"string\"}\n",
"\n",
"response = client.chat.completions.create(\n",
" model=\"meta/llama3-405b-instruct-maas\",\n",
" model=\"meta/llama-3.1-405b-instruct-maas\",\n",
" messages=[{\"role\": \"user\", \"content\": CHAT_COMPLETIONS_PROMPT}],\n",
" extra_body={\n",
" \"extra_body\": {\n",
" \"google\": {\n",
" \"vertex_rag_store\": {\n",
" \"rag_resources\": {\n",
" \"rag_corpus\": rag_corpus.name\n",
" },\n",
" \"similarity_top_k\": 10\n",
" \"rag_resources\": {\"rag_corpus\": rag_corpus.name},\n",
" \"similarity_top_k\": 10,\n",
" }\n",
" }\n",
" }\n",
" },\n",
")"
],
"outputs": [],
"execution_count": null
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "76TW5K5kT_56"
},
"cell_type": "code",
"outputs": [],
"source": [
"response"
],
"outputs": [],
"execution_count": null
]
},
{
"cell_type": "markdown",
@@ -757,6 +795,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "dK7YmoIGtyki"
},
"outputs": [],
@@ -784,6 +823,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "--GwKlAO29bZ"
},
"outputs": [],
@@ -806,6 +846,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "tGtR_G2Rg1HC"
},
"outputs": [],
@@ -819,7 +860,7 @@
" rag.delete_corpus(name=rag_corpus.name)\n",
"\n",
"if delete_bucket:\n",
" ! gsutil rm -r gs://{BUCKET_NAME}"
" ! gsutil -m rm -r $GS_BUCKET"
]
},
{
@@ -830,14 +871,13 @@
"source": [
"## API reference\n",
"\n",
"For more details on RAG corpus/file management and detailed support please visit https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/rag-api\n"
"For more details on RAG corpus/file management and detailed support, visit https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/rag-api\n"
]
}
],
"metadata": {
"colab": {
"name": "model_garden_rag.ipynb",
"provenance": [],
"toc_visible": true
},
"kernelspec": {
notebooks/community/model_garden/model_garden_reasoning_engine_llama3_1.ipynb
+25 -14
View File
@@ -223,8 +223,8 @@
"outputs": [],
"source": [
"PROJECT_ID = \"\" # @param {type:\"string\", placeholder: \"[your-project-id]\"}\n",
"LOCATION = \"\" # @param {type:\"string\", placeholder: \"us-central1\"}\n",
"BUCKET_NAME = \"\" # @param {type:\"string\", placeholder: \"[your-bucket-name]\"}\n",
"LOCATION = \"\" # @param {type:\"string\", placeholder: \"us-central1\"}\n",
"BUCKET_NAME = \"\" # @param {type:\"string\", placeholder: \"[your-bucket-name]\"}\n",
"STAGING_BUCKET = f\"gs://{BUCKET_NAME}\""
]
},
@@ -300,7 +300,7 @@
" *,\n",
" model_name: str,\n",
" model_kwargs=None,\n",
" project: str, # Specified via vertexai.init\n",
" project: str, # Specified via vertexai.init\n",
" location: str, # Specified via vertexai.init\n",
" **kwargs,\n",
"):\n",
@@ -309,7 +309,9 @@
"\n",
" # Note: the credential lives for 1 hour by default.\n",
" # After expiration, it must be refreshed.\n",
" creds, _ = google.auth.default(scopes=[\"https://www.googleapis.com/auth/cloud-platform\"])\n",
" creds, _ = google.auth.default(\n",
" scopes=[\"https://www.googleapis.com/auth/cloud-platform\"]\n",
" )\n",
" auth_req = google.auth.transport.requests.Request()\n",
" creds.refresh(auth_req)\n",
"\n",
@@ -317,7 +319,9 @@
" model_kwargs = {}\n",
"\n",
" endpoint = f\"https://{location}-aiplatform.googleapis.com\"\n",
" base_url = f'{endpoint}/v1beta1/projects/{project}/locations/{location}/endpoints/openapi'\n",
" base_url = (\n",
" f\"{endpoint}/v1/projects/{project}/locations/{location}/endpoints/openapi\"\n",
" )\n",
"\n",
" return ChatOpenAI(\n",
" model=model_name,\n",
@@ -351,7 +355,7 @@
},
"outputs": [],
"source": [
"MODEL_ID = \"meta/llama3-405b-instruct-maas\" # @param {type:\"string\"} [\"meta/llama3-8b-instruct-maas\", \"meta/llama3-70b-instruct-maas\", \"meta/llama3-405b-instruct-maas\"]"
"MODEL_ID = \"meta/llama-3.1-405b-instruct-maas\" # @param {type:\"string\"} [\"meta/llama-3.1-8b-instruct-maas\", \"meta/llama-3.1-8b-instruct-maas\", \"meta/llama-3.1-405b-instruct-maas\"]"
]
},
{
@@ -408,13 +412,13 @@
"outputs": [],
"source": [
"agent = reasoning_engines.LangchainAgent(\n",
" model=MODEL_ID, # Required.\n",
" model_builder=model_builder, # Required.\n",
" model=MODEL_ID, # Required.\n",
" model_builder=model_builder, # Required.\n",
" model_kwargs={\n",
" \"temperature\": temperature, # Optional.\n",
" \"max_tokens\": max_tokens, # Optional.\n",
" \"top_p\": top_p, # Optional.\n",
" \"extra_body\": { # Optional.\n",
" \"max_tokens\": max_tokens, # Optional.\n",
" \"top_p\": top_p, # Optional.\n",
" \"extra_body\": { # Optional.\n",
" \"google\": {\n",
" \"model_safety_settings\": {\n",
" \"enabled\": apply_llama_guard,\n",
@@ -614,10 +618,13 @@
" from langchain_core.prompts import PromptTemplate\n",
"\n",
" template = \"\"\"Translate the following {text} to {target_language}:\"\"\"\n",
" prompt = PromptTemplate(input_variables=[\"text\", \"target_language\"], template=template)\n",
" prompt = PromptTemplate(\n",
" input_variables=[\"text\", \"target_language\"], template=template\n",
" )\n",
"\n",
" return prompt | model | StrOutputParser()\n",
"\n",
"\n",
"agent = reasoning_engines.LangchainAgent(\n",
" model=MODEL_ID,\n",
" model_builder=model_builder,\n",
@@ -672,7 +679,9 @@
}
],
"source": [
"response = agent.query(input={\"text\": text_to_translate, \"target_language\": target_language})\n",
"response = agent.query(\n",
" input={\"text\": text_to_translate, \"target_language\": target_language}\n",
")\n",
"print(response)"
]
},
@@ -734,7 +743,9 @@
}
],
"source": [
"response = remote_agent.query(input={\"text\": text_to_translate, \"target_language\": target_language})\n",
"response = remote_agent.query(\n",
" input={\"text\": text_to_translate, \"target_language\": target_language}\n",
")\n",
"print(response)"
]
},
notebooks/community/model_garden/synthetic_data_generation_using_llama3_1.ipynb
+32 -7
View File
@@ -4,6 +4,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "NoEDALsivNDl"
},
"outputs": [],
@@ -53,7 +54,7 @@
"source": [
"## Overview\n",
"\n",
"This notebook demonstrates generating synthetic data using the [Llama 3.1 405B service API](https://console.cloud.google.com/vertex-ai/publishers/meta/model-garden/llama3-405b-instruct-maas).\n",
"This notebook demonstrates generating synthetic data using the [Llama 3.1 405B service API](https://console.cloud.google.com/vertex-ai/publishers/meta/model-garden/llama-3.1-405b-instruct-maas).\n",
"\n",
"\n",
"### Objective\n",
@@ -83,6 +84,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "AjA_UYD25_1w"
},
"outputs": [],
@@ -92,6 +94,7 @@
"!pip install --upgrade --user -q openai snowfakery==3.6.2 wikipedia-api==0.6.0\n",
"\n",
"import os\n",
"import uuid\n",
"from datetime import datetime\n",
"\n",
"from google.cloud import aiplatform\n",
@@ -107,15 +110,15 @@
"# Cloud Storage bucket for storing the experiment artifacts.\n",
"# A unique GCS bucket will be created for the purpose of this notebook. If you\n",
"# prefer using your own GCS bucket, change the value yourself below.\n",
"now = datetime.now().strftime(\"%Y%m%d%H%M%S\")\n",
"BUCKET_URI = \"gs://\" # @param {type:\"string\"}\n",
"now = datetime.datetime.now().strftime(\"%Y%m%d%H%M%S\")\n",
"BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n",
"\n",
"if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n",
" BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}\"\n",
" BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n",
" BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n",
" ! gsutil mb -l {REGION} {BUCKET_URI}\n",
"else:\n",
" assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n",
" BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n",
" shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n",
" bucket_region = shell_output[0].strip().lower()\n",
" if bucket_region != REGION:\n",
@@ -153,6 +156,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "Yl8gDtmA75hD"
},
"outputs": [],
@@ -172,12 +176,12 @@
"from snowfakery import generate_data\n",
"from snowfakery.plugins import SnowfakeryPlugin\n",
"\n",
"MODEL_ID = \"meta/llama3-405b-instruct-maas\"\n",
"MODEL_ID = \"meta/llama-3.1-405b-instruct-maas\"\n",
"ENDPOINT = \"aiplatform.googleapis.com\"\n",
"\n",
"# Pass the Vertex endpoint and authentication to the OpenAI SDK\n",
"client = openai.OpenAI(\n",
" base_url=f\"https://us-central1-{ENDPOINT}/v1beta1/projects/{PROJECT_ID}/locations/{REGION}/endpoints/openapi\",\n",
" base_url=f\"https://us-central1-{ENDPOINT}/v1/projects/{PROJECT_ID}/locations/{REGION}/endpoints/openapi\",\n",
" api_key=creds.token,\n",
")\n",
"\n",
@@ -289,6 +293,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "WSryene29Dan"
},
"outputs": [],
@@ -306,6 +311,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "cJaPNlBi9bJr"
},
"outputs": [],
@@ -327,6 +333,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "BbOYKhD5-Bht"
},
"outputs": [],
@@ -348,6 +355,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "sPe-UoD_-Ilo"
},
"outputs": [],
@@ -432,6 +440,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "JTEQU-u--ei7"
},
"outputs": [],
@@ -447,6 +456,22 @@
"\n",
"# @markdown Results The synthetic data has been generated and stored as CSV files in the `outputs` folder."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "ffe32dda3585"
},
"outputs": [],
"source": [
"# @title Delete Bucket\n",
"\n",
"delete_bucket = False # @param {type:\"boolean\"}\n",
"if delete_bucket:\n",
" ! gsutil -m rm -r $BUCKET_NAME"
]
}
],
"metadata": {