mirror of
https://github.com/GoogleCloudPlatform/vertex-ai-samples.git
synced 2026-09-26 22:51:56 +00:00
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Andrew Ferlitsch
|
0c05fdd926 |
+54
-408
@@ -24,13 +24,12 @@
|
||||
]
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "JAPoU8Sm5E6e"
|
||||
},
|
||||
"source": [
|
||||
"# Using Vertex AI Matching Engine and Vertex AI Embeddings for Text for StackOverflow Questions \n",
|
||||
"# Using Vertex AI Vector Search and Vertex AI Embeddings for Text for StackOverflow Questions \n",
|
||||
"\n",
|
||||
"<table align=\"left\">\n",
|
||||
" <td>\n",
|
||||
@@ -54,7 +53,6 @@
|
||||
]
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "b0a74aaf1481"
|
||||
@@ -62,13 +60,12 @@
|
||||
"source": [
|
||||
"## Overview\n",
|
||||
"\n",
|
||||
"This example demonstrates how to encode text embeddings using the Vertex AI Embeddings for Text service and the StackOverflow dataset. These are uploaded to the Vertex AI Matching Engine service, which is a high-scale, low-latency solution, for finding similar vectors from a large corpus. Matching Engine is a fully managed offering, further reducing operational overhead. It is built upon [Approximate Nearest Neighbor (ANN) technology](https://ai.googleblog.com/2020/07/announcing-scann-efficient-vector.html) developed by Google Research.\n",
|
||||
"This example demonstrates how to encode text embeddings using the Vertex AI Embeddings for Text service and the StackOverflow dataset. These are uploaded to the Vertex AI Vector Search Engine service, which is a high-scale, low-latency solution, for finding similar vectors from a large corpus. Vector Search is a fully managed offering, further reducing operational overhead. It is built upon [Approximate Nearest Neighbor (ANN) technology](https://ai.googleblog.com/2020/07/announcing-scann-efficient-vector.html) developed by Google Research.\n",
|
||||
"\n",
|
||||
"Learn more about [Vertex AI Matching Engine](https://cloud.google.com/vertex-ai/docs/matching-engine/overview) and [Vertex AI Embeddings for Text](https://cloud.google.com/vertex-ai/docs/generative-ai/embeddings/get-text-embeddings)."
|
||||
"Learn more about [Vertex AI Vector Search](https://cloud.google.com/vertex-ai/docs/matching-engine/overview) and [Vertex AI Embeddings for Text](https://cloud.google.com/vertex-ai/docs/generative-ai/embeddings/get-text-embeddings)."
|
||||
]
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "34a4b245e795"
|
||||
@@ -80,7 +77,7 @@
|
||||
"\n",
|
||||
"This tutorial uses the following Google Cloud ML services:\n",
|
||||
"\n",
|
||||
"- `Vertex AI Matching Engine`\n",
|
||||
"- `Vertex AI Vector Search`\n",
|
||||
"- `Vertex AI Embeddings for Text`\n",
|
||||
"\n",
|
||||
"The steps performed include:\n",
|
||||
@@ -90,8 +87,7 @@
|
||||
"* Upload embeddings to the index\n",
|
||||
"* Create an index endpoint\n",
|
||||
"* Deploy the index to the index endpoint\n",
|
||||
"* Perform an online query\n",
|
||||
"* Add metadata to a Redis store"
|
||||
"* Perform an online query"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -108,7 +104,6 @@
|
||||
]
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "f0f1bea346db"
|
||||
@@ -116,7 +111,7 @@
|
||||
"source": [
|
||||
"## Installation\n",
|
||||
"\n",
|
||||
"Install the latest version of Cloud Storage, BigQuery, and the Vertex AI SDK for Python. Also install the latest version of Redis for low-latency data retrieval."
|
||||
"Install the latest version of Cloud Storage, BigQuery, and the Vertex AI SDK for Python. "
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -130,12 +125,10 @@
|
||||
"# Install the packages\n",
|
||||
"! pip3 install --upgrade google-cloud-aiplatform \\\n",
|
||||
" google-cloud-storage \\\n",
|
||||
" 'google-cloud-bigquery[pandas]' \\\n",
|
||||
" redis"
|
||||
" 'google-cloud-bigquery[pandas]' "
|
||||
]
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "5b08ba354c6e"
|
||||
@@ -160,7 +153,6 @@
|
||||
]
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "dd28c9e4f067"
|
||||
@@ -188,20 +180,19 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "80c0215f05a0"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"PROJECT_ID = \"[YOUR-PROJECT-ID]\" # @param {type:\"string\"}\n",
|
||||
"PROJECT_ID = \"[your-project-id]\" # @param {type:\"string\"}\n",
|
||||
"\n",
|
||||
"# Set the project id\n",
|
||||
"! gcloud config set project {PROJECT_ID}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "4f4512bf63b3"
|
||||
@@ -214,7 +205,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "474be5183c27"
|
||||
},
|
||||
@@ -255,7 +246,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "fbc9cd30cc4b"
|
||||
},
|
||||
@@ -275,7 +266,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "a336a05c6149"
|
||||
},
|
||||
@@ -305,7 +296,6 @@
|
||||
]
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "hhq5zEbGg0XX"
|
||||
@@ -316,7 +306,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "EzrelQZ22IZj"
|
||||
},
|
||||
@@ -342,7 +332,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 27,
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "MzGDU7TWdts_"
|
||||
},
|
||||
@@ -362,17 +352,16 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "NIq7R4HZCfIc"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI"
|
||||
"! gsutil mb -l {REGION} -p {PROJECT_ID} {BUCKET_URI}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "lR6Wwv-hCCN-"
|
||||
@@ -389,7 +378,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 11,
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "ed1b3f87c475"
|
||||
},
|
||||
@@ -423,106 +412,11 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 176,
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "b43937b6065d"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/html": [
|
||||
"<div>\n",
|
||||
"<style scoped>\n",
|
||||
" .dataframe tbody tr th:only-of-type {\n",
|
||||
" vertical-align: middle;\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" .dataframe tbody tr th {\n",
|
||||
" vertical-align: top;\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" .dataframe thead th {\n",
|
||||
" text-align: right;\n",
|
||||
" }\n",
|
||||
"</style>\n",
|
||||
"<table border=\"1\" class=\"dataframe\">\n",
|
||||
" <thead>\n",
|
||||
" <tr style=\"text-align: right;\">\n",
|
||||
" <th></th>\n",
|
||||
" <th>id</th>\n",
|
||||
" <th>title</th>\n",
|
||||
" <th>body</th>\n",
|
||||
" <th>title_with_body</th>\n",
|
||||
" </tr>\n",
|
||||
" </thead>\n",
|
||||
" <tbody>\n",
|
||||
" <tr>\n",
|
||||
" <th>0</th>\n",
|
||||
" <td>12615525</td>\n",
|
||||
" <td>What are the different use cases of joblib ver...</td>\n",
|
||||
" <td><p>Background: I'm just getting started with s...</td>\n",
|
||||
" <td>What are the different use cases of joblib ver...</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1</th>\n",
|
||||
" <td>12788972</td>\n",
|
||||
" <td>Set database timeout in Entity Framework</td>\n",
|
||||
" <td><p>My command keeps timing out, so I need to c...</td>\n",
|
||||
" <td>Set database timeout in Entity Framework\\n<p>M...</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>2</th>\n",
|
||||
" <td>18405374</td>\n",
|
||||
" <td>Test a factory of a 3rd party class</td>\n",
|
||||
" <td><p>My application uses a third party jar (no a...</td>\n",
|
||||
" <td>Test a factory of a 3rd party class\\n<p>My app...</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>3</th>\n",
|
||||
" <td>18350790</td>\n",
|
||||
" <td>Sublime Text 3 (and 2): newly installed dictio...</td>\n",
|
||||
" <td><p>I'm a well-experienced mac user but no prog...</td>\n",
|
||||
" <td>Sublime Text 3 (and 2): newly installed dictio...</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>4</th>\n",
|
||||
" <td>18695061</td>\n",
|
||||
" <td>Closing dropdown in Spinner in Android</td>\n",
|
||||
" <td><p>I need to animate an icon of an arrow when ...</td>\n",
|
||||
" <td>Closing dropdown in Spinner in Android\\n<p>I n...</td>\n",
|
||||
" </tr>\n",
|
||||
" </tbody>\n",
|
||||
"</table>\n",
|
||||
"</div>"
|
||||
],
|
||||
"text/plain": [
|
||||
" id title \\\n",
|
||||
"0 12615525 What are the different use cases of joblib ver... \n",
|
||||
"1 12788972 Set database timeout in Entity Framework \n",
|
||||
"2 18405374 Test a factory of a 3rd party class \n",
|
||||
"3 18350790 Sublime Text 3 (and 2): newly installed dictio... \n",
|
||||
"4 18695061 Closing dropdown in Spinner in Android \n",
|
||||
"\n",
|
||||
" body \\\n",
|
||||
"0 <p>Background: I'm just getting started with s... \n",
|
||||
"1 <p>My command keeps timing out, so I need to c... \n",
|
||||
"2 <p>My application uses a third party jar (no a... \n",
|
||||
"3 <p>I'm a well-experienced mac user but no prog... \n",
|
||||
"4 <p>I need to animate an icon of an arrow when ... \n",
|
||||
"\n",
|
||||
" title_with_body \n",
|
||||
"0 What are the different use cases of joblib ver... \n",
|
||||
"1 Set database timeout in Entity Framework\\n<p>M... \n",
|
||||
"2 Test a factory of a 3rd party class\\n<p>My app... \n",
|
||||
"3 Sublime Text 3 (and 2): newly installed dictio... \n",
|
||||
"4 Closing dropdown in Spinner in Android\\n<p>I n... "
|
||||
]
|
||||
},
|
||||
"execution_count": 176,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Get a dataframe of 1000 rows for demonstration purposes\n",
|
||||
"df = next(query_bigquery_chunks(max_rows=1000, rows_per_chunk=1000))\n",
|
||||
@@ -532,7 +426,6 @@
|
||||
]
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "1124422cc200"
|
||||
@@ -563,7 +456,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "5c4520ae99f8"
|
||||
},
|
||||
@@ -587,7 +480,6 @@
|
||||
]
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "eda80c5936ea"
|
||||
@@ -601,7 +493,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 217,
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "a0370bd840d2"
|
||||
},
|
||||
@@ -658,7 +550,6 @@
|
||||
]
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "ba45d58bf96e"
|
||||
@@ -671,7 +562,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "9b01baa906b5"
|
||||
},
|
||||
@@ -698,22 +589,11 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 221,
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "d296e181205d"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"768"
|
||||
]
|
||||
},
|
||||
"execution_count": 221,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"DIMENSIONS = len(question_embeddings[0])\n",
|
||||
"\n",
|
||||
@@ -721,7 +601,6 @@
|
||||
]
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "d503db448252"
|
||||
@@ -738,39 +617,11 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 222,
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "95e408daf219"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Query question = SignalR - connection.hubName is undefined\n",
|
||||
"\t0: SignalR - connection.hubName is undefined: 0.9999994517616809\n",
|
||||
"\t1: How to solve a login/database missing error regarding Sitecore Training Website?: 0.6360554028470855\n",
|
||||
"\t2: NameError: name 'helloworld' is not defined: 0.6137728867637725\n",
|
||||
"\t3: Cannot Resolve @style/Theme.Sherlock: 0.6077803902687016\n",
|
||||
"\t4: Load an assembly (dll) from a network drive in C#: 0.6053678317274827\n",
|
||||
"\t5: OleDbException: No value given for one or more required parameters: 0.6047448126425075\n",
|
||||
"\t6: C# HTTPModule Could not load type CGI Request: 0.6003740454733572\n",
|
||||
"\t7: SecurityException - Dapper on shared hosting: 0.5930472323803362\n",
|
||||
"\t8: Wix 3.5 preprocessor extension - undefined preprocessor function: 0.5873751287246225\n",
|
||||
"\t9: need to parse refname in post-receive script: 0.5854348455753308\n",
|
||||
"\t10: @Url.Content not encoding text - ASP.NET MVC with Razor: 0.5845889360671046\n",
|
||||
"\t11: The symbol you provided is not a function: 0.58359112051759\n",
|
||||
"\t12: SharePoint Redirect site logo link to the root site collection home page: 0.5829007492528733\n",
|
||||
"\t13: PHP WebSocketServer can't connect to WebKit (Safari): 0.5823678399403036\n",
|
||||
"\t14: Trouble with PayPal Adaptive Payments in Node.js: 0.5792609168047103\n",
|
||||
"\t15: Push Notification not Receiving after bb10 restart in Android Runtime: 0.5770136072688865\n",
|
||||
"\t16: Best way to not run rufus-scheduler when starting a rails console: 0.5768953788628541\n",
|
||||
"\t17: Ruby gem error: no such file to load -- bundler: 0.5742615556787587\n",
|
||||
"\t18: Cannot load drivers for SQL Server on WSO2 Data Services Server: 0.5699260860941642\n",
|
||||
"\t19: jQuery.getJSON callback does not fire in IE7: 0.5693486248098263\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import random\n",
|
||||
"\n",
|
||||
@@ -789,7 +640,6 @@
|
||||
]
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "aQIQSyF9GtSv"
|
||||
@@ -804,19 +654,11 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 226,
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "7c1193aca5d1"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Embeddings directory: /var/tmp/tmpb4hc2lc4\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import tempfile\n",
|
||||
"from pathlib import Path\n",
|
||||
@@ -828,7 +670,6 @@
|
||||
]
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "279c0bbfc6bb"
|
||||
@@ -913,7 +754,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "3PgsA_vbI8Vg"
|
||||
},
|
||||
@@ -943,7 +784,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 231,
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "qiIg9b5zJLi1"
|
||||
},
|
||||
@@ -954,7 +795,6 @@
|
||||
]
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "svLYiDf0OD2G"
|
||||
@@ -965,7 +805,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 232,
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "Y4zooldkGoM4"
|
||||
},
|
||||
@@ -978,7 +818,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "dffb00b23f5a"
|
||||
},
|
||||
@@ -1000,7 +840,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "17jrQi501QyX"
|
||||
},
|
||||
@@ -1021,7 +861,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 235,
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "1ddb70647d98"
|
||||
},
|
||||
@@ -1031,7 +871,6 @@
|
||||
]
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "qV2xjAnDDObD"
|
||||
@@ -1042,7 +881,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "QuARXzJVGyQX"
|
||||
},
|
||||
@@ -1075,22 +914,11 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 240,
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "nLOYTGygIlMK"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"'stack_overflow_8M_d298'"
|
||||
]
|
||||
},
|
||||
"execution_count": 240,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"DEPLOYED_INDEX_ID = \"deployed_index_id_unique\"\n",
|
||||
"\n",
|
||||
@@ -1125,19 +953,11 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 244,
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "93f89a15f642"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Expected: 50000, Actual: 49992\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"number_of_vectors = sum(\n",
|
||||
" aiplatform.MatchingEngineIndex(\n",
|
||||
@@ -1150,7 +970,6 @@
|
||||
]
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "6LCGvBNvBd8D"
|
||||
@@ -1165,7 +984,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 249,
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "ae9996f185fe"
|
||||
},
|
||||
@@ -1176,31 +995,11 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 250,
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "A3KYVw5HB-4v"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"[[MatchNeighbor(id='68704846', distance=0.7809396386146545),\n",
|
||||
" MatchNeighbor(id='41366327', distance=0.7776130437850952),\n",
|
||||
" MatchNeighbor(id='70984360', distance=0.776228666305542),\n",
|
||||
" MatchNeighbor(id='66525883', distance=0.7567992806434631),\n",
|
||||
" MatchNeighbor(id='47248054', distance=0.7561374306678772),\n",
|
||||
" MatchNeighbor(id='52857901', distance=0.7410272359848022),\n",
|
||||
" MatchNeighbor(id='58525872', distance=0.7396792769432068),\n",
|
||||
" MatchNeighbor(id='61884137', distance=0.7386407852172852),\n",
|
||||
" MatchNeighbor(id='64786672', distance=0.7366656064987183),\n",
|
||||
" MatchNeighbor(id='58561680', distance=0.7357790470123291)]]"
|
||||
]
|
||||
},
|
||||
"execution_count": 250,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Test query\n",
|
||||
"NUM_NEIGHBOURS = 10\n",
|
||||
@@ -1215,7 +1014,6 @@
|
||||
]
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "8a2879d3d9ca"
|
||||
@@ -1226,154 +1024,16 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 252,
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "7c8682079e21"
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"https://stackoverflow.com/questions/68704846\n",
|
||||
"https://stackoverflow.com/questions/41366327\n",
|
||||
"https://stackoverflow.com/questions/70984360\n",
|
||||
"https://stackoverflow.com/questions/66525883\n",
|
||||
"https://stackoverflow.com/questions/47248054\n",
|
||||
"https://stackoverflow.com/questions/52857901\n",
|
||||
"https://stackoverflow.com/questions/58525872\n",
|
||||
"https://stackoverflow.com/questions/61884137\n",
|
||||
"https://stackoverflow.com/questions/64786672\n",
|
||||
"https://stackoverflow.com/questions/58561680\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"for match_index, neighbor in enumerate(response[0]):\n",
|
||||
" print(f\"https://stackoverflow.com/questions/{neighbor.id}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "05514825ba7d"
|
||||
},
|
||||
"source": [
|
||||
"## Storing and retrieving titles from a Redis data store\n",
|
||||
"When you productionize this code into a service, you need to convert the nearest ID's returned from Vertex AI Matching Engine into data ready to be used by downstream services.\n",
|
||||
"\n",
|
||||
"In this case, you need to convert the ID's to titles. You can use Google Cloud's Memorystore to deploy a managed Redis instance to save the id-title key-value pairs.\n",
|
||||
"\n",
|
||||
"To learn more, see [Create and manage Redis instances](https://cloud.google.com/memorystore/docs/redis/create-manage-instances?hl=en)."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "5d2b240f0d52"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"REDIS_INSTANCE_NAME = \"stackoverflow-questions-vertex\"\n",
|
||||
"\n",
|
||||
"# Create a Redis instance\n",
|
||||
"! gcloud redis instances create '{REDIS_INSTANCE_NAME}' --size=10 --region='{REGION}' --connect-mode=private-service-access"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "371ccc0d2eb2"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Get host and port info\n",
|
||||
"REDIS_HOST = ! gcloud redis instances list --filter=\"INSTANCE_NAME:'{REDIS_INSTANCE_NAME}'\" --region {REGION} --format='value(HOST)'\n",
|
||||
"REDIS_PORT = ! gcloud redis instances list --filter=\"INSTANCE_NAME:'{REDIS_INSTANCE_NAME}'\" --region {REGION} --format='value(PORT)'\n",
|
||||
"\n",
|
||||
"if isinstance(REDIS_HOST, list):\n",
|
||||
" REDIS_HOST = REDIS_HOST[0]\n",
|
||||
"\n",
|
||||
"if isinstance(REDIS_PORT, list):\n",
|
||||
" REDIS_PORT = REDIS_PORT[0]\n",
|
||||
"\n",
|
||||
"print(f\"REDIS_HOST = {REDIS_HOST}\")\n",
|
||||
"print(f\"REDIS_PORT = {REDIS_PORT}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "73796089386a"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Connect to the instance\n",
|
||||
"import redis\n",
|
||||
"\n",
|
||||
"redis_client = redis.StrictRedis(host=REDIS_HOST, port=REDIS_PORT)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "f000f5432d13"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%time\n",
|
||||
"# Convert the id -> (title, body) relationship into a dict and write to Redis\n",
|
||||
"for df in tqdm(\n",
|
||||
" query_bigquery_chunks(\n",
|
||||
" max_rows=BQ_NUM_ROWS, rows_per_chunk=BQ_CHUNK_SIZE, start_chunk=0\n",
|
||||
" ),\n",
|
||||
" total=BQ_NUM_CHUNKS,\n",
|
||||
" position=0,\n",
|
||||
" desc=\"Chunk of rows from BigQuery\",\n",
|
||||
"):\n",
|
||||
" ids = df.id.tolist()\n",
|
||||
" titles = df.title.tolist()\n",
|
||||
" bodies = df.body.tolist()\n",
|
||||
"\n",
|
||||
" # create a Redis pipeline\n",
|
||||
" pipe = redis_client.pipeline()\n",
|
||||
"\n",
|
||||
" # iterate over the data and add hset commands to the pipeline\n",
|
||||
" for (id, title, body) in tqdm(zip(ids, titles, bodies), total=len(ids), position=1):\n",
|
||||
" pipe.hset(\n",
|
||||
" str(id),\n",
|
||||
" mapping={\n",
|
||||
" \"title\": str(title),\n",
|
||||
" \"body\": str(body[:100]),\n",
|
||||
" },\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" # execute the pipeline\n",
|
||||
" _ = pipe.execute()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "b1f8b396aeb1"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Verify that Redis can retrieve the correct information\n",
|
||||
"df = next(query_bigquery_chunks(max_rows=10, rows_per_chunk=10))\n",
|
||||
"\n",
|
||||
"[\n",
|
||||
" f\"Actual = {title}, Retrieved = {redis_client.hgetall(str(id))}\"\n",
|
||||
" for id, title in zip(df.id, df.title)\n",
|
||||
"]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
@@ -1395,32 +1055,18 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"\n",
|
||||
"delete_bucket = False\n",
|
||||
"\n",
|
||||
"# Force undeployment of indexes and delete endpoint\n",
|
||||
"my_index_endpoint.delete(force=True)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "omj7N9iWv-Tq"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"my_index_endpoint.delete(force=True)\n",
|
||||
"\n",
|
||||
"# Delete indexes\n",
|
||||
"tree_ah_index.delete()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "d2fcf9468031"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Delete redis instance\n",
|
||||
"! gcloud redis instances delete '{REDIS_INSTANCE_NAME}' --region {REGION} --quiet"
|
||||
"tree_ah_index.delete()\n",
|
||||
"\n",
|
||||
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
|
||||
" ! gsutil rm -rf {BUCKET_URI}"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
Reference in New Issue
Block a user