Ran linter

fix: issue 1251

.cloud-build/execute_changed_notebooks_helper.py

@@ -156,7 +156,7 @@ def _create_tag(filepath: str) -> str:
     return tag
 
 
-rate_limit = RateLimit(max_count=50, per=60, greedy=True)
+rate_limit = RateLimit(max_count=25, per=60, greedy=True)
 
 
 def process_and_execute_notebook(

CONTRIBUTING.md

@@ -44,8 +44,10 @@ Finally, run this code block to check for errors. Each step will attempt to
 automatically fix any issues. If the fixes can't be performed automatically,
 then you will need to manually address them before submitting your PR.
 
+Note: For official, only submit one notebook per PR.
+
 ```shell
-docker run -v ${PWD}:/setup/app gcr.io/cloud-devrel-public-resources/notebook_linter:latest <your_notebooks>
+docker run -v ${PWD}:/setup/app gcr.io/cloud-devrel-public-resources/notebook_linter:latest your_notebook
 ```
 
 ## Code Reviews

notebooks/official/matching_engine/sdk_matching_engine_create_stack_overflow_embeddings.ipynb

@@ -150,6 +150,28 @@
         "                        tensorflow-hub"
       ]
     },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "54ac7ebac10b"
+      },
+      "source": [
+        "Install the latest version of Redis for low-latency data retrieval"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "e3dd53b3c06c"
+      },
+      "outputs": [],
+      "source": [
+        "# Install the redis package\n",
+        "! pip install --upgrade redis"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -1015,6 +1037,7 @@
       ]
     },
     {
+      "attachments": {},
       "cell_type": "markdown",
       "metadata": {
         "id": "6LCGvBNvBd8D"
@@ -1022,7 +1045,9 @@
       "source": [
         "## Create Online Queries\n",
         "\n",
-        "After you built your indexes, you may query against the deployed index to find nearest neighbors."
+        "After you built your indexes, you may query against the deployed index to find nearest neighbors.\n",
+        "\n",
+        "Note: For the DOT_PRODUCT_DISTANCE distance type, the \"distance\" property returned with each MatchNeighbor actually refers to the similarity."
       ]
     },
     {
@@ -1087,6 +1112,100 @@
         "        )"
       ]
     },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "05514825ba7d"
+      },
+      "source": [
+        "## Storing and retrieving titles from a Redis data store\n",
+        "When you productionize this code into a service, you will need to convert the nearest nearest id's returned from Vertex AI Matching Engine into data usable by downstream services.\n",
+        "\n",
+        "In this case, you'll need to convert the id's to titles.\n",
+        "\n",
+        "You can use Google Cloud's Memorystore to deploy a managed Redis instance to save the id-title key-value pairs.\n",
+        "\n",
+        "See more information on [Memorystore](https://cloud.google.com/memorystore/docs/redis/create-manage-instances?hl=en)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "5d2b240f0d52"
+      },
+      "outputs": [],
+      "source": [
+        "REDIS_INSTANCE_NAME = \"stackoverflow-questions-unique\"\n",
+        "\n",
+        "# Create a Redis instance\n",
+        "! gcloud redis instances create '{REDIS_INSTANCE_NAME}' --size=5 --region={REGION} --network={VPC_NETWORK_FULL} --connect-mode=private-service-access"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "371ccc0d2eb2"
+      },
+      "outputs": [],
+      "source": [
+        "# Get host and port info\n",
+        "REDIS_HOST = ! gcloud redis instances list --filter=\"INSTANCE_NAME:'{REDIS_INSTANCE_NAME}'\" --region {REGION}  --format='value(HOST)'\n",
+        "REDIS_PORT = ! gcloud redis instances list --filter=\"INSTANCE_NAME:'{REDIS_INSTANCE_NAME}'\" --region {REGION} --format='value(PORT)'\n",
+        "\n",
+        "if isinstance(REDIS_HOST, list):\n",
+        "    REDIS_HOST = REDIS_HOST[0]\n",
+        "\n",
+        "if isinstance(REDIS_PORT, list):\n",
+        "    REDIS_PORT = REDIS_PORT[0]\n",
+        "\n",
+        "print(f\"REDIS_HOST = {REDIS_HOST}\")\n",
+        "print(f\"REDIS_PORT = {REDIS_PORT}\")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "73796089386a"
+      },
+      "outputs": [],
+      "source": [
+        "# Connect to the instance\n",
+        "import redis\n",
+        "\n",
+        "redis_client = redis.StrictRedis(host=REDIS_HOST, port=REDIS_PORT)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "f000f5432d13"
+      },
+      "outputs": [],
+      "source": [
+        "# Convert the id -> title relationship into a dict and write to redis\n",
+        "redis_client.mset({str(id): str(title) for id, title in zip(df.id, df.title)})"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "b1f8b396aeb1"
+      },
+      "outputs": [],
+      "source": [
+        "# Verify that redis can retrieve the correct information\n",
+        "[\n",
+        "    f\"Actual = {title}, Retrieved = {redis_client.get(str(id))}\"\n",
+        "    for id, title in list(zip(df.id, df.title))[:10]\n",
+        "]"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -1123,6 +1242,18 @@
         "# Delete indexes\n",
         "tree_ah_index.delete()"
       ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "d2fcf9468031"
+      },
+      "outputs": [],
+      "source": [
+        "# Delete redis instance\n",
+        "! gcloud redis instances delete '{REDIS_INSTANCE_NAME}' --region {REGION} --quiet"
+      ]
     }
   ],
   "metadata": {

notebooks/official/ml_metadata/vertex-pipelines-ml-metadata.ipynb

@@ -719,7 +719,7 @@
       "outputs": [],
       "source": [
         "@component(\n",
-        "    packages_to_install=[\"sklearn\", \"pandas\", \"joblib\"],\n",
+        "    packages_to_install=[\"scikit-learn\", \"pandas\", \"joblib\"],\n",
         "    base_image=\"python:3.9\",\n",
         "    output_component_file=\"beans_model_component.yaml\",\n",
         ")\n",

notebooks/official/pipelines/metrics_viz_run_compare_kfp.ipynb

@@ -713,7 +713,7 @@
       "outputs": [],
       "source": [
         "@component(\n",
-        "    packages_to_install=[\"sklearn\"],\n",
+        "    packages_to_install=[\"scikit-learn\"],\n",
         "    base_image=\"python:3.9\",\n",
         "    output_component_file=\"wine_classification_component.yaml\",\n",
         ")\n",
@@ -758,7 +758,7 @@
       },
       "outputs": [],
       "source": [
-        "@component(packages_to_install=[\"sklearn\"], base_image=\"python:3.9\")\n",
+        "@component(packages_to_install=[\"scikit-learn\"], base_image=\"python:3.9\")\n",
         "def iris_sgdclassifier(\n",
         "    test_samples_fraction: float,\n",
         "    metricsc: Output[ClassificationMetrics],\n",
@@ -805,7 +805,7 @@
       "outputs": [],
       "source": [
         "@component(\n",
-        "    packages_to_install=[\"sklearn\"],\n",
+        "    packages_to_install=[\"scikit-learn\"],\n",
         "    base_image=\"python:3.9\",\n",
         ")\n",
         "def iris_logregression(\n",