fix: get eval by id

.cloud-build/execute_changed_notebooks_helper.py

@@ -156,7 +156,7 @@ def _create_tag(filepath: str) -> str:
     return tag
 
 
-rate_limit = RateLimit(max_count=25, per=60, greedy=True)
+rate_limit = RateLimit(max_count=50, per=60, greedy=True)
 
 
 def process_and_execute_notebook(

CONTRIBUTING.md

@@ -44,10 +44,8 @@ Finally, run this code block to check for errors. Each step will attempt to
 automatically fix any issues. If the fixes can't be performed automatically,
 then you will need to manually address them before submitting your PR.
 
-Note: For official, only submit one notebook per PR.
-
 ```shell
-docker run -v ${PWD}:/setup/app gcr.io/cloud-devrel-public-resources/notebook_linter:latest your_notebook
+docker run -v ${PWD}:/setup/app gcr.io/cloud-devrel-public-resources/notebook_linter:latest <your_notebooks>
 ```
 
 ## Code Reviews

notebooks/official/matching_engine/sdk_matching_engine_create_stack_overflow_embeddings.ipynb

@@ -150,28 +150,6 @@
         "                        tensorflow-hub"
       ]
     },
-    {
-      "attachments": {},
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "54ac7ebac10b"
-      },
-      "source": [
-        "Install the latest version of Redis for low-latency data retrieval"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "e3dd53b3c06c"
-      },
-      "outputs": [],
-      "source": [
-        "# Install the redis package\n",
-        "! pip install --upgrade redis"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -1037,7 +1015,6 @@
       ]
     },
     {
-      "attachments": {},
       "cell_type": "markdown",
       "metadata": {
         "id": "6LCGvBNvBd8D"
@@ -1045,9 +1022,7 @@
       "source": [
         "## Create Online Queries\n",
         "\n",
-        "After you built your indexes, you may query against the deployed index to find nearest neighbors.\n",
-        "\n",
-        "Note: For the DOT_PRODUCT_DISTANCE distance type, the \"distance\" property returned with each MatchNeighbor actually refers to the similarity."
+        "After you built your indexes, you may query against the deployed index to find nearest neighbors."
       ]
     },
     {
@@ -1112,100 +1087,6 @@
         "        )"
       ]
     },
-    {
-      "attachments": {},
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "05514825ba7d"
-      },
-      "source": [
-        "## Storing and retrieving titles from a Redis data store\n",
-        "When you productionize this code into a service, you will need to convert the nearest nearest id's returned from Vertex AI Matching Engine into data usable by downstream services.\n",
-        "\n",
-        "In this case, you'll need to convert the id's to titles.\n",
-        "\n",
-        "You can use Google Cloud's Memorystore to deploy a managed Redis instance to save the id-title key-value pairs.\n",
-        "\n",
-        "See more information on [Memorystore](https://cloud.google.com/memorystore/docs/redis/create-manage-instances?hl=en)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "5d2b240f0d52"
-      },
-      "outputs": [],
-      "source": [
-        "REDIS_INSTANCE_NAME = \"stackoverflow-questions-unique\"\n",
-        "\n",
-        "# Create a Redis instance\n",
-        "! gcloud redis instances create '{REDIS_INSTANCE_NAME}' --size=5 --region={REGION} --network={VPC_NETWORK_FULL} --connect-mode=private-service-access"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "371ccc0d2eb2"
-      },
-      "outputs": [],
-      "source": [
-        "# Get host and port info\n",
-        "REDIS_HOST = ! gcloud redis instances list --filter=\"INSTANCE_NAME:'{REDIS_INSTANCE_NAME}'\" --region {REGION}  --format='value(HOST)'\n",
-        "REDIS_PORT = ! gcloud redis instances list --filter=\"INSTANCE_NAME:'{REDIS_INSTANCE_NAME}'\" --region {REGION} --format='value(PORT)'\n",
-        "\n",
-        "if isinstance(REDIS_HOST, list):\n",
-        "    REDIS_HOST = REDIS_HOST[0]\n",
-        "\n",
-        "if isinstance(REDIS_PORT, list):\n",
-        "    REDIS_PORT = REDIS_PORT[0]\n",
-        "\n",
-        "print(f\"REDIS_HOST = {REDIS_HOST}\")\n",
-        "print(f\"REDIS_PORT = {REDIS_PORT}\")"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "73796089386a"
-      },
-      "outputs": [],
-      "source": [
-        "# Connect to the instance\n",
-        "import redis\n",
-        "\n",
-        "redis_client = redis.StrictRedis(host=REDIS_HOST, port=REDIS_PORT)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "f000f5432d13"
-      },
-      "outputs": [],
-      "source": [
-        "# Convert the id -> title relationship into a dict and write to redis\n",
-        "redis_client.mset({str(id): str(title) for id, title in zip(df.id, df.title)})"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "b1f8b396aeb1"
-      },
-      "outputs": [],
-      "source": [
-        "# Verify that redis can retrieve the correct information\n",
-        "[\n",
-        "    f\"Actual = {title}, Retrieved = {redis_client.get(str(id))}\"\n",
-        "    for id, title in list(zip(df.id, df.title))[:10]\n",
-        "]"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -1242,18 +1123,6 @@
         "# Delete indexes\n",
         "tree_ah_index.delete()"
       ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "d2fcf9468031"
-      },
-      "outputs": [],
-      "source": [
-        "# Delete redis instance\n",
-        "! gcloud redis instances delete '{REDIS_INSTANCE_NAME}' --region {REGION} --quiet"
-      ]
     }
   ],
   "metadata": {

notebooks/official/ml_metadata/vertex-pipelines-ml-metadata.ipynb

@@ -719,7 +719,7 @@
       "outputs": [],
       "source": [
         "@component(\n",
-        "    packages_to_install=[\"scikit-learn\", \"pandas\", \"joblib\"],\n",
+        "    packages_to_install=[\"sklearn\", \"pandas\", \"joblib\"],\n",
         "    base_image=\"python:3.9\",\n",
         "    output_component_file=\"beans_model_component.yaml\",\n",
         ")\n",

notebooks/official/model_evaluation/automl_text_classification_model_evaluation.ipynb

@@ -88,7 +88,7 @@
         "The steps performed include:\n",
         "\n",
         "- Create a Vertex AI `Dataset`.\n",
-        "- Train a Automl Tabular Classification model on the `Dataset` resource.\n",
+        "- Train a Automl Text Classification model on the `Dataset` resource.\n",
         "- Import the trained `AutoML model resource` into the pipeline.\n",
         "- Run a `Batch Prediction` job.\n",
         "- Evaulate the AutoML model using the `Classification Evaluation Component`.\n",
@@ -195,7 +195,7 @@
         "\n",
         "! pip3 install --upgrade google-cloud-aiplatform \\\n",
         "                         google-cloud-storage \\\n",
-        "                         kfp google-cloud-pipeline-components \\\n",
+        "                         kfp google-cloud-pipeline-components==1.0.25 \\\n",
         "                         ndjson {USER_FLAG} -q"
       ]
     },
@@ -789,7 +789,7 @@
         "    enable_caching=False,\n",
         ")\n",
         "\n",
-        "job.run()\n",
+        "job.run(sync=True)\n",
         "\n",
         "! rm text_classification_pipeline.json"
       ]
@@ -1209,7 +1209,7 @@
         "The pipeline uses the following components:\n",
         "\n",
         "- `GetVertexModelOp`: Gets a Vertex AI Model Artifact. \n",
-        "- `EvaluationDataSamplerOp`: Randomly downsamples an input dataset to a specified size for computing Vertex Explainable AI feature attributions for AutoML Tabular and custom models. Creates a Dataflow job with Apache Beam to downsample the dataset. \n",
+        "- `EvaluationDataSamplerOp`: Randomly downsamples an input dataset to a specified size for computing Vertex Explainable AI feature attributions for AutoML Text and custom models. Creates a Dataflow job with Apache Beam to downsample the dataset. \n",
         "- `EvaluationDataSplitterOp`: Removes the Ground Truth columns from the input dataset for supporting unstructured AutoML models and custom models in Batch Prediction. Creates a Dataflow job with Apache Beam to remove the ground truth columns.\n",
         "- `ModelBatchPredictOp`: Creates a Google Cloud Vertex BatchPredictionJob and waits for it to complete. \n",
         "- `ModelEvaluationClassificationOp`: Compute evaluation metrics on a trained model’s batch prediction results. Creates a Dataflow job with Apache Beam and TFMA to compute evaluation metrics. Supports mutliclass classification evaluation for tabular, image, video, and text data. \n",
@@ -1392,7 +1392,7 @@
         "- `project`: Project ID.\n",
         "- `location`: Region where the pipeline is run.\n",
         "- `root_dir`: The GCS directory for keeping staging files and artifacts. A random subdirectory is created under the directory to keep job info for resuming the job in case of failure.\n",
-        "- `model_name`: Resource name of the trained AutoML Tabular Classification model.\n",
+        "- `model_name`: Resource name of the trained AutoML Text Classification model.\n",
         "- `target_column_name`: Name of the column to be used as the target for classification.\n",
         "- `batch_predict_gcs_source_uris`: List of the Cloud Storage bucket uris of input instances for batch prediction.\n",
         "- `batch_predict_instances_format`: Format of the input instances for batch prediction. Format used here is'**jsonl**'.\n",
@@ -1459,7 +1459,7 @@
         "    enable_caching=False,\n",
         ")\n",
         "\n",
-        "evaluation_job.run(service_account=SERVICE_ACCOUNT)"
+        "evaluation_job.run(service_account=SERVICE_ACCOUNT, sync=True)"
       ]
     },
     {
@@ -1591,7 +1591,7 @@
         "model_evaluation_id = model_evaluation[\"resourceUri\"].split(\"/\")[-1]\n",
         "print(model_evaluation_id)\n",
         "\n",
-        "evaluation = model.get_model_evaluation(evaluation_id=model_evaluation_id)\n",
+        "evaluation = model.get_model_evaluation()  # evaluation_id=model_evaluation_id)\n",
         "evaluation = evaluation.to_dict()\n",
         "print(\"Model's evaluation metrics from Training:\\n\")\n",
         "metrics = evaluation[\"metrics\"]\n",

notebooks/official/pipelines/metrics_viz_run_compare_kfp.ipynb

@@ -713,7 +713,7 @@
       "outputs": [],
       "source": [
         "@component(\n",
-        "    packages_to_install=[\"scikit-learn\"],\n",
+        "    packages_to_install=[\"sklearn\"],\n",
         "    base_image=\"python:3.9\",\n",
         "    output_component_file=\"wine_classification_component.yaml\",\n",
         ")\n",
@@ -758,7 +758,7 @@
       },
       "outputs": [],
       "source": [
-        "@component(packages_to_install=[\"scikit-learn\"], base_image=\"python:3.9\")\n",
+        "@component(packages_to_install=[\"sklearn\"], base_image=\"python:3.9\")\n",
         "def iris_sgdclassifier(\n",
         "    test_samples_fraction: float,\n",
         "    metricsc: Output[ClassificationMetrics],\n",
@@ -805,7 +805,7 @@
       "outputs": [],
       "source": [
         "@component(\n",
-        "    packages_to_install=[\"scikit-learn\"],\n",
+        "    packages_to_install=[\"sklearn\"],\n",
         "    base_image=\"python:3.9\",\n",
         ")\n",
         "def iris_logregression(\n",