Ran linter

fix: issue 1251

.cloud-build/cleanup/cleanup.py

@@ -7,6 +7,12 @@ from resource_cleanup_manager import (
     ResourceCleanupManager,
     MatchingEngineIndexEndpointResourceCleanupManager,
     MatchingEngineIndexResourceCleanupManager,
+    FeatureStoreCleanupManager,
+    PipelineJobCleanupManager,
+    TrainingJobCleanupManager,
+    HyperparameterTuningCleanupManager,
+    BatchPredictionJobCleanupManager,
+    ExperimentCleanupManager
 )
 
 rate_limit = RateLimit(max_count=25, per=60, greedy=False)
@@ -48,6 +54,12 @@ managers: List[ResourceCleanupManager] = [
     ModelResourceCleanupManager(),  # ModelResourceCleanupManager must follow EndpointResourceCleanupManager due to deployed models blocking model deletion.
     MatchingEngineIndexEndpointResourceCleanupManager(),
     MatchingEngineIndexResourceCleanupManager(),
+    FeatureStoreCleanupManager(),
+    PipelineJobCleanupManager(),
+    TrainingJobCleanupManager(),
+    HyperparameterTuningCleanupManager(),
+    BatchPredictionJobCleanupManager(),
+    # ExperimentCleanupManager(), # Experiment missing _resource_noun
 ]
 
 run_cleanup_managers(managers=managers, is_dry_run=is_dry_run)

.cloud-build/cleanup/resource_cleanup_manager.py

@@ -97,8 +97,6 @@ class EndpointResourceCleanupManager(VertexAIResourceCleanupManager):
     vertex_ai_resource = aiplatform.Endpoint
 
     def delete(self, resource):
-        # TODO: Remove this once https://github.com/googleapis/python-aiplatform/issues/1441 is fixed
-        resource._sync_gca_resource()
         for deployed_model_id in [
             models.id for models in resource._gca_resource.deployed_models
         ]:
@@ -119,4 +117,42 @@ class MatchingEngineIndexEndpointResourceCleanupManager(VertexAIResourceCleanupM
 
     def delete(self, resource):
         resource.undeploy_all()
-        resource.delete(force=True)
+        resource.delete(force=True)
+
+class FeatureStoreCleanupManager(VertexAIResourceCleanupManager):
+    vertex_ai_resource = aiplatform.Featurestore
+
+class PipelineJobCleanupManager(VertexAIResourceCleanupManager):
+    vertex_ai_resource = aiplatform.PipelineJob
+
+class TrainingJobCleanupManager(VertexAIResourceCleanupManager):
+    vertex_ai_resource = aiplatform.training_jobs._CustomTrainingJob
+
+    job_types = [
+            aiplatform.AutoMLImageTrainingJob,
+            aiplatform.AutoMLTextTrainingJob,
+            aiplatform.AutoMLTabularTrainingJob,
+            aiplatform.AutoMLVideoTrainingJob,
+            aiplatform.AutoMLForecastingTrainingJob,
+            aiplatform.CustomJob,
+            aiplatform.CustomTrainingJob,
+            aiplatform.CustomContainerTrainingJob,
+            aiplatform.CustomPythonPackageTrainingJob
+    ]
+
+    def list(self) -> Any:
+        return [
+            job
+            for job_type in self.job_types
+            for job in job_type.list()
+        ]
+
+class HyperparameterTuningCleanupManager(VertexAIResourceCleanupManager):
+    vertex_ai_resource = aiplatform.HyperparameterTuningJob
+
+
+class BatchPredictionJobCleanupManager(VertexAIResourceCleanupManager):
+    vertex_ai_resource = aiplatform.BatchPredictionJob
+
+class ExperimentCleanupManager(VertexAIResourceCleanupManager):
+    vertex_ai_resource = aiplatform.Experiment

.cloud-build/execute_changed_notebooks_helper.py

@@ -156,7 +156,7 @@ def _create_tag(filepath: str) -> str:
     return tag
 
 
-rate_limit = RateLimit(max_count=50, per=60, greedy=True)
+rate_limit = RateLimit(max_count=25, per=60, greedy=True)
 
 
 def process_and_execute_notebook(

notebooks/community/prediction/feature_store_integration/prediction_feature_store_integration.ipynb

@@ -1019,9 +1019,11 @@
         "    // \"projects/acme/locations/us-central1/featurestores/fs/entityTypes/movies\"\n",
         "    string entity_type = 2;\n",
         "\n",
-        "    // Required. Specifies the field holding the entityId to be fetched for this\n",
-        "    // sources. Currently, input request is a (JSON) dictionary, it simply\n",
-        "    // corresponds to the dictionary entry with this key.\n",
+        "    // Required. Specifies the name of the field in the request sent by the user\n",
+        "    // (NOT the auto generated prediction request with feature values) that\n",
+        "    // holds the entityID to be fetched from this source. The input request sent\n",
+        "    // by the user is a JSON dictionary. The entity ID to be fetched is held by\n",
+        "    // the dictionary entry with this key.\n",
         "    string entity_id_field = 3;\n",
         "  }\n",
         "\n",
@@ -1256,7 +1258,7 @@
         "\n",
         "We need a provide service account for this new feature because the prediction workload's default identity does not have access to Feature Store. The service account needs to have `Vertex AI Feature Store Data Viewer` in your project.\n",
         "\n",
-        "For this bug bash, a service account has already been created for you."
+        "Open a terminal and run `gcloud auth login` before running the commands below."
       ]
     },
     {

notebooks/official.csv

@@ -1,70 +0,0 @@
-tag,notebook,doc
-"AutoML, Text data",official/automl/automl-text-classification.ipynb,vertex-ai/docs/text-data/classification/train-model
-"AutoML, Text data",official/automl/sdk_automl_text_entity_extraction_online.ipynb,
-"AutoML, Text data",official/automl/sdk_automl_text_sentiment_analysis_online.ipynb,
-"AutoML, Tabular data",official/automl/sdk_automl_tabular_forecasting_batch.ipynb,vertex-ai/docs/tabular-data/forecasting/tutorials-samples
-"AutoML, Tabular Data",official/automl/automl_tabular_on_vertex_pipelines.ipynb,vertex-ai/docs/tabular-data/tabular-workflows/e2e-automl
-"AutoML, Tabular Data",official/automl/sdk_automl_tabular_regression_batch_bq.ipynb,
-"AutoML, Tabular Data",official/automl/sdk_automl_tabular_regression_batch_bq.ipynb,
-"AutoML, Forecasting",official/automl/automl_forecasting_bqml_arima_plus_comparison.ipynb,vertex-ai/docs/tabular-data/forecasting-arima/overview
-"AutoML, Forecasting",official/automl/sdk_automl_tabular_forecasting_batch.ipynb,
-"AutoML, Image data",official/automl/sdk_automl_text_sentiment_analysis_online.ipynb,
-"AutoML, Video data",official/automl/sdk_automl_text_sentiment_analysis_online.ipynb,
-"AutoML, Video data",official/automl/sdk_automl_video_classification_batch.ipynb,
-"AutoML, Video data",official/automl/sdk_automl_video_object_tracking_batch.ipynb,
-"AutoML, Video data",official/sdk/SDK_AutoML_Video_Classification.ipynb,
-"BigQuery, Vertex AI Workbench",official/workbench/exploratory_data_analysis/explore_data_in_bigquery_with_workbench.ipynb,
-"BigQuery ML, Vertex AI Model Registry, Batch prediction",official/model_registry/bqml_vertexai_model_registry.ipynb,
-"BigQuery ML, Vertex AI Model Registry, Online prediction",official/bigquery_ml/bqml-online-prediction.ipynb,
-"BigQuery ML",official/structured_data/rapid_prototyping_bqml_automl.ipynb,
-Custom Training,official/custom/sdk-custom-image-classification-batch.ipynb,
-Custom Training,official/custom/sdk-custom-image-classification-online.ipynb,
-Custom Training,official/custom/SDK_Custom_Container_Prediction.ipynb,
-"Custom Training, BiqQuery dataset",official/custom/custom-tabular-bq-managed-dataset.ipynb,
-"Custom Training, TensorBoard",official/custom/custom-tabular-bq-managed-dataset.ipynb,
-"Custom Training, TensorBoard",official/tensorboard/tensorboard_custom_training_with_custom_container.ipynb,
-"Custom Training, TensorBoard",official/tensorboard/tensorboard_custom_training_with_prebuilt_container.ipynb
-"Custom Training, Managed dataset",official/sdk/SDK_Custom_Training_Python_Package_Managed_Text_Dataset_Tensorflow_Serving_Container.ipynb,
-"Custom Training, Distributed",official/training/multi_node_ddp_gloo_vertex_training_with_custom_container.ipynb,
-"Custom Training, Distributed",official/training/multi_node_ddp_nccl_vertex_training_with_custom_container.ipynb
-Vertex AI Experiments,official/experiments/comparing_pipeline_runs.ipynb,
-Vertex AI Experiments,official/experiments/build_model_experimentation_lineage_with_prebuild_code.ipynb,
-Vertex AI Experiments,official/experiments/comparing_local_trained_models.ipynb,
-"Vertex Explainable AI, Tabular data",official/explainable_ai/sdk_automl_tabular_binary_classification_batch_explain.ipynb,vertex-ai/docs/explainable-ai/overview
-"Vertex Explainable AI, Tabular data",official/explainable_ai/sdk_automl_tabular_classification_online_explain.ipynb,vertex-ai/docs/explainable-ai/overview
-"Vertex Explainable AI, Image data",official/explainable_ai/sdk_custom_image_classification_batch_explain.ipynb,vertex-ai/docs/explainable-ai/overview
-"Vertex Explainable AI, Tabular data",official/explainable_ai/sdk_custom_tabular_regression_batch_explain.ipynb,vertex-ai/docs/explainable-ai/overview
-"Vertex Explainable AI, Tabular data",official/explainable_ai/sdk_custom_tabular_regression_online_explain.ipynb,vertex-ai/docs/explainable-ai/overview
-Vertex ML Metadata,official/ml_metadata/sdk-metric-parameter-tracking-for-custom-jobs.ipynb,
-"Vertex Explainable AI, Image data",official/explainable_ai/sdk_custom_image_classification_online_explain.ipynb,vertex-ai/docs/explainable-ai/overview
-Vertex AI Feature Store,official/feature_store/sdk-feature-store.ipynb,
-Vertex AI Feature Store,official/feature_store/sdk-feature-store-pandas.ipynb,
-Vertex AI Matching Engine,official/matching_engine/sdk_matching_engine_for_indexing.ipynb,
-Vertex ML Metadata,official/ml_metadata/sdk-metric-parameter-tracking-for-custom-jobs.ipynb,
-Vertex ML Metadata,official/ml_metadata/sdk-metric-parameter-tracking-for-locally-trained-models.ipynb,
-"Vertex ML Metadata, Vertex AI Pipelines",official/ml_metadata/sdk-metric-parameter-tracking-for-locally-trained-models.ipynb,
-"Vertex AI Model Evaluation, AutoML",official/model_evaluation/automl_tabular_classification_model_evaluation.ipynb,
-"Vertex AI Model Evaluation, AutoML",official/model_evaluation/automl_tabular_regression_model_evaluation.ipynb,
-"Vertex AI Model Evaluation, AutoML",official/model_evaluation/automl_text_classification_model_evaluation.ipynb,
-"Vertex AI Model Evaluation, AutoML",official/model_evaluation/automl_video_classification_model_evaluation.ipynb,
-"Vertex AI Model Evaluation, Custom Training",official/model_evaluation/custom_tabular_regression_model_evaluation.ipynb,
-Model Monitoring,official/model_monitoring/model_monitoring.ipynb,
-Vertex AI Pipelines,official/pipelines/pipelines_intro_kfp.ipynb,
-Vertex AI Pipelines,official/pipelines/control_flow_kfp.ipynb,
-Vertex AI Pipelines,official/pipelines/metrics_viz_run_compare_kfp.ipynb,
-Vertex AI Pipelines,official/pipelines/lightweight_functions_component_io_kfp.ipynb,
-"Vertex AI Pipelines Image data",official/pipelines/google_cloud_pipeline_components_automl_images.ipynb,
-"Vertex AI Pipelines, Tabular data",official/pipelines/automl_tabular_classification_beans.ipynb,
-"Vertex AI Pipelines, Tabular data",official/pipelines/google_cloud_pipeline_components_automl_tabular.ipynb,
-"Vertex AI Pipelines, Tabular data",official/pipelines/google_cloud_pipeline_components_dataproc_tabular.ipynb,
-"Vertex AI Pipelines, Text data",official/pipelines/google_cloud_pipeline_components_automl_text.ipynb,
-"Vertex AI Pipelines, Text data",official/pipelines/google_cloud_pipeline_components_bqml_text.ipynb,
-Vertex AI Pipelines,official/pipelines/custom_model_training_and_batch_prediction.ipynb,
-Vertex AI Pipelines,official/pipelines/google_cloud_pipeline_components_model_train_upload_deploy.ipynb,
-Vertex AI Pipelines,official/pipelines/google_cloud_pipeline_components_model_upload_predict_evaluate.ipynb,
-"Vertex AI Training, Reduction Server, PyTorch",official/reduction_server/pytorch_distributed_training_reduction_server.ipynb,
-"Tabular Workflows, Vertex AI TabNet",official/tabnet/tabnet_vertex_tutorial.ipynb,
-"Tabular Workflows, Vertex AI TabNet, Vertex Explainablee AI",official/tabnet/ai-explanations-tabnet-algorithm.ipynb,
-"Tabular Workflows, Vertex AI TabNet, Vertex AI Pipelines",official/tabular_workflows/tabnet_on_vertex_pipelines.ipynb,
-"Tabular Workflows, Vertex AI Wide and Deep",official/tabular_workflows/wide_and_deep_on_vertex_pipelines.ipynb,
-Vertex AI Vizier,official/vizier/gapic-vizier-multi-objective-optimization.ipynb,vertex-ai/docs/vizier/using-vizier

notebooks/official/matching_engine/sdk_matching_engine_create_stack_overflow_embeddings.ipynb

@@ -150,6 +150,28 @@
         "                        tensorflow-hub"
       ]
     },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "54ac7ebac10b"
+      },
+      "source": [
+        "Install the latest version of Redis for low-latency data retrieval"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "e3dd53b3c06c"
+      },
+      "outputs": [],
+      "source": [
+        "# Install the redis package\n",
+        "! pip install --upgrade redis"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -1015,6 +1037,7 @@
       ]
     },
     {
+      "attachments": {},
       "cell_type": "markdown",
       "metadata": {
         "id": "6LCGvBNvBd8D"
@@ -1022,7 +1045,9 @@
       "source": [
         "## Create Online Queries\n",
         "\n",
-        "After you built your indexes, you may query against the deployed index to find nearest neighbors."
+        "After you built your indexes, you may query against the deployed index to find nearest neighbors.\n",
+        "\n",
+        "Note: For the DOT_PRODUCT_DISTANCE distance type, the \"distance\" property returned with each MatchNeighbor actually refers to the similarity."
       ]
     },
     {
@@ -1087,6 +1112,100 @@
         "        )"
       ]
     },
+    {
+      "attachments": {},
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "05514825ba7d"
+      },
+      "source": [
+        "## Storing and retrieving titles from a Redis data store\n",
+        "When you productionize this code into a service, you will need to convert the nearest nearest id's returned from Vertex AI Matching Engine into data usable by downstream services.\n",
+        "\n",
+        "In this case, you'll need to convert the id's to titles.\n",
+        "\n",
+        "You can use Google Cloud's Memorystore to deploy a managed Redis instance to save the id-title key-value pairs.\n",
+        "\n",
+        "See more information on [Memorystore](https://cloud.google.com/memorystore/docs/redis/create-manage-instances?hl=en)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "5d2b240f0d52"
+      },
+      "outputs": [],
+      "source": [
+        "REDIS_INSTANCE_NAME = \"stackoverflow-questions-unique\"\n",
+        "\n",
+        "# Create a Redis instance\n",
+        "! gcloud redis instances create '{REDIS_INSTANCE_NAME}' --size=5 --region={REGION} --network={VPC_NETWORK_FULL} --connect-mode=private-service-access"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "371ccc0d2eb2"
+      },
+      "outputs": [],
+      "source": [
+        "# Get host and port info\n",
+        "REDIS_HOST = ! gcloud redis instances list --filter=\"INSTANCE_NAME:'{REDIS_INSTANCE_NAME}'\" --region {REGION}  --format='value(HOST)'\n",
+        "REDIS_PORT = ! gcloud redis instances list --filter=\"INSTANCE_NAME:'{REDIS_INSTANCE_NAME}'\" --region {REGION} --format='value(PORT)'\n",
+        "\n",
+        "if isinstance(REDIS_HOST, list):\n",
+        "    REDIS_HOST = REDIS_HOST[0]\n",
+        "\n",
+        "if isinstance(REDIS_PORT, list):\n",
+        "    REDIS_PORT = REDIS_PORT[0]\n",
+        "\n",
+        "print(f\"REDIS_HOST = {REDIS_HOST}\")\n",
+        "print(f\"REDIS_PORT = {REDIS_PORT}\")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "73796089386a"
+      },
+      "outputs": [],
+      "source": [
+        "# Connect to the instance\n",
+        "import redis\n",
+        "\n",
+        "redis_client = redis.StrictRedis(host=REDIS_HOST, port=REDIS_PORT)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "f000f5432d13"
+      },
+      "outputs": [],
+      "source": [
+        "# Convert the id -> title relationship into a dict and write to redis\n",
+        "redis_client.mset({str(id): str(title) for id, title in zip(df.id, df.title)})"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "b1f8b396aeb1"
+      },
+      "outputs": [],
+      "source": [
+        "# Verify that redis can retrieve the correct information\n",
+        "[\n",
+        "    f\"Actual = {title}, Retrieved = {redis_client.get(str(id))}\"\n",
+        "    for id, title in list(zip(df.id, df.title))[:10]\n",
+        "]"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -1123,6 +1242,18 @@
         "# Delete indexes\n",
         "tree_ah_index.delete()"
       ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "d2fcf9468031"
+      },
+      "outputs": [],
+      "source": [
+        "# Delete redis instance\n",
+        "! gcloud redis instances delete '{REDIS_INSTANCE_NAME}' --region {REGION} --quiet"
+      ]
     }
   ],
   "metadata": {

notebooks/official/ml_metadata/vertex-pipelines-ml-metadata.ipynb

@@ -719,7 +719,7 @@
       "outputs": [],
       "source": [
         "@component(\n",
-        "    packages_to_install=[\"sklearn\", \"pandas\", \"joblib\"],\n",
+        "    packages_to_install=[\"scikit-learn\", \"pandas\", \"joblib\"],\n",
         "    base_image=\"python:3.9\",\n",
         "    output_component_file=\"beans_model_component.yaml\",\n",
         ")\n",

notebooks/official/pipelines/metrics_viz_run_compare_kfp.ipynb

@@ -713,7 +713,7 @@
       "outputs": [],
       "source": [
         "@component(\n",
-        "    packages_to_install=[\"sklearn\"],\n",
+        "    packages_to_install=[\"scikit-learn\"],\n",
         "    base_image=\"python:3.9\",\n",
         "    output_component_file=\"wine_classification_component.yaml\",\n",
         ")\n",
@@ -758,7 +758,7 @@
       },
       "outputs": [],
       "source": [
-        "@component(packages_to_install=[\"sklearn\"], base_image=\"python:3.9\")\n",
+        "@component(packages_to_install=[\"scikit-learn\"], base_image=\"python:3.9\")\n",
         "def iris_sgdclassifier(\n",
         "    test_samples_fraction: float,\n",
         "    metricsc: Output[ClassificationMetrics],\n",
@@ -805,7 +805,7 @@
       "outputs": [],
       "source": [
         "@component(\n",
-        "    packages_to_install=[\"sklearn\"],\n",
+        "    packages_to_install=[\"scikit-learn\"],\n",
         "    base_image=\"python:3.9\",\n",
         ")\n",
         "def iris_logregression(\n",