fix: get eval by id

modify protobuf docs and add instructions

.cloud-build/cleanup/cleanup.py

@@ -7,6 +7,12 @@ from resource_cleanup_manager import (
     ResourceCleanupManager,
     MatchingEngineIndexEndpointResourceCleanupManager,
     MatchingEngineIndexResourceCleanupManager,
+    FeatureStoreCleanupManager,
+    PipelineJobCleanupManager,
+    TrainingJobCleanupManager,
+    HyperparameterTuningCleanupManager,
+    BatchPredictionJobCleanupManager,
+    ExperimentCleanupManager
 )
 
 rate_limit = RateLimit(max_count=25, per=60, greedy=False)
@@ -48,6 +54,12 @@ managers: List[ResourceCleanupManager] = [
     ModelResourceCleanupManager(),  # ModelResourceCleanupManager must follow EndpointResourceCleanupManager due to deployed models blocking model deletion.
     MatchingEngineIndexEndpointResourceCleanupManager(),
     MatchingEngineIndexResourceCleanupManager(),
+    FeatureStoreCleanupManager(),
+    PipelineJobCleanupManager(),
+    TrainingJobCleanupManager(),
+    HyperparameterTuningCleanupManager(),
+    BatchPredictionJobCleanupManager(),
+    # ExperimentCleanupManager(), # Experiment missing _resource_noun
 ]
 
 run_cleanup_managers(managers=managers, is_dry_run=is_dry_run)

.cloud-build/cleanup/resource_cleanup_manager.py

@@ -97,8 +97,6 @@ class EndpointResourceCleanupManager(VertexAIResourceCleanupManager):
     vertex_ai_resource = aiplatform.Endpoint
 
     def delete(self, resource):
-        # TODO: Remove this once https://github.com/googleapis/python-aiplatform/issues/1441 is fixed
-        resource._sync_gca_resource()
         for deployed_model_id in [
             models.id for models in resource._gca_resource.deployed_models
         ]:
@@ -119,4 +117,42 @@ class MatchingEngineIndexEndpointResourceCleanupManager(VertexAIResourceCleanupM
 
     def delete(self, resource):
         resource.undeploy_all()
-        resource.delete(force=True)
+        resource.delete(force=True)
+
+class FeatureStoreCleanupManager(VertexAIResourceCleanupManager):
+    vertex_ai_resource = aiplatform.Featurestore
+
+class PipelineJobCleanupManager(VertexAIResourceCleanupManager):
+    vertex_ai_resource = aiplatform.PipelineJob
+
+class TrainingJobCleanupManager(VertexAIResourceCleanupManager):
+    vertex_ai_resource = aiplatform.training_jobs._CustomTrainingJob
+
+    job_types = [
+            aiplatform.AutoMLImageTrainingJob,
+            aiplatform.AutoMLTextTrainingJob,
+            aiplatform.AutoMLTabularTrainingJob,
+            aiplatform.AutoMLVideoTrainingJob,
+            aiplatform.AutoMLForecastingTrainingJob,
+            aiplatform.CustomJob,
+            aiplatform.CustomTrainingJob,
+            aiplatform.CustomContainerTrainingJob,
+            aiplatform.CustomPythonPackageTrainingJob
+    ]
+
+    def list(self) -> Any:
+        return [
+            job
+            for job_type in self.job_types
+            for job in job_type.list()
+        ]
+
+class HyperparameterTuningCleanupManager(VertexAIResourceCleanupManager):
+    vertex_ai_resource = aiplatform.HyperparameterTuningJob
+
+
+class BatchPredictionJobCleanupManager(VertexAIResourceCleanupManager):
+    vertex_ai_resource = aiplatform.BatchPredictionJob
+
+class ExperimentCleanupManager(VertexAIResourceCleanupManager):
+    vertex_ai_resource = aiplatform.Experiment

notebooks/community/prediction/feature_store_integration/prediction_feature_store_integration.ipynb

@@ -1019,9 +1019,11 @@
         "    // \"projects/acme/locations/us-central1/featurestores/fs/entityTypes/movies\"\n",
         "    string entity_type = 2;\n",
         "\n",
-        "    // Required. Specifies the field holding the entityId to be fetched for this\n",
-        "    // sources. Currently, input request is a (JSON) dictionary, it simply\n",
-        "    // corresponds to the dictionary entry with this key.\n",
+        "    // Required. Specifies the name of the field in the request sent by the user\n",
+        "    // (NOT the auto generated prediction request with feature values) that\n",
+        "    // holds the entityID to be fetched from this source. The input request sent\n",
+        "    // by the user is a JSON dictionary. The entity ID to be fetched is held by\n",
+        "    // the dictionary entry with this key.\n",
         "    string entity_id_field = 3;\n",
         "  }\n",
         "\n",
@@ -1256,7 +1258,7 @@
         "\n",
         "We need a provide service account for this new feature because the prediction workload's default identity does not have access to Feature Store. The service account needs to have `Vertex AI Feature Store Data Viewer` in your project.\n",
         "\n",
-        "For this bug bash, a service account has already been created for you."
+        "Open a terminal and run `gcloud auth login` before running the commands below."
       ]
     },
     {

notebooks/official.csv

@@ -1,70 +0,0 @@
-tag,notebook,doc
-"AutoML, Text data",official/automl/automl-text-classification.ipynb,vertex-ai/docs/text-data/classification/train-model
-"AutoML, Text data",official/automl/sdk_automl_text_entity_extraction_online.ipynb,
-"AutoML, Text data",official/automl/sdk_automl_text_sentiment_analysis_online.ipynb,
-"AutoML, Tabular data",official/automl/sdk_automl_tabular_forecasting_batch.ipynb,vertex-ai/docs/tabular-data/forecasting/tutorials-samples
-"AutoML, Tabular Data",official/automl/automl_tabular_on_vertex_pipelines.ipynb,vertex-ai/docs/tabular-data/tabular-workflows/e2e-automl
-"AutoML, Tabular Data",official/automl/sdk_automl_tabular_regression_batch_bq.ipynb,
-"AutoML, Tabular Data",official/automl/sdk_automl_tabular_regression_batch_bq.ipynb,
-"AutoML, Forecasting",official/automl/automl_forecasting_bqml_arima_plus_comparison.ipynb,vertex-ai/docs/tabular-data/forecasting-arima/overview
-"AutoML, Forecasting",official/automl/sdk_automl_tabular_forecasting_batch.ipynb,
-"AutoML, Image data",official/automl/sdk_automl_text_sentiment_analysis_online.ipynb,
-"AutoML, Video data",official/automl/sdk_automl_text_sentiment_analysis_online.ipynb,
-"AutoML, Video data",official/automl/sdk_automl_video_classification_batch.ipynb,
-"AutoML, Video data",official/automl/sdk_automl_video_object_tracking_batch.ipynb,
-"AutoML, Video data",official/sdk/SDK_AutoML_Video_Classification.ipynb,
-"BigQuery, Vertex AI Workbench",official/workbench/exploratory_data_analysis/explore_data_in_bigquery_with_workbench.ipynb,
-"BigQuery ML, Vertex AI Model Registry, Batch prediction",official/model_registry/bqml_vertexai_model_registry.ipynb,
-"BigQuery ML, Vertex AI Model Registry, Online prediction",official/bigquery_ml/bqml-online-prediction.ipynb,
-"BigQuery ML",official/structured_data/rapid_prototyping_bqml_automl.ipynb,
-Custom Training,official/custom/sdk-custom-image-classification-batch.ipynb,
-Custom Training,official/custom/sdk-custom-image-classification-online.ipynb,
-Custom Training,official/custom/SDK_Custom_Container_Prediction.ipynb,
-"Custom Training, BiqQuery dataset",official/custom/custom-tabular-bq-managed-dataset.ipynb,
-"Custom Training, TensorBoard",official/custom/custom-tabular-bq-managed-dataset.ipynb,
-"Custom Training, TensorBoard",official/tensorboard/tensorboard_custom_training_with_custom_container.ipynb,
-"Custom Training, TensorBoard",official/tensorboard/tensorboard_custom_training_with_prebuilt_container.ipynb
-"Custom Training, Managed dataset",official/sdk/SDK_Custom_Training_Python_Package_Managed_Text_Dataset_Tensorflow_Serving_Container.ipynb,
-"Custom Training, Distributed",official/training/multi_node_ddp_gloo_vertex_training_with_custom_container.ipynb,
-"Custom Training, Distributed",official/training/multi_node_ddp_nccl_vertex_training_with_custom_container.ipynb
-Vertex AI Experiments,official/experiments/comparing_pipeline_runs.ipynb,
-Vertex AI Experiments,official/experiments/build_model_experimentation_lineage_with_prebuild_code.ipynb,
-Vertex AI Experiments,official/experiments/comparing_local_trained_models.ipynb,
-"Vertex Explainable AI, Tabular data",official/explainable_ai/sdk_automl_tabular_binary_classification_batch_explain.ipynb,vertex-ai/docs/explainable-ai/overview
-"Vertex Explainable AI, Tabular data",official/explainable_ai/sdk_automl_tabular_classification_online_explain.ipynb,vertex-ai/docs/explainable-ai/overview
-"Vertex Explainable AI, Image data",official/explainable_ai/sdk_custom_image_classification_batch_explain.ipynb,vertex-ai/docs/explainable-ai/overview
-"Vertex Explainable AI, Tabular data",official/explainable_ai/sdk_custom_tabular_regression_batch_explain.ipynb,vertex-ai/docs/explainable-ai/overview
-"Vertex Explainable AI, Tabular data",official/explainable_ai/sdk_custom_tabular_regression_online_explain.ipynb,vertex-ai/docs/explainable-ai/overview
-Vertex ML Metadata,official/ml_metadata/sdk-metric-parameter-tracking-for-custom-jobs.ipynb,
-"Vertex Explainable AI, Image data",official/explainable_ai/sdk_custom_image_classification_online_explain.ipynb,vertex-ai/docs/explainable-ai/overview
-Vertex AI Feature Store,official/feature_store/sdk-feature-store.ipynb,
-Vertex AI Feature Store,official/feature_store/sdk-feature-store-pandas.ipynb,
-Vertex AI Matching Engine,official/matching_engine/sdk_matching_engine_for_indexing.ipynb,
-Vertex ML Metadata,official/ml_metadata/sdk-metric-parameter-tracking-for-custom-jobs.ipynb,
-Vertex ML Metadata,official/ml_metadata/sdk-metric-parameter-tracking-for-locally-trained-models.ipynb,
-"Vertex ML Metadata, Vertex AI Pipelines",official/ml_metadata/sdk-metric-parameter-tracking-for-locally-trained-models.ipynb,
-"Vertex AI Model Evaluation, AutoML",official/model_evaluation/automl_tabular_classification_model_evaluation.ipynb,
-"Vertex AI Model Evaluation, AutoML",official/model_evaluation/automl_tabular_regression_model_evaluation.ipynb,
-"Vertex AI Model Evaluation, AutoML",official/model_evaluation/automl_text_classification_model_evaluation.ipynb,
-"Vertex AI Model Evaluation, AutoML",official/model_evaluation/automl_video_classification_model_evaluation.ipynb,
-"Vertex AI Model Evaluation, Custom Training",official/model_evaluation/custom_tabular_regression_model_evaluation.ipynb,
-Model Monitoring,official/model_monitoring/model_monitoring.ipynb,
-Vertex AI Pipelines,official/pipelines/pipelines_intro_kfp.ipynb,
-Vertex AI Pipelines,official/pipelines/control_flow_kfp.ipynb,
-Vertex AI Pipelines,official/pipelines/metrics_viz_run_compare_kfp.ipynb,
-Vertex AI Pipelines,official/pipelines/lightweight_functions_component_io_kfp.ipynb,
-"Vertex AI Pipelines Image data",official/pipelines/google_cloud_pipeline_components_automl_images.ipynb,
-"Vertex AI Pipelines, Tabular data",official/pipelines/automl_tabular_classification_beans.ipynb,
-"Vertex AI Pipelines, Tabular data",official/pipelines/google_cloud_pipeline_components_automl_tabular.ipynb,
-"Vertex AI Pipelines, Tabular data",official/pipelines/google_cloud_pipeline_components_dataproc_tabular.ipynb,
-"Vertex AI Pipelines, Text data",official/pipelines/google_cloud_pipeline_components_automl_text.ipynb,
-"Vertex AI Pipelines, Text data",official/pipelines/google_cloud_pipeline_components_bqml_text.ipynb,
-Vertex AI Pipelines,official/pipelines/custom_model_training_and_batch_prediction.ipynb,
-Vertex AI Pipelines,official/pipelines/google_cloud_pipeline_components_model_train_upload_deploy.ipynb,
-Vertex AI Pipelines,official/pipelines/google_cloud_pipeline_components_model_upload_predict_evaluate.ipynb,
-"Vertex AI Training, Reduction Server, PyTorch",official/reduction_server/pytorch_distributed_training_reduction_server.ipynb,
-"Tabular Workflows, Vertex AI TabNet",official/tabnet/tabnet_vertex_tutorial.ipynb,
-"Tabular Workflows, Vertex AI TabNet, Vertex Explainablee AI",official/tabnet/ai-explanations-tabnet-algorithm.ipynb,
-"Tabular Workflows, Vertex AI TabNet, Vertex AI Pipelines",official/tabular_workflows/tabnet_on_vertex_pipelines.ipynb,
-"Tabular Workflows, Vertex AI Wide and Deep",official/tabular_workflows/wide_and_deep_on_vertex_pipelines.ipynb,
-Vertex AI Vizier,official/vizier/gapic-vizier-multi-objective-optimization.ipynb,vertex-ai/docs/vizier/using-vizier

notebooks/official/model_evaluation/automl_text_classification_model_evaluation.ipynb

@@ -88,7 +88,7 @@
         "The steps performed include:\n",
         "\n",
         "- Create a Vertex AI `Dataset`.\n",
-        "- Train a Automl Tabular Classification model on the `Dataset` resource.\n",
+        "- Train a Automl Text Classification model on the `Dataset` resource.\n",
         "- Import the trained `AutoML model resource` into the pipeline.\n",
         "- Run a `Batch Prediction` job.\n",
         "- Evaulate the AutoML model using the `Classification Evaluation Component`.\n",
@@ -195,7 +195,7 @@
         "\n",
         "! pip3 install --upgrade google-cloud-aiplatform \\\n",
         "                         google-cloud-storage \\\n",
-        "                         kfp google-cloud-pipeline-components \\\n",
+        "                         kfp google-cloud-pipeline-components==1.0.25 \\\n",
         "                         ndjson {USER_FLAG} -q"
       ]
     },
@@ -789,7 +789,7 @@
         "    enable_caching=False,\n",
         ")\n",
         "\n",
-        "job.run()\n",
+        "job.run(sync=True)\n",
         "\n",
         "! rm text_classification_pipeline.json"
       ]
@@ -1209,7 +1209,7 @@
         "The pipeline uses the following components:\n",
         "\n",
         "- `GetVertexModelOp`: Gets a Vertex AI Model Artifact. \n",
-        "- `EvaluationDataSamplerOp`: Randomly downsamples an input dataset to a specified size for computing Vertex Explainable AI feature attributions for AutoML Tabular and custom models. Creates a Dataflow job with Apache Beam to downsample the dataset. \n",
+        "- `EvaluationDataSamplerOp`: Randomly downsamples an input dataset to a specified size for computing Vertex Explainable AI feature attributions for AutoML Text and custom models. Creates a Dataflow job with Apache Beam to downsample the dataset. \n",
         "- `EvaluationDataSplitterOp`: Removes the Ground Truth columns from the input dataset for supporting unstructured AutoML models and custom models in Batch Prediction. Creates a Dataflow job with Apache Beam to remove the ground truth columns.\n",
         "- `ModelBatchPredictOp`: Creates a Google Cloud Vertex BatchPredictionJob and waits for it to complete. \n",
         "- `ModelEvaluationClassificationOp`: Compute evaluation metrics on a trained model’s batch prediction results. Creates a Dataflow job with Apache Beam and TFMA to compute evaluation metrics. Supports mutliclass classification evaluation for tabular, image, video, and text data. \n",
@@ -1392,7 +1392,7 @@
         "- `project`: Project ID.\n",
         "- `location`: Region where the pipeline is run.\n",
         "- `root_dir`: The GCS directory for keeping staging files and artifacts. A random subdirectory is created under the directory to keep job info for resuming the job in case of failure.\n",
-        "- `model_name`: Resource name of the trained AutoML Tabular Classification model.\n",
+        "- `model_name`: Resource name of the trained AutoML Text Classification model.\n",
         "- `target_column_name`: Name of the column to be used as the target for classification.\n",
         "- `batch_predict_gcs_source_uris`: List of the Cloud Storage bucket uris of input instances for batch prediction.\n",
         "- `batch_predict_instances_format`: Format of the input instances for batch prediction. Format used here is'**jsonl**'.\n",
@@ -1459,7 +1459,7 @@
         "    enable_caching=False,\n",
         ")\n",
         "\n",
-        "evaluation_job.run(service_account=SERVICE_ACCOUNT)"
+        "evaluation_job.run(service_account=SERVICE_ACCOUNT, sync=True)"
       ]
     },
     {
@@ -1591,7 +1591,7 @@
         "model_evaluation_id = model_evaluation[\"resourceUri\"].split(\"/\")[-1]\n",
         "print(model_evaluation_id)\n",
         "\n",
-        "evaluation = model.get_model_evaluation(evaluation_id=model_evaluation_id)\n",
+        "evaluation = model.get_model_evaluation()  # evaluation_id=model_evaluation_id)\n",
         "evaluation = evaluation.to_dict()\n",
         "print(\"Model's evaluation metrics from Training:\\n\")\n",
         "metrics = evaluation[\"metrics\"]\n",