debug: set explicit timeout on operation.result()

.cloud-build/cleanup/cleanup.py

@@ -5,8 +5,6 @@ from resource_cleanup_manager import (
     ModelResourceCleanupManager,
     EndpointResourceCleanupManager,
     ResourceCleanupManager,
-    MatchingEngineIndexEndpointResourceCleanupManager,
-    MatchingEngineIndexResourceCleanupManager,
 )
 
 rate_limit = RateLimit(max_count=25, per=60, greedy=False)
@@ -42,12 +40,10 @@ if is_dry_run:
     print("Starting cleanup in dry run mode...")
 
 # List of all cleanup managers
-managers: List[ResourceCleanupManager] = [
+managers = [
     DatasetResourceCleanupManager(),
     EndpointResourceCleanupManager(),
     ModelResourceCleanupManager(),  # ModelResourceCleanupManager must follow EndpointResourceCleanupManager due to deployed models blocking model deletion.
-    MatchingEngineIndexEndpointResourceCleanupManager(),
-    MatchingEngineIndexResourceCleanupManager(),
 ]
 
 run_cleanup_managers(managers=managers, is_dry_run=is_dry_run)

.cloud-build/cleanup/resource_cleanup_manager.py

@@ -110,10 +110,3 @@ class EndpointResourceCleanupManager(VertexAIResourceCleanupManager):
 class ModelResourceCleanupManager(VertexAIResourceCleanupManager):
     vertex_ai_resource = aiplatform.Model
 
-
-class MatchingEngineIndexResourceCleanupManager(VertexAIResourceCleanupManager):
-    vertex_ai_resource = aiplatform.MatchingEngineIndex
-
-
-class MatchingEngineIndexEndpointResourceCleanupManager(VertexAIResourceCleanupManager):
-    vertex_ai_resource = aiplatform.MatchingEngineIndexEndpoint

.cloud-build/execute_changed_notebooks_helper.py

@@ -245,13 +245,15 @@ def process_and_execute_notebook(
         result.logs_bucket = operation_metadata.build.logs_bucket
 
         # Block and wait for the result
-        operation_result = operation.result()
+        operation_result = operation.result(timeout=84600)
 
         result.duration = datetime.datetime.now() - time_start
         result.is_pass = True
         print(f"{notebook} PASSED in {format_timedelta(result.duration)}.")
     except Exception as error:
         result.error_message = str(error)
+        import traceback
+        traceback.print_exc()
 
         if operation and should_get_tail_logs:
             # Extract the logs

.cloud-build/execute_notebook_helper.py

@@ -66,6 +66,7 @@ def execute_notebook(
 
     # Execute notebook
     try:
+        print("DEBUG HERE\n")
         # Execute notebook
         pm.execute_notebook(
             input_path=notebook_source,

.cloud-build/execute_notebook_remote.py

@@ -45,6 +45,9 @@ def execute_notebook_remote(
     """Create and execute a single notebook on Google Cloud Build"""
     # Load build steps from YAML
 
+    print(f"DEBUG TIMEOUT {timeout_in_seconds}\n")
+    
+
     cloudbuild_config = yaml.load(open(CLOUD_BUILD_FILEPATH), Loader=FullLoader)
 
     substitutions = {
@@ -95,7 +98,14 @@ def execute_notebook_remote(
     if tag:
         build.tags = [tag]
 
-    operation = client.create_build(project_id=project_id, build=build)
+    try:
+        print("DEBUG: START\n")
+        operation = client.create_build(project_id=project_id, build=build)
+    except Exception as e:
+        import traceback
+        traceback.print_exc()
+    print("DEBUG: FINISH\n")
+    print(operation)
     # Print the in-progress operation
     # print("IN PROGRESS:")
     # print(operation.metadata)

.cloud-build/notebook-execution-test-cloudbuild.yaml

@@ -36,7 +36,7 @@ steps:
     - -c
     - |
       . workspace/env/bin/activate &&
-      python3 .cloud-build/execute_changed_notebooks_cli.py --test_paths_file "${_TEST_PATHS_FILE}" --base_branch "${_FORCED_BASE_BRANCH}" --container_uri ${_PYTHON_IMAGE} --staging_bucket ${_GCS_STAGING_BUCKET}  --artifacts_bucket ${_GCS_STAGING_BUCKET}/executed_notebooks/PR_${_PR_NUMBER}/BUILD_${BUILD_ID} --variable_project_id ${PROJECT_ID} --variable_region ${_GCP_REGION} --variable_service_account ${_GCP_SERVICE_ACCOUNT} --variable_vpc_network "${_GPC_VPC_NETWORK_NAME}" `if [ ! -z "${_PRIVATE_POOL_NAME}" ]; then echo "--private_pool_id ${_PRIVATE_POOL_NAME}"; fi`
+      python3 .cloud-build/execute_changed_notebooks_cli.py --test_paths_file "${_TEST_PATHS_FILE}" --base_branch "${_FORCED_BASE_BRANCH}" --container_uri ${_PYTHON_IMAGE} --staging_bucket ${_GCS_STAGING_BUCKET}  --artifacts_bucket ${_GCS_STAGING_BUCKET}/executed_notebooks/PR_${_PR_NUMBER}/BUILD_${BUILD_ID} --variable_project_id ${PROJECT_ID} --variable_region ${_GCP_REGION} --variable_service_account ${_GCP_SERVICE_ACCOUNT} --variable_vpc_network "${_GPC_VPC_NETWORK_NAME}" --timeout 86400 `if [ ! -z "${_PRIVATE_POOL_NAME}" ]; then echo "--private_pool_id ${_PRIVATE_POOL_NAME}"; fi`
     env:
     - 'IS_TESTING=1'
 timeout: 86400s

.cloud-build/requirements.txt

@@ -8,6 +8,7 @@ matplotlib
 tabulate
 google-cloud-aiplatform
 google-cloud-storage
-google-cloud-build
+google-cloud-build==3.9.3
+protobuf==4.21.9
 ratemate
-GitPython
+GitPython

notebooks/official/test.ipynb

@@ -0,0 +1,113 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "id": "82b522b9e130"
+   },
+   "outputs": [],
+   "source": [
+    "import time"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "id": "7dc682542238"
+   },
+   "outputs": [],
+   "source": [
+    "# 5 mins\n",
+    "time.sleep(5 * 60)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "id": "3ab331234b42"
+   },
+   "outputs": [],
+   "source": [
+    "print(\"5 minutes\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "id": "2a5ea0ae33f0"
+   },
+   "outputs": [],
+   "source": [
+    "time.sleep(10 * 60)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "id": "ab9a3894d60e"
+   },
+   "outputs": [],
+   "source": [
+    "print(\"15 minutes\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "id": "d9efa8e9b70a"
+   },
+   "outputs": [],
+   "source": [
+    "time.sleep(5 * 60)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "id": "bc63b3db2e71"
+   },
+   "outputs": [],
+   "source": [
+    "print(\"20 minutes\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "colab": {
+   "name": "test.ipynb",
+   "toc_visible": true
+  },
+  "environment": {
+   "kernel": "python3",
+   "name": "common-cpu.m95",
+   "type": "gcloud",
+   "uri": "gcr.io/deeplearning-platform-release/base-cpu:m95"
+  },
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}