diff --git a/.cloud-build/execute_changed_notebooks_helper.py b/.cloud-build/execute_changed_notebooks_helper.py index 6372eefe8..c977e9dd5 100755 --- a/.cloud-build/execute_changed_notebooks_helper.py +++ b/.cloud-build/execute_changed_notebooks_helper.py @@ -89,11 +89,10 @@ class NotebookExecutionResult: return None -def load_results(results_bucket: str, - results_file: str) -> Dict[str, Any]: - ''' +def load_results(results_bucket: str, results_file: str) -> Dict[str, Any]: + """ Load accumulated notebook test results - ''' + """ print("Loading existing accumulative results ...") accumulative_results = {} @@ -105,10 +104,14 @@ def load_results(results_bucket: str, blobs = client.list_blobs(results_bucket, prefix=build_results_dir) for blob in blobs: time_created = blob.time_created.replace(tzinfo=None) - if (datetime.datetime.now().replace(tzinfo=None) - time_created).total_seconds() > MAX_RESULTS_AGE_SECONDS: + if ( + datetime.datetime.now().replace(tzinfo=None) - time_created + ).total_seconds() > MAX_RESULTS_AGE_SECONDS: continue - content = util.download_blob_into_memory(results_bucket, blob.name, download_as_text=True) + content = util.download_blob_into_memory( + results_bucket, blob.name, download_as_text=True + ) try: build_results = json.loads(content) @@ -116,8 +119,12 @@ def load_results(results_bucket: str, continue # skip corrupted build results files for notebook in build_results: if notebook in accumulative_results: - accumulative_results[notebook]['passed'] += build_results[notebook]['passed'] - accumulative_results[notebook]['failed'] += build_results[notebook]['failed'] + accumulative_results[notebook]["passed"] += build_results[notebook][ + "passed" + ] + accumulative_results[notebook]["failed"] += build_results[notebook][ + "failed" + ] else: accumulative_results[notebook] = build_results[notebook] @@ -128,16 +135,17 @@ def load_results(results_bucket: str, # If there are no accumulative results, an empty dict is returned return accumulative_results -def select_notebook(changed_notebook: str, - accumulative_results: Dict[str, Any], - test_percent: int) -> bool: - ''' + +def select_notebook( + changed_notebook: str, accumulative_results: Dict[str, Any], test_percent: int +) -> bool: + """ Algorithm to randomly select a notebook, but weight the propbability of selected based on past failures - ''' + """ if changed_notebook in accumulative_results: - pass_count = accumulative_results[changed_notebook]['passed'] - fail_count = accumulative_results[changed_notebook]['failed'] + pass_count = accumulative_results[changed_notebook]["passed"] + fail_count = accumulative_results[changed_notebook]["failed"] else: pass_count = 1 fail_count = 0 @@ -151,7 +159,9 @@ def select_notebook(changed_notebook: str, should_test_due_to_random_subset = random.uniform(0, 1) <= (test_percent / 100) if should_test_due_to_failure or should_test_due_to_random_subset: - print(f"Selected: {changed_notebook}, {should_test_due_to_failure}, {should_test_due_to_random_subset}") + print( + f"Selected: {changed_notebook}, {should_test_due_to_failure}, {should_test_due_to_random_subset}" + ) return True else: print(f"Not Selected: {changed_notebook}, pass {pass_count}, fail {fail_count}") @@ -176,7 +186,7 @@ def _process_notebook( "PROJECT_ID": variable_project_id, "REGION": variable_region, "SERVICE_ACCOUNT": variable_service_account, - "VPC_NETWORK": variable_vpc_network, + "VPC_NETWORK": variable_vpc_network or "", }, ) unique_strings_preprocessor = NotebookProcessors.UniqueStringsPreprocessor() @@ -233,8 +243,6 @@ def _create_tag(filepath: str) -> str: return tag - - def process_and_execute_notebook( container_uri: str, staging_bucket: str, @@ -248,7 +256,6 @@ def process_and_execute_notebook( notebook: str, should_get_tail_logs: bool = False, ) -> NotebookExecutionResult: - print(f"Running notebook: {notebook}") # Handle empty strings @@ -413,11 +420,11 @@ def get_changed_notebooks( return notebooks -def _save_results(results: List[NotebookExecutionResult], - artifacts_bucket: str, - results_file: str): - artifacts_bucket = artifacts_bucket.replace("gs://", "").split('/')[0] +def _save_results( + results: List[NotebookExecutionResult], artifacts_bucket: str, results_file: str +): + artifacts_bucket = artifacts_bucket.replace("gs://", "").split("/")[0] print("Updating build results ...") build_results = {} @@ -429,10 +436,10 @@ def _save_results(results: List[NotebookExecutionResult], pass_count = 0 fail_count = 1 build_results[result.path] = { - 'duration': result.duration.total_seconds(), - 'start_time': str(result.start_time), - 'passed': pass_count, - 'failed': fail_count + "duration": result.duration.total_seconds(), + "start_time": str(result.start_time), + "passed": pass_count, + "failed": fail_count, } print(f"adding {result.path}") @@ -441,8 +448,7 @@ def _save_results(results: List[NotebookExecutionResult], client = storage.Client() bucket = client.get_bucket(artifacts_bucket) - bucket.blob(str(results_file)).upload_from_string(content, 'text/json') - + bucket.blob(str(results_file)).upload_from_string(content, "text/json") def process_and_execute_notebooks( @@ -480,7 +486,7 @@ def process_and_execute_notebooks( artifacts_bucket (str): Required. The GCS staging bucket to write executed notebooks to. results_file (str): - Required: The path to the artifacts bucket to save results + Required: The path to the artifacts bucket to save results variable_project_id (str): Required. The value for PROJECT_ID to inject into notebooks. variable_region (str): @@ -507,8 +513,9 @@ def process_and_execute_notebooks( "Running notebooks in parallel, so no logs will be displayed. Please wait..." ) - - with concurrent.futures.ThreadPoolExecutor(max_workers=concurrent_notebooks) as executor: + with concurrent.futures.ThreadPoolExecutor( + max_workers=concurrent_notebooks + ) as executor: print(f"Max workers: {executor._max_workers}") notebook_execution_results = list( @@ -604,9 +611,7 @@ def process_and_execute_notebooks( else: print(log_contents) - _save_results(results_sorted, - artifacts_bucket, - results_file) + _save_results(results_sorted, artifacts_bucket, results_file) print("\n=== END RESULTS===\n") diff --git a/.cloud-build/notebook-execution-test-cloudbuild.yaml b/.cloud-build/notebook-execution-test-cloudbuild.yaml index eb0a1046c..552b77067 100644 --- a/.cloud-build/notebook-execution-test-cloudbuild.yaml +++ b/.cloud-build/notebook-execution-test-cloudbuild.yaml @@ -36,7 +36,7 @@ steps: - -c - | . workspace/env/bin/activate && - python3 .cloud-build/execute_changed_notebooks_cli.py --test_paths_file "${_TEST_PATHS_FILE}" --base_branch "${_FORCED_BASE_BRANCH}" --container_uri ${_PYTHON_IMAGE} --staging_bucket ${_GCS_STAGING_BUCKET} --artifacts_bucket ${_GCS_STAGING_BUCKET}/executed_notebooks/PR_${_PR_NUMBER}/BUILD_${BUILD_ID} --variable_project_id ${PROJECT_ID} --variable_region ${_GCP_REGION} --variable_service_account ${_GCP_SERVICE_ACCOUNT} --variable_vpc_network "${_GPC_VPC_NETWORK_NAME}" `if [ ! -z "${_PRIVATE_POOL_NAME}" ]; then echo "--private_pool_id ${_PRIVATE_POOL_NAME}"; fi` --build_id ${BUILD_ID} --test_percent=${_TEST_PERCENT} --concurrent_notebooks=${_CONCURRENT_NOTEBOOKS} + python3 .cloud-build/execute_changed_notebooks_cli.py --test_paths_file "${_TEST_PATHS_FILE}" --base_branch "${_FORCED_BASE_BRANCH}" --container_uri ${_PYTHON_IMAGE} --staging_bucket ${_GCS_STAGING_BUCKET} --artifacts_bucket ${_GCS_STAGING_BUCKET}/executed_notebooks/PR_${_PR_NUMBER}/BUILD_${BUILD_ID} --variable_project_id ${PROJECT_ID} --variable_region ${_GCP_REGION} --variable_service_account ${_GCP_SERVICE_ACCOUNT} --variable_vpc_network "${_GCP_VPC_NETWORK_NAME}" `if [ ! -z "${_PRIVATE_POOL_NAME}" ]; then echo "--private_pool_id ${_PRIVATE_POOL_NAME}"; fi` --build_id ${BUILD_ID} --test_percent=${_TEST_PERCENT} --concurrent_notebooks=${_CONCURRENT_NOTEBOOKS} env: - 'IS_TESTING=1' timeout: 86400s diff --git a/.cloud-build/utils/NotebookProcessors.py b/.cloud-build/utils/NotebookProcessors.py index dfce82eb8..29d24c723 100644 --- a/.cloud-build/utils/NotebookProcessors.py +++ b/.cloud-build/utils/NotebookProcessors.py @@ -35,7 +35,7 @@ class RemoveNoExecuteCells(Preprocessor): class UpdateVariablesPreprocessor(Preprocessor): - def __init__(self, replacement_map: Dict): + def __init__(self, replacement_map: Dict[str, str]): self._replacement_map = replacement_map @staticmethod diff --git a/notebooks/official/matching_engine/sdk_matching_engine_for_indexing.ipynb b/notebooks/official/matching_engine/sdk_matching_engine_for_indexing.ipynb index 31a4785f7..bd1dc5ade 100644 --- a/notebooks/official/matching_engine/sdk_matching_engine_for_indexing.ipynb +++ b/notebooks/official/matching_engine/sdk_matching_engine_for_indexing.ipynb @@ -24,13 +24,12 @@ ] }, { - "attachments": {}, "cell_type": "markdown", "metadata": { "id": "JAPoU8Sm5E6e" }, "source": [ - "# Create Vertex AI Matching Engine index\n", + "# Create Vertex AI Vector Search index\n", "![ ](https://www.google-analytics.com/collect?v=2&tid=G-L6X3ECH596&cid=1&en=page_view&sid=1&dt=sdk_matching_engine_for_indexing.ipynb&dl=notebooks%2Fofficial%2Fmatching_engine%2Fsdk_matching_engine_for_indexing.ipynb)\n", "\n", "
\n", @@ -63,7 +62,7 @@ "\n", "This example demonstrates how to use the Vertex AI ANN Service. It is a high scale, low latency solution, to find similar vectors (or more specifically \"embeddings\") for a large corpus. Moreover, it is a fully managed offering, further reducing operational overhead. It is built upon [Approximate Nearest Neighbor (ANN) technology](https://ai.googleblog.com/2020/07/announcing-scann-efficient-vector.html) developed by Google Research.\n", "\n", - "Learn more about [Vertex AI Matching Engine](https://cloud.google.com/vertex-ai/docs/matching-engine/overview)." + "Learn more about [Vertex AI Vector Search](https://cloud.google.com/vertex-ai/docs/matching-engine/overview)." ] }, { @@ -78,7 +77,7 @@ "\n", "This tutorial uses the following Google Cloud ML services:\n", "\n", - "- `Vertex AI Matching Engine`\n", + "- `Vertex AI Vector Search`\n", "\n", "The steps performed include:\n", "\n", @@ -110,7 +109,7 @@ "source": [ "## Installation\n", "\n", - "Install the latest version of Cloud Storage, BigQuery and Vertex AI SDKs for Python." + "Install the latest versions of packages required to execute this notebook." ] }, { @@ -123,7 +122,9 @@ "source": [ "# Install the packages\n", "! pip3 install --upgrade google-cloud-aiplatform \\\n", - " google-cloud-storage" + " google-cloud-storage \\\n", + " grpcio-tools \\\n", + " h5py" ] }, { @@ -353,37 +354,6 @@ " * If you run it in the colab or a Vertex AI Workbench notebook instance in a different VPC network or region, \"Create Online Queries\" section will fail." ] }, - { - "cell_type": "markdown", - "metadata": { - "id": "i7EUnXsZhAGF" - }, - "source": [ - "### Installation\n", - "\n", - "Download and install the latest version of the Vertex SDK for Python." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "wyy5Lbnzg5fi" - }, - "outputs": [], - "source": [ - "! pip install --upgrade --quiet google-cloud-aiplatform grpcio-tools h5py" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "irSMQn6gZ19l" - }, - "source": [ - "Install the `h5py` to prepare sample dataset, and the `grpcio-tools` for querying against the index. " - ] - }, { "cell_type": "markdown", "metadata": { @@ -427,7 +397,7 @@ }, "outputs": [], "source": [ - "BUCKET_URI = \"gs://your-bucket-name-unique\" # @param {type:\"string\"}" + "BUCKET_URI = f\"gs://your-bucket-name-{PROJECT_ID}-unique\" # @param {type:\"string\"}" ] }, { @@ -447,7 +417,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI" + "! gsutil mb -l {REGION} -p {PROJECT_ID} {BUCKET_URI}" ] }, { @@ -1097,21 +1067,19 @@ }, "outputs": [], "source": [ + "import os\n", + "\n", + "delete_bucket = False\n", + "\n", "# Force undeployment of indexes and delete endpoint\n", - "my_index_endpoint.delete(force=True)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "omj7N9iWv-Tq" - }, - "outputs": [], - "source": [ + "my_index_endpoint.delete(force=True)\n", + "\n", "# Delete indexes\n", "tree_ah_index.delete()\n", - "brute_force_index.delete()" + "brute_force_index.delete()\n", + "\n", + "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", + " ! gsutil rm -rf {BUCKET_URI}" ] } ],