From f184411666c354460457780d5ef422d5cff6be01 Mon Sep 17 00:00:00 2001 From: Soheila Zangeneh <49654056+soheilazangeneh@users.noreply.github.com> Date: Mon, 26 Sep 2022 17:32:53 -0400 Subject: [PATCH] Create a function to get notebook python version for execution (#828) * Create a function to get notebook python version for execution * Inject python version to yaml file * Fix python version references * Add python string to python version variable * add python version extraction script (untested) * Create a function to get notebook python version for execution * Inject python version to yaml file * Fix python version references * Add python string to python version variable * Remove one notebook condition * Remove extra check and use python 3 as default version * Use python3.9 as default version * Update python version notebook parser * Add python version to the notebook template * Fix bug * Update python version parser function * Add python version to a notebook for testing * Run linter * Use regex in python version parser function * Add new notebook for testing * Use better variable name * fix typo * Use f string instead + * Use python from env instead of using _PYTHON_VERSION * Use simpler regex * Add python version test notebook * Fixed a mistake * Updated notebook template with python version * Fixed python version format * Print log contents to stdout * Remove failing notebook * Run linter * Remove extra steps in the printed log * Edit comments * Run linter * Run linter * Revert test changes Co-authored-by: AG Sol --- .../execute_changed_notebooks_helper.py | 87 ++++++++++++++----- .cloud-build/execute_notebook_remote.py | 5 ++ ...book-execution-test-cloudbuild-single.yaml | 10 +-- .cloud-build/test_notebook_vm.txt | 5 +- .cloud-build/test_single_notebook_vm.txt | 2 +- .cloud-build/tests/python_version_test.ipynb | 61 +++++++++++++ .cloud-build/utils/util.py | 33 ++++++- .../batch_prediction_model_monitoring.ipynb | 4 + notebooks/notebook_template.ipynb | 24 +++-- 9 files changed, 196 insertions(+), 35 deletions(-) create mode 100644 .cloud-build/tests/python_version_test.ipynb diff --git a/.cloud-build/execute_changed_notebooks_helper.py b/.cloud-build/execute_changed_notebooks_helper.py index e5830a15c..9926bec2e 100755 --- a/.cloud-build/execute_changed_notebooks_helper.py +++ b/.cloud-build/execute_changed_notebooks_helper.py @@ -17,13 +17,16 @@ import concurrent import dataclasses import datetime import functools +import json import git import operator import os import pathlib import re import subprocess +import utils from typing import List, Optional +from utils import util import execute_notebook_helper import execute_notebook_remote @@ -35,6 +38,7 @@ from utils import NotebookProcessors, util # A buffer so that workers finish before the orchestrating job WORKER_TIMEOUT_BUFFER_IN_SECONDS: int = 60 * 60 +PYTHON_VERSION = "3.9" # Set default python version def format_timedelta(delta: datetime.timedelta) -> str: @@ -66,6 +70,7 @@ class NotebookExecutionResult: log_url: str output_uri: str build_id: str + logs_bucket: str error_message: Optional[str] @property @@ -110,6 +115,33 @@ def _process_notebook( nbformat.write(nb, new_file) +def _get_notebook_python_version(notebook_path: str) -> str: + """ + Get the python version for running the notebook if it is specified in + the notebook. + """ + python_version = PYTHON_VERSION + + # Load the notebook + file = open(notebook_path) + src = file.read() + nb_json = json.loads(src) + + #Iterate over the cells in the ipynb + for cell in nb_json['cells']: + if cell['cell_type'] == 'markdown': + markdown = str.join('', cell['source']) + + # Look for the python version specification pattern + re_match = re.search('python version = (\d\.\d)', markdown, flags=re.IGNORECASE) + if re_match: + # get the version number + python_version = re_match.group(1) + break + + return python_version + + def _create_tag(filepath: str) -> str: tag = os.path.basename(os.path.normpath(filepath)) tag = re.sub("[^0-9a-zA-Z_.-]+", "-", tag) @@ -160,6 +192,7 @@ def process_and_execute_notebook( output_uri=notebook_output_uri, log_url="", build_id="", + logs_bucket="", error_message=None, ) @@ -167,6 +200,10 @@ def process_and_execute_notebook( time_start = datetime.datetime.now() operation = None try: + # Get the python version for running the notebook if specified + notebook_exec_python_version = _get_notebook_python_version(notebook_path=notebook) + print(f"Running notebook with python {notebook_exec_python_version}") + # Pre-process notebook by substituting variable names _process_notebook( notebook_path=notebook, @@ -193,11 +230,13 @@ def process_and_execute_notebook( private_pool_id=private_pool_id, private_pool_region=variable_region, timeout_in_seconds=timeout_in_seconds, + python_version=notebook_exec_python_version ) operation_metadata = BuildOperationMetadata(mapping=operation.metadata) result.build_id = operation_metadata.build.id result.log_url = operation_metadata.build.log_url + result.logs_bucket = operation_metadata.build.logs_bucket # Block and wait for the result operation_result = operation.result() @@ -339,7 +378,7 @@ def process_and_execute_notebooks( seconds=max(timeout - WORKER_TIMEOUT_BUFFER_IN_SECONDS, 0) ) - if len(notebooks) > 1: + if len(notebooks) >= 1: notebook_execution_results: List[NotebookExecutionResult] = [] print(f"Found {len(notebooks)} modified notebooks: {notebooks}") @@ -404,6 +443,7 @@ def process_and_execute_notebooks( result.log_url, result.output_uri, result.output_uri_web, + result.logs_bucket ] for result in results_sorted ], @@ -414,10 +454,35 @@ def process_and_execute_notebooks( "log_url", "output_uri", "output_uri_web", + "logs_bucket" ], ) ) + if len(notebooks) == 1: + print("="*100) + print("The notebook execution build log:\n") + print("="*100) + + build_id = results_sorted[0].build_id + logs_bucket_name = (results_sorted[0].logs_bucket).removeprefix("gs://") + log_file_name = f"log-{build_id}.txt" + + log_contents = util.download_blob_into_memory( + bucket_name=logs_bucket_name, + blob_name=log_file_name, + download_as_text=True + ) + + # Remove extra steps from the log + match = re.search("starting Step #4", log_contents, flags=re.IGNORECASE) + + if match is not None: + match_index = match.span()[0] + print(log_contents[match_index:]) + else: + print(log_contents) + print("\n=== END RESULTS===\n") total_notebook_duration = functools.reduce( @@ -433,25 +498,5 @@ def process_and_execute_notebooks( # Raise error if any notebooks failed if not all([result.is_pass for result in results_sorted]): raise RuntimeError("Notebook failures detected. See logs for details") - - elif len(notebooks) == 1: - notebook = notebooks[0] - - # Pre-process notebook by substituting variable names - _process_notebook( - notebook_path=notebook, - variable_project_id=variable_project_id, - variable_region=variable_region, - variable_service_account=variable_service_account, - variable_vpc_network=variable_vpc_network, - ) - - execute_notebook_helper.execute_notebook( - notebook_source=notebook, - output_file_or_uri="/".join( - [artifacts_bucket, pathlib.Path(notebook).name] - ), - should_log_output=True, - ) else: print("No notebooks modified in this pull request.") diff --git a/.cloud-build/execute_notebook_remote.py b/.cloud-build/execute_notebook_remote.py index f15b01605..49df88a67 100644 --- a/.cloud-build/execute_notebook_remote.py +++ b/.cloud-build/execute_notebook_remote.py @@ -40,6 +40,7 @@ def execute_notebook_remote( private_pool_region: Optional[str], tag: Optional[str], timeout_in_seconds: Optional[int] = None, + python_version: Optional[str] = None ) -> operation.Operation: """Create and execute a single notebook on Google Cloud Build""" # Load build steps from YAML @@ -50,8 +51,12 @@ def execute_notebook_remote( "_PYTHON_IMAGE": container_uri, "_NOTEBOOK_GCS_URI": notebook_uri, "_NOTEBOOK_OUTPUT_GCS_URI": notebook_output_uri, + "_PYTHON_VERSION" : f"python{python_version}" } + if python_version is not None: + substitutions["_PYTHON_VERSION"] = "python" + python_version + build = cloudbuild_v1.Build() options: Optional[client_options.ClientOptions] = None diff --git a/.cloud-build/notebook-execution-test-cloudbuild-single.yaml b/.cloud-build/notebook-execution-test-cloudbuild-single.yaml index 8ed633247..4ff74ec8b 100644 --- a/.cloud-build/notebook-execution-test-cloudbuild-single.yaml +++ b/.cloud-build/notebook-execution-test-cloudbuild-single.yaml @@ -10,21 +10,21 @@ steps: entrypoint: /bin/sh args: - -c - - python3 .cloud-build/CheckPythonVersion.py -q + - ${_PYTHON_VERSION} .cloud-build/CheckPythonVersion.py -q # Create a virtual environment - name: ${_PYTHON_IMAGE} entrypoint: /bin/sh args: - -c - - python3 -m venv workspace/env + - ${_PYTHON_VERSION} -m venv workspace/env # Install Python dependencies - name: ${_PYTHON_IMAGE} entrypoint: /bin/sh args: - -c - . workspace/env/bin/activate && - python3 -m pip -q install -U pip && - python3 -m pip -q install -U -r .cloud-build/requirements.txt + python -m pip -q install -U pip && + python -m pip -q install -U -r .cloud-build/requirements.txt # Install Python dependencies and run testing script - name: ${_PYTHON_IMAGE} entrypoint: /bin/sh @@ -32,7 +32,7 @@ steps: - -c - | . workspace/env/bin/activate && - python3 .cloud-build/execute_notebook_cli.py --notebook_source "${_NOTEBOOK_GCS_URI}" --output_file_or_uri "${_NOTEBOOK_OUTPUT_GCS_URI}" + python .cloud-build/execute_notebook_cli.py --notebook_source "${_NOTEBOOK_GCS_URI}" --output_file_or_uri "${_NOTEBOOK_OUTPUT_GCS_URI}" env: - 'IS_TESTING=1' timeout: 86400s diff --git a/.cloud-build/test_notebook_vm.txt b/.cloud-build/test_notebook_vm.txt index 92094ddc0..b6124c7a2 100644 --- a/.cloud-build/test_notebook_vm.txt +++ b/.cloud-build/test_notebook_vm.txt @@ -1,5 +1,6 @@ notebooks/official/vizier/gapic-vizier-multi-objective-optimization.ipynb notebooks/official/pipelines/lightweight_functions_component_io_kfp.ipynb notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-locally-trained-models.ipynb -notebooks/official/pipelines/metrics_viz_run_compare_kfp.ipynb -notebooks/official/matching_engine/sdk_matching_engine_for_indexing.ipynb \ No newline at end of file +notebooks/official/custom/custom-tabular-bq-managed-dataset.ipynb +notebooks/official/matching_engine/sdk_matching_engine_for_indexing.ipynb +.cloud-build/tests/python_version_test.ipynb diff --git a/.cloud-build/test_single_notebook_vm.txt b/.cloud-build/test_single_notebook_vm.txt index bf39fba80..8d9a5dbf3 100644 --- a/.cloud-build/test_single_notebook_vm.txt +++ b/.cloud-build/test_single_notebook_vm.txt @@ -1 +1 @@ -notebooks/official/pipelines/metrics_viz_run_compare_kfp.ipynb +notebooks/official/custom/custom-tabular-bq-managed-dataset.ipynb diff --git a/.cloud-build/tests/python_version_test.ipynb b/.cloud-build/tests/python_version_test.ipynb new file mode 100644 index 000000000..cfd5bcfbe --- /dev/null +++ b/.cloud-build/tests/python_version_test.ipynb @@ -0,0 +1,61 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "57a3d44ed8a8" + }, + "source": [ + "### Set up your Google Cloud project\n", + "\n", + "**_NOTE_**: This notebook has been tested in the following environment:\n", + "\n", + "* Python version = 3.7\n", + "\n", + "**The following steps are required, regardless of your notebook environment.**\n", + "\n", + "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n", + "\n", + "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n", + "\n", + "1. [Enable the Vertex AI API and Compute Engine API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component).\n", + "\n", + "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n", + "\n", + "1. Enter your project ID in the cell below. Then run the cell to make sure the\n", + "Cloud SDK uses the right project for all the commands in this notebook.\n", + "\n", + "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "c6516f90311b" + }, + "outputs": [], + "source": [ + "# test if the right python version is being used\n", + "import sys\n", + "\n", + "actual_python_version = f\"{sys.version_info.major}.{sys.version_info.minor}\"\n", + "print(f\"Runtime python version: {actual_python_version}\")\n", + "\n", + "assert actual_python_version == \"3.7\", \"Wrong python version!\"" + ] + } + ], + "metadata": { + "colab": { + "name": "python_version_test.ipynb", + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/.cloud-build/utils/util.py b/.cloud-build/utils/util.py index 86eadece6..010165f04 100644 --- a/.cloud-build/utils/util.py +++ b/.cloud-build/utils/util.py @@ -3,7 +3,7 @@ import subprocess import tarfile import uuid from datetime import datetime -from typing import Optional +from typing import Optional, Union from google.auth import credentials as auth_credentials from google.cloud import storage @@ -58,3 +58,34 @@ def archive_code_and_upload(staging_bucket: str): print(f"Uploaded source code archive to {source_archived_file_gcs}") return source_archived_file_gcs + + +def download_blob_into_memory( + bucket_name: str, + blob_name: str, + download_as_text: Optional[bool]=False +) -> Union[bytes, str]: + """ + Downloads a blob into memory as byte or as text if + download_as_text is set to True. + """ + + storage_client = storage.Client() + + bucket = storage_client.bucket(bucket_name) + + # Construct a client side representation of a blob. + blob = bucket.blob(blob_name) + + # Download the blob content + if download_as_text: + contents = blob.download_as_text() + else: + contents = blob.download_as_bytes() + + print( + f"Downloaded storage object {blob_name} from bucket {bucket_name}." + ) + + return contents + diff --git a/notebooks/community/model_monitoring/batch_prediction_model_monitoring.ipynb b/notebooks/community/model_monitoring/batch_prediction_model_monitoring.ipynb index 3d3c16657..57408a671 100644 --- a/notebooks/community/model_monitoring/batch_prediction_model_monitoring.ipynb +++ b/notebooks/community/model_monitoring/batch_prediction_model_monitoring.ipynb @@ -216,6 +216,10 @@ "source": [ "### Set up your Google Cloud project\n", "\n", + "**_NOTE_**: This notebook has been tested in the following environment:\n", + "\n", + "* Python version = 3.7\n", + "\n", "**The following steps are required, regardless of your notebook environment.**\n", "\n", "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n", diff --git a/notebooks/notebook_template.ipynb b/notebooks/notebook_template.ipynb index fd1259595..8f1033e6f 100644 --- a/notebooks/notebook_template.ipynb +++ b/notebooks/notebook_template.ipynb @@ -134,16 +134,30 @@ "### Set up your local development environment\n", "\n", "**If you are using Colab or Vertex AI Workbench Notebooks**, your environment already meets\n", - "all the requirements to run this notebook. You can skip this step.\n", + "all the requirements to run this notebook. You can skip this step." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "24743cf4a1e1" + }, + "source": [ + "**_NOTE_**: This notebook has been tested in the following environment:\n", "\n", + "* Python version = 3.9" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "gCuSR8GkAgzl" + }, + "source": [ "**Otherwise**, make sure your environment meets this notebook's requirements.\n", "You need the following:\n", "\n", "* The Google Cloud SDK\n", - "* Git\n", - "* Python 3\n", - "* virtualenv\n", - "* Jupyter notebook running in a virtual environment with Python 3\n", "\n", "The Google Cloud guide to [Setting up a Python development\n", "environment](https://cloud.google.com/python/setup) and the [Jupyter\n",