xiaohei/model_garden
1
0
Fork 0
mirror of https://github.com/GoogleCloudPlatform/vertex-ai-samples.git synced 2026-09-26 14:42:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Create a function to get notebook python version for execution (#828)

Browse Source 
* Create a function to get notebook python version for execution

* Inject python version to yaml file

* Fix python version references

* Add python string to python version variable

* add python version extraction script (untested)

* Create a function to get notebook python version for execution

* Inject python version to yaml file

* Fix python version references

* Add python string to python version variable

* Remove one notebook condition

* Remove extra check and use python 3 as default version

* Use python3.9 as default version

* Update python version notebook parser

* Add python version to the notebook template

* Fix bug

* Update python version parser function

* Add python version to a notebook for testing

* Run linter

* Use regex in python version parser function

* Add new notebook for testing

* Use better variable name

* fix typo

* Use f string instead +

* Use python from env instead of using _PYTHON_VERSION

* Use simpler regex

* Add python version test notebook

* Fixed a mistake

* Updated notebook template with python version

* Fixed python version format

* Print log contents to stdout

* Remove failing notebook

* Run linter

* Remove extra steps in the printed log

* Edit comments

* Run linter

* Run linter

* Revert test changes

Co-authored-by: AG Sol <aarongabriel@google.com>
This commit is contained in:
Soheila Zangeneh
2022-09-26 14:32:53 -07:00
committed by GitHub
co-authored by AG Sol
parent bdad774100
commit f184411666
9 changed files with 196 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.cloud-build/execute_changed_notebooks_helper.py
+66 -21
View File
@@ -17,13 +17,16 @@ import concurrent
import dataclasses
import datetime
import functools
import json
import git
import operator
import os
import pathlib
import re
import subprocess
import utils
from typing import List, Optional
from utils import util
import execute_notebook_helper
import execute_notebook_remote
@@ -35,6 +38,7 @@ from utils import NotebookProcessors, util
# A buffer so that workers finish before the orchestrating job
WORKER_TIMEOUT_BUFFER_IN_SECONDS: int = 60 * 60
PYTHON_VERSION = "3.9" # Set default python version
def format_timedelta(delta: datetime.timedelta) -> str:
@@ -66,6 +70,7 @@ class NotebookExecutionResult:
log_url: str
output_uri: str
build_id: str
logs_bucket: str
error_message: Optional[str]
@property
@@ -110,6 +115,33 @@ def _process_notebook(
nbformat.write(nb, new_file)
def _get_notebook_python_version(notebook_path: str) -> str:
"""
Get the python version for running the notebook if it is specified in
the notebook.
"""
python_version = PYTHON_VERSION
# Load the notebook
file = open(notebook_path)
src = file.read()
nb_json = json.loads(src)
#Iterate over the cells in the ipynb
for cell in nb_json['cells']:
if cell['cell_type'] == 'markdown':
markdown = str.join('', cell['source'])
# Look for the python version specification pattern
re_match = re.search('python version = (\d\.\d)', markdown, flags=re.IGNORECASE)
if re_match:
# get the version number
python_version = re_match.group(1)
break
return python_version
def _create_tag(filepath: str) -> str:
tag = os.path.basename(os.path.normpath(filepath))
tag = re.sub("[^0-9a-zA-Z_.-]+", "-", tag)
@@ -160,6 +192,7 @@ def process_and_execute_notebook(
output_uri=notebook_output_uri,
log_url="",
build_id="",
logs_bucket="",
error_message=None,
)
@@ -167,6 +200,10 @@ def process_and_execute_notebook(
time_start = datetime.datetime.now()
operation = None
try:
# Get the python version for running the notebook if specified
notebook_exec_python_version = _get_notebook_python_version(notebook_path=notebook)
print(f"Running notebook with python {notebook_exec_python_version}")
# Pre-process notebook by substituting variable names
_process_notebook(
notebook_path=notebook,
@@ -193,11 +230,13 @@ def process_and_execute_notebook(
private_pool_id=private_pool_id,
private_pool_region=variable_region,
timeout_in_seconds=timeout_in_seconds,
python_version=notebook_exec_python_version
)
operation_metadata = BuildOperationMetadata(mapping=operation.metadata)
result.build_id = operation_metadata.build.id
result.log_url = operation_metadata.build.log_url
result.logs_bucket = operation_metadata.build.logs_bucket
# Block and wait for the result
operation_result = operation.result()
@@ -339,7 +378,7 @@ def process_and_execute_notebooks(
seconds=max(timeout - WORKER_TIMEOUT_BUFFER_IN_SECONDS, 0)
)
if len(notebooks) > 1:
if len(notebooks) >= 1:
notebook_execution_results: List[NotebookExecutionResult] = []
print(f"Found {len(notebooks)} modified notebooks: {notebooks}")
@@ -404,6 +443,7 @@ def process_and_execute_notebooks(
result.log_url,
result.output_uri,
result.output_uri_web,
result.logs_bucket
]
for result in results_sorted
],
@@ -414,10 +454,35 @@ def process_and_execute_notebooks(
"log_url",
"output_uri",
"output_uri_web",
"logs_bucket"
],
)
)
if len(notebooks) == 1:
print("="*100)
print("The notebook execution build log:\n")
print("="*100)
build_id = results_sorted[0].build_id
logs_bucket_name = (results_sorted[0].logs_bucket).removeprefix("gs://")
log_file_name = f"log-{build_id}.txt"
log_contents = util.download_blob_into_memory(
bucket_name=logs_bucket_name,
blob_name=log_file_name,
download_as_text=True
)
# Remove extra steps from the log
match = re.search("starting Step #4", log_contents, flags=re.IGNORECASE)
if match is not None:
match_index = match.span()[0]
print(log_contents[match_index:])
else:
print(log_contents)
print("\n=== END RESULTS===\n")
total_notebook_duration = functools.reduce(
@@ -433,25 +498,5 @@ def process_and_execute_notebooks(
# Raise error if any notebooks failed
if not all([result.is_pass for result in results_sorted]):
raise RuntimeError("Notebook failures detected. See logs for details")
elif len(notebooks) == 1:
notebook = notebooks[0]
# Pre-process notebook by substituting variable names
_process_notebook(
notebook_path=notebook,
variable_project_id=variable_project_id,
variable_region=variable_region,
variable_service_account=variable_service_account,
variable_vpc_network=variable_vpc_network,
)
execute_notebook_helper.execute_notebook(
notebook_source=notebook,
output_file_or_uri="/".join(
[artifacts_bucket, pathlib.Path(notebook).name]
),
should_log_output=True,
)
else:
print("No notebooks modified in this pull request.")
.cloud-build/execute_notebook_remote.py
+5
View File
@@ -40,6 +40,7 @@ def execute_notebook_remote(
private_pool_region: Optional[str],
tag: Optional[str],
timeout_in_seconds: Optional[int] = None,
python_version: Optional[str] = None
) -> operation.Operation:
"""Create and execute a single notebook on Google Cloud Build"""
# Load build steps from YAML
@@ -50,8 +51,12 @@ def execute_notebook_remote(
"_PYTHON_IMAGE": container_uri,
"_NOTEBOOK_GCS_URI": notebook_uri,
"_NOTEBOOK_OUTPUT_GCS_URI": notebook_output_uri,
"_PYTHON_VERSION" : f"python{python_version}"
}
if python_version is not None:
substitutions["_PYTHON_VERSION"] = "python" + python_version
build = cloudbuild_v1.Build()
options: Optional[client_options.ClientOptions] = None
.cloud-build/notebook-execution-test-cloudbuild-single.yaml
+5 -5
View File
@@ -10,21 +10,21 @@ steps:
entrypoint: /bin/sh
args:
- -c
- python3 .cloud-build/CheckPythonVersion.py -q
- ${_PYTHON_VERSION} .cloud-build/CheckPythonVersion.py -q
# Create a virtual environment
- name: ${_PYTHON_IMAGE}
entrypoint: /bin/sh
args:
- -c
- python3 -m venv workspace/env
- ${_PYTHON_VERSION} -m venv workspace/env
# Install Python dependencies
- name: ${_PYTHON_IMAGE}
entrypoint: /bin/sh
args:
- -c
- . workspace/env/bin/activate &&
python3 -m pip -q install -U pip &&
python3 -m pip -q install -U -r .cloud-build/requirements.txt
python -m pip -q install -U pip &&
python -m pip -q install -U -r .cloud-build/requirements.txt
# Install Python dependencies and run testing script
- name: ${_PYTHON_IMAGE}
entrypoint: /bin/sh
@@ -32,7 +32,7 @@ steps:
- -c
- |
. workspace/env/bin/activate &&
python3 .cloud-build/execute_notebook_cli.py --notebook_source "${_NOTEBOOK_GCS_URI}" --output_file_or_uri "${_NOTEBOOK_OUTPUT_GCS_URI}"
python .cloud-build/execute_notebook_cli.py --notebook_source "${_NOTEBOOK_GCS_URI}" --output_file_or_uri "${_NOTEBOOK_OUTPUT_GCS_URI}"
env:
- 'IS_TESTING=1'
timeout: 86400s
.cloud-build/test_notebook_vm.txt
+2 -1
View File
@@ -1,5 +1,6 @@
notebooks/official/vizier/gapic-vizier-multi-objective-optimization.ipynb
notebooks/official/pipelines/lightweight_functions_component_io_kfp.ipynb
notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-locally-trained-models.ipynb
notebooks/official/pipelines/metrics_viz_run_compare_kfp.ipynb
notebooks/official/custom/custom-tabular-bq-managed-dataset.ipynb
notebooks/official/matching_engine/sdk_matching_engine_for_indexing.ipynb
.cloud-build/tests/python_version_test.ipynb
.cloud-build/test_single_notebook_vm.txt
+1 -1
View File
@@ -1 +1 @@
notebooks/official/pipelines/metrics_viz_run_compare_kfp.ipynb
notebooks/official/custom/custom-tabular-bq-managed-dataset.ipynb
.cloud-build/tests/python_version_test.ipynb
+61
View File
@@ -0,0 +1,61 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "57a3d44ed8a8"
},
"source": [
"### Set up your Google Cloud project\n",
"\n",
"**_NOTE_**: This notebook has been tested in the following environment:\n",
"\n",
"* Python version = 3.7\n",
"\n",
"**The following steps are required, regardless of your notebook environment.**\n",
"\n",
"1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
"\n",
"1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
"\n",
"1. [Enable the Vertex AI API and Compute Engine API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component).\n",
"\n",
"1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
"\n",
"1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
"Cloud SDK uses the right project for all the commands in this notebook.\n",
"\n",
"**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "c6516f90311b"
},
"outputs": [],
"source": [
"# test if the right python version is being used\n",
"import sys\n",
"\n",
"actual_python_version = f\"{sys.version_info.major}.{sys.version_info.minor}\"\n",
"print(f\"Runtime python version: {actual_python_version}\")\n",
"\n",
"assert actual_python_version == \"3.7\", \"Wrong python version!\""
]
}
],
"metadata": {
"colab": {
"name": "python_version_test.ipynb",
"toc_visible": true
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
}
},
"nbformat": 4,
"nbformat_minor": 0
}
.cloud-build/utils/util.py
+32 -1
View File
@@ -3,7 +3,7 @@ import subprocess
import tarfile
import uuid
from datetime import datetime
from typing import Optional
from typing import Optional, Union
from google.auth import credentials as auth_credentials
from google.cloud import storage
@@ -58,3 +58,34 @@ def archive_code_and_upload(staging_bucket: str):
print(f"Uploaded source code archive to {source_archived_file_gcs}")
return source_archived_file_gcs
def download_blob_into_memory(
bucket_name: str,
blob_name: str,
download_as_text: Optional[bool]=False
) -> Union[bytes, str]:
"""
Downloads a blob into memory as byte or as text if
download_as_text is set to True.
"""
storage_client = storage.Client()
bucket = storage_client.bucket(bucket_name)
# Construct a client side representation of a blob.
blob = bucket.blob(blob_name)
# Download the blob content
if download_as_text:
contents = blob.download_as_text()
else:
contents = blob.download_as_bytes()
print(
f"Downloaded storage object {blob_name} from bucket {bucket_name}."
)
return contents
notebooks/community/model_monitoring/batch_prediction_model_monitoring.ipynb
+4
View File
@@ -216,6 +216,10 @@
"source": [
"### Set up your Google Cloud project\n",
"\n",
"**_NOTE_**: This notebook has been tested in the following environment:\n",
"\n",
"* Python version = 3.7\n",
"\n",
"**The following steps are required, regardless of your notebook environment.**\n",
"\n",
"1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
notebooks/notebook_template.ipynb
+19 -5
View File
@@ -134,16 +134,30 @@
"### Set up your local development environment\n",
"\n",
"**If you are using Colab or Vertex AI Workbench Notebooks**, your environment already meets\n",
"all the requirements to run this notebook. You can skip this step.\n",
"all the requirements to run this notebook. You can skip this step."
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "24743cf4a1e1"
},
"source": [
"**_NOTE_**: This notebook has been tested in the following environment:\n",
"\n",
"* Python version = 3.9"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "gCuSR8GkAgzl"
},
"source": [
"**Otherwise**, make sure your environment meets this notebook's requirements.\n",
"You need the following:\n",
"\n",
"* The Google Cloud SDK\n",
"* Git\n",
"* Python 3\n",
"* virtualenv\n",
"* Jupyter notebook running in a virtual environment with Python 3\n",
"\n",
"The Google Cloud guide to [Setting up a Python development\n",
"environment](https://cloud.google.com/python/setup) and the [Jupyter\n",