mirror of
https://github.com/GoogleCloudPlatform/vertex-ai-samples.git
synced 2026-09-26 14:42:04 +00:00
103 lines
3.5 KiB
Python
103 lines
3.5 KiB
Python
#!/usr/bin/env python
|
|
# Copyright 2021 Google LLC
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
"""Methods to run a notebook locally"""
|
|
|
|
import errno
|
|
import os
|
|
import shutil
|
|
import sys
|
|
|
|
import papermill as pm
|
|
from google.cloud.aiplatform import utils
|
|
from utils import util
|
|
|
|
# This script is used to execute a notebook and write out the output notebook.
|
|
|
|
# This is used to force papermill to use this kernel to run the notebook instead of any defined inside the notebook itself
|
|
DEFAULT_KERNEL_NAME = "python3"
|
|
|
|
|
|
def execute_notebook(
|
|
notebook_source: str,
|
|
output_file_or_uri: str,
|
|
should_log_output: bool,
|
|
):
|
|
"""Execute a single notebook using Papermill"""
|
|
file_name = os.path.basename(os.path.normpath(notebook_source))
|
|
|
|
# Download notebook if it's a GCS URI
|
|
if notebook_source.startswith("gs://"):
|
|
# Extract uri components
|
|
bucket_name, prefix = utils.extract_bucket_and_prefix_from_gcs_path(
|
|
notebook_source
|
|
)
|
|
|
|
# Download remote notebook to local file system
|
|
notebook_source = file_name
|
|
util.download_file(
|
|
bucket_name=bucket_name, blob_name=prefix, destination_file=notebook_source
|
|
)
|
|
|
|
execution_exception = None
|
|
|
|
print("\n=== DOWNLOAD EXECUTED NOTEBOOK ===\n")
|
|
print(f"Please debug the executed notebook by downloading the executed notebook:")
|
|
|
|
print("Option 1. Using gsutil. Run the following command in your terminal.")
|
|
print(f'\tgsutil cp "{output_file_or_uri}" .')
|
|
|
|
print("Option 2. Using this link.")
|
|
print(f"\thttps://storage.googleapis.com/{output_file_or_uri[5:]}")
|
|
|
|
print("\n======\n")
|
|
|
|
# Execute notebook
|
|
try:
|
|
# Execute notebook
|
|
pm.execute_notebook(
|
|
input_path=notebook_source,
|
|
output_path=notebook_source,
|
|
progress_bar=should_log_output,
|
|
request_save_on_cell_execute=should_log_output,
|
|
kernel_name=DEFAULT_KERNEL_NAME,
|
|
log_output=should_log_output,
|
|
stdout_file=sys.stdout if should_log_output else None,
|
|
stderr_file=sys.stderr if should_log_output else None,
|
|
)
|
|
except Exception as exception:
|
|
execution_exception = exception
|
|
finally:
|
|
# Copy executed notebook
|
|
if output_file_or_uri.startswith("gs://"):
|
|
# Upload to GCS path
|
|
util.upload_file(notebook_source, remote_file_path=output_file_or_uri)
|
|
|
|
print("\n=== EXECUTION FINISHED ===\n")
|
|
else:
|
|
# Create directories if they don't exist
|
|
if not os.path.exists(os.path.dirname(output_file_or_uri)):
|
|
try:
|
|
os.makedirs(os.path.dirname(output_file_or_uri))
|
|
except OSError as exc: # Guard against race condition
|
|
if exc.errno != errno.EEXIST:
|
|
raise
|
|
|
|
print(f"Writing output to: {output_file_or_uri}")
|
|
shutil.move(notebook_source, output_file_or_uri)
|
|
|
|
if execution_exception:
|
|
raise execution_exception
|