mirror of
https://github.com/GoogleCloudPlatform/vertex-ai-samples.git
synced 2026-09-26 14:42:04 +00:00
Boilerplate reduction: Notebook template (#1204)
* Reduced notebook boilerplate * Fixed lint issues * Added unique suffix note * Added unique string processor and moved tests to own folder * Fixed broken link * Added message about updating links * Fixed typo * Added missing import * Added back useful instructions * Addressed comments * Removed matching engine * Fixed comments
This commit is contained in:
Ivan Cheung
@@ -38,7 +38,7 @@ from utils import NotebookProcessors, util
|
||||
|
||||
# A buffer so that workers finish before the orchestrating job
|
||||
WORKER_TIMEOUT_BUFFER_IN_SECONDS: int = 60 * 60
|
||||
PYTHON_VERSION = "3.9" # Set default python version
|
||||
PYTHON_VERSION = "3.9" # Set default python version
|
||||
|
||||
|
||||
def format_timedelta(delta: datetime.timedelta) -> str:
|
||||
@@ -102,6 +102,7 @@ def _process_notebook(
|
||||
"VPC_NETWORK": variable_vpc_network,
|
||||
},
|
||||
)
|
||||
unique_strings_preprocessor = NotebookProcessors.UniqueStringsPreprocessor()
|
||||
|
||||
# Use no-execute preprocessor
|
||||
(
|
||||
@@ -127,13 +128,15 @@ def _get_notebook_python_version(notebook_path: str) -> str:
|
||||
src = file.read()
|
||||
nb_json = json.loads(src)
|
||||
|
||||
#Iterate over the cells in the ipynb
|
||||
for cell in nb_json['cells']:
|
||||
if cell['cell_type'] == 'markdown':
|
||||
markdown = str.join('', cell['source'])
|
||||
# Iterate over the cells in the ipynb
|
||||
for cell in nb_json["cells"]:
|
||||
if cell["cell_type"] == "markdown":
|
||||
markdown = str.join("", cell["source"])
|
||||
|
||||
# Look for the python version specification pattern
|
||||
re_match = re.search('python version = (\d\.\d)', markdown, flags=re.IGNORECASE)
|
||||
re_match = re.search(
|
||||
"python version = (\d\.\d)", markdown, flags=re.IGNORECASE
|
||||
)
|
||||
if re_match:
|
||||
# get the version number
|
||||
python_version = re_match.group(1)
|
||||
@@ -201,7 +204,9 @@ def process_and_execute_notebook(
|
||||
operation = None
|
||||
try:
|
||||
# Get the python version for running the notebook if specified
|
||||
notebook_exec_python_version = _get_notebook_python_version(notebook_path=notebook)
|
||||
notebook_exec_python_version = _get_notebook_python_version(
|
||||
notebook_path=notebook
|
||||
)
|
||||
print(f"Running notebook with python {notebook_exec_python_version}")
|
||||
|
||||
# Pre-process notebook by substituting variable names
|
||||
@@ -230,7 +235,7 @@ def process_and_execute_notebook(
|
||||
private_pool_id=private_pool_id,
|
||||
private_pool_region=variable_region,
|
||||
timeout_in_seconds=timeout_in_seconds,
|
||||
python_version=notebook_exec_python_version
|
||||
python_version=notebook_exec_python_version,
|
||||
)
|
||||
|
||||
operation_metadata = BuildOperationMetadata(mapping=operation.metadata)
|
||||
@@ -443,7 +448,7 @@ def process_and_execute_notebooks(
|
||||
result.log_url,
|
||||
result.output_uri,
|
||||
result.output_uri_web,
|
||||
result.logs_bucket
|
||||
result.logs_bucket,
|
||||
]
|
||||
for result in results_sorted
|
||||
],
|
||||
@@ -454,34 +459,34 @@ def process_and_execute_notebooks(
|
||||
"log_url",
|
||||
"output_uri",
|
||||
"output_uri_web",
|
||||
"logs_bucket"
|
||||
"logs_bucket",
|
||||
],
|
||||
)
|
||||
)
|
||||
|
||||
if len(notebooks) == 1:
|
||||
print("="*100)
|
||||
print("The notebook execution build log:\n")
|
||||
print("="*100)
|
||||
print("=" * 100)
|
||||
print("The notebook execution build log:\n")
|
||||
print("=" * 100)
|
||||
|
||||
build_id = results_sorted[0].build_id
|
||||
logs_bucket_name = (results_sorted[0].logs_bucket).removeprefix("gs://")
|
||||
log_file_name = f"log-{build_id}.txt"
|
||||
build_id = results_sorted[0].build_id
|
||||
logs_bucket_name = (results_sorted[0].logs_bucket).removeprefix("gs://")
|
||||
log_file_name = f"log-{build_id}.txt"
|
||||
|
||||
log_contents = util.download_blob_into_memory(
|
||||
bucket_name=logs_bucket_name,
|
||||
blob_name=log_file_name,
|
||||
download_as_text=True
|
||||
log_contents = util.download_blob_into_memory(
|
||||
bucket_name=logs_bucket_name,
|
||||
blob_name=log_file_name,
|
||||
download_as_text=True,
|
||||
)
|
||||
|
||||
# Remove extra steps from the log
|
||||
match = re.search("starting Step #4", log_contents, flags=re.IGNORECASE)
|
||||
# Remove extra steps from the log
|
||||
match = re.search("starting Step #4", log_contents, flags=re.IGNORECASE)
|
||||
|
||||
if match is not None:
|
||||
match_index = match.span()[0]
|
||||
print(log_contents[match_index:])
|
||||
else:
|
||||
print(log_contents)
|
||||
if match is not None:
|
||||
match_index = match.span()[0]
|
||||
print(log_contents[match_index:])
|
||||
else:
|
||||
print(log_contents)
|
||||
|
||||
print("\n=== END RESULTS===\n")
|
||||
|
||||
|
||||
@@ -2,5 +2,4 @@ notebooks/official/vizier/gapic-vizier-multi-objective-optimization.ipynb
|
||||
notebooks/official/pipelines/lightweight_functions_component_io_kfp.ipynb
|
||||
notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-locally-trained-models.ipynb
|
||||
notebooks/official/custom/custom-tabular-bq-managed-dataset.ipynb
|
||||
notebooks/official/matching_engine/sdk_matching_engine_for_indexing.ipynb
|
||||
.cloud-build/tests/python_version_test.ipynb
|
||||
|
||||
@@ -14,6 +14,8 @@
|
||||
# limitations under the License.
|
||||
|
||||
from typing import Dict
|
||||
import random
|
||||
import string
|
||||
|
||||
from nbconvert.preprocessors import Preprocessor
|
||||
|
||||
@@ -63,3 +65,30 @@ class UpdateVariablesPreprocessor(Preprocessor):
|
||||
executable_cells.append(cell)
|
||||
notebook.cells = executable_cells
|
||||
return notebook, resources
|
||||
|
||||
|
||||
# Generate a uuid of a specifed length
|
||||
def generate_uuid(length: int = 8) -> str:
|
||||
return "".join(random.choices(string.ascii_lowercase + string.digits, k=length))
|
||||
|
||||
|
||||
class UniqueStringsPreprocessor(Preprocessor):
|
||||
# A preprocessor that replaces strings that end with "-unique" with a uuid.
|
||||
|
||||
@staticmethod
|
||||
def update_unique_strings(content: str):
|
||||
# Replace strings that end with "-unique" with a uuid.
|
||||
|
||||
return content.replace('-unique"', f'-{generate_uuid()}"')
|
||||
|
||||
def preprocess(self, notebook, resources=None):
|
||||
executable_cells = []
|
||||
for cell in notebook.cells:
|
||||
if cell.cell_type == "code":
|
||||
cell.source = self.update_unique_strings(
|
||||
content=cell.source,
|
||||
)
|
||||
|
||||
executable_cells.append(cell)
|
||||
notebook.cells = executable_cells
|
||||
return notebook, resources
|
||||
|
||||
@@ -40,65 +40,3 @@ def get_updated_value(content: str, variable_name: str, variable_value: str) ->
|
||||
content,
|
||||
flags=re.M,
|
||||
)
|
||||
|
||||
|
||||
def test_update_value():
|
||||
new_content = get_updated_value(
|
||||
content='asdf\nPROJECT_ID = "[your-project-id]" #@param {type:"string"} \nasdf',
|
||||
variable_name="PROJECT_ID",
|
||||
variable_value="sample-project",
|
||||
)
|
||||
assert (
|
||||
new_content
|
||||
== 'asdf\nPROJECT_ID = "sample-project" #@param {type:"string"} \nasdf'
|
||||
)
|
||||
|
||||
|
||||
def test_update_value_single_quotes():
|
||||
new_content = get_updated_value(
|
||||
content="PROJECT_ID = '[your-project-id]'",
|
||||
variable_name="PROJECT_ID",
|
||||
variable_value="sample-project",
|
||||
)
|
||||
assert new_content == "PROJECT_ID = 'sample-project'"
|
||||
|
||||
|
||||
def test_update_value_avoidance():
|
||||
new_content = get_updated_value(
|
||||
content="PROJECT_ID = shell_output[0] ",
|
||||
variable_name="PROJECT_ID",
|
||||
variable_value="sample-project",
|
||||
)
|
||||
assert new_content == "PROJECT_ID = shell_output[0] "
|
||||
|
||||
|
||||
def test_region():
|
||||
new_content = get_updated_value(
|
||||
content='REGION = "[your-region]" # @param {type:"string"}',
|
||||
variable_name="REGION",
|
||||
variable_value="us-central1",
|
||||
)
|
||||
assert new_content == 'REGION = "us-central1" # @param {type:"string"}'
|
||||
|
||||
|
||||
def test_region_equal_equals_ignore():
|
||||
# Tests that == is ignored
|
||||
new_content = get_updated_value(
|
||||
content='REGION == "[your-region]" # @param {type:"string"}',
|
||||
variable_name="REGION",
|
||||
variable_value="us-central1",
|
||||
)
|
||||
assert new_content == 'REGION == "[your-region]" # @param {type:"string"}'
|
||||
|
||||
|
||||
def test_service_account():
|
||||
# Tests that == is ignored
|
||||
new_content = get_updated_value(
|
||||
content='SERVICE_ACCOUNT = "[your-service-account]" # @param {type:"string"}',
|
||||
variable_name="SERVICE_ACCOUNT",
|
||||
variable_value="12345-compute@developer.gserviceaccount.com",
|
||||
)
|
||||
assert (
|
||||
new_content
|
||||
== 'SERVICE_ACCOUNT = "12345-compute@developer.gserviceaccount.com" # @param {type:"string"}'
|
||||
)
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
from utils import NotebookProcessors
|
||||
|
||||
|
||||
def test_update_value():
|
||||
# Test that the content was updated
|
||||
preprocessor = NotebookProcessors.UniqueStringsPreprocessor()
|
||||
|
||||
content = 'PROJECT_ID = "your-project-id-unique"'
|
||||
|
||||
new_content = preprocessor.update_unique_strings(content)
|
||||
|
||||
assert new_content != content
|
||||
assert new_content.startswith('PROJECT_ID = "your-project-id-')
|
||||
assert new_content.endswith('"')
|
||||
@@ -0,0 +1,63 @@
|
||||
from utils import UpdateNotebookVariables
|
||||
|
||||
|
||||
def test_update_value():
|
||||
new_content = UpdateNotebookVariables.get_updated_value(
|
||||
content='asdf\nPROJECT_ID = "[your-project-id]" #@param {type:"string"} \nasdf',
|
||||
variable_name="PROJECT_ID",
|
||||
variable_value="sample-project",
|
||||
)
|
||||
assert (
|
||||
new_content
|
||||
== 'asdf\nPROJECT_ID = "sample-project" #@param {type:"string"} \nasdf'
|
||||
)
|
||||
|
||||
|
||||
def test_update_value_single_quotes():
|
||||
new_content = UpdateNotebookVariables.get_updated_value(
|
||||
content="PROJECT_ID = '[your-project-id]'",
|
||||
variable_name="PROJECT_ID",
|
||||
variable_value="sample-project",
|
||||
)
|
||||
assert new_content == "PROJECT_ID = 'sample-project'"
|
||||
|
||||
|
||||
def test_update_value_avoidance():
|
||||
new_content = UpdateNotebookVariables.get_updated_value(
|
||||
content="PROJECT_ID = shell_output[0] ",
|
||||
variable_name="PROJECT_ID",
|
||||
variable_value="sample-project",
|
||||
)
|
||||
assert new_content == "PROJECT_ID = shell_output[0] "
|
||||
|
||||
|
||||
def test_region():
|
||||
new_content = UpdateNotebookVariables.get_updated_value(
|
||||
content='REGION = "[your-region]" # @param {type:"string"}',
|
||||
variable_name="REGION",
|
||||
variable_value="us-central1",
|
||||
)
|
||||
assert new_content == 'REGION = "us-central1" # @param {type:"string"}'
|
||||
|
||||
|
||||
def test_region_equal_equals_ignore():
|
||||
# Tests that == is ignored
|
||||
new_content = UpdateNotebookVariables.get_updated_value(
|
||||
content='REGION == "[your-region]" # @param {type:"string"}',
|
||||
variable_name="REGION",
|
||||
variable_value="us-central1",
|
||||
)
|
||||
assert new_content == 'REGION == "[your-region]" # @param {type:"string"}'
|
||||
|
||||
|
||||
def test_service_account():
|
||||
# Tests that == is ignored
|
||||
new_content = UpdateNotebookVariables.get_updated_value(
|
||||
content='SERVICE_ACCOUNT = "[your-service-account]" # @param {type:"string"}',
|
||||
variable_name="SERVICE_ACCOUNT",
|
||||
variable_value="12345-compute@developer.gserviceaccount.com",
|
||||
)
|
||||
assert (
|
||||
new_content
|
||||
== 'SERVICE_ACCOUNT = "12345-compute@developer.gserviceaccount.com" # @param {type:"string"}'
|
||||
)
|
||||
+101
-372
@@ -31,6 +31,8 @@
|
||||
"source": [
|
||||
"# [TODO] Add your H1 title heading here\n",
|
||||
"\n",
|
||||
"{TODO: Update the links below.} \n",
|
||||
"\n",
|
||||
"<table align=\"left\">\n",
|
||||
"\n",
|
||||
" <td>\n",
|
||||
@@ -53,6 +55,17 @@
|
||||
"</table>"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "24743cf4a1e1"
|
||||
},
|
||||
"source": [
|
||||
"**_NOTE_**: This notebook has been tested in the following environment:\n",
|
||||
"\n",
|
||||
"* Python version = 3.9"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
@@ -111,7 +124,7 @@
|
||||
"This tutorial uses billable components of Google Cloud:\n",
|
||||
"\n",
|
||||
"* Vertex AI\n",
|
||||
"* {TODO: BigQyuery}\n",
|
||||
"* {TODO: BigQuery}\n",
|
||||
"* Cloud Storage\n",
|
||||
"\n",
|
||||
"{TODO: Include links to pricing documentation for each product you listed above.\n",
|
||||
@@ -125,62 +138,6 @@
|
||||
"to generate a cost estimate based on your projected usage."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "gCuSR8GkAgzl"
|
||||
},
|
||||
"source": [
|
||||
"### Set up your local development environment\n",
|
||||
"\n",
|
||||
"**If you are using Colab or Vertex AI Workbench Notebooks**, your environment already meets\n",
|
||||
"all the requirements to run this notebook. You can skip this step."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "24743cf4a1e1"
|
||||
},
|
||||
"source": [
|
||||
"**_NOTE_**: This notebook has been tested in the following environment:\n",
|
||||
"\n",
|
||||
"* Python version = 3.9"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "gCuSR8GkAgzl"
|
||||
},
|
||||
"source": [
|
||||
"**Otherwise**, make sure your environment meets this notebook's requirements.\n",
|
||||
"You need the following:\n",
|
||||
"\n",
|
||||
"* The Google Cloud SDK\n",
|
||||
"\n",
|
||||
"The Google Cloud guide to [Setting up a Python development\n",
|
||||
"environment](https://cloud.google.com/python/setup) and the [Jupyter\n",
|
||||
"installation guide](https://jupyter.org/install) provide detailed instructions\n",
|
||||
"for meeting these requirements. The following steps provide a condensed set of\n",
|
||||
"instructions:\n",
|
||||
"\n",
|
||||
"1. [Install and initialize the Cloud SDK.](https://cloud.google.com/sdk/docs/)\n",
|
||||
"\n",
|
||||
"1. [Install Python 3.](https://cloud.google.com/python/setup#installing_python)\n",
|
||||
"\n",
|
||||
"1. [Install\n",
|
||||
" virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv)\n",
|
||||
" and create a virtual environment that uses Python 3. Activate the virtual environment.\n",
|
||||
"\n",
|
||||
"1. To install Jupyter, run `pip3 install jupyter` on the\n",
|
||||
"command-line in a terminal shell.\n",
|
||||
"\n",
|
||||
"1. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
|
||||
"\n",
|
||||
"1. Open this notebook in the Jupyter Notebook Dashboard."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
@@ -202,51 +159,32 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"\n",
|
||||
"# The Vertex AI Workbench Notebook product has specific requirements\n",
|
||||
"IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
|
||||
"IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
|
||||
" \"/opt/deeplearning/metadata/env_version\"\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
|
||||
"USER_FLAG = \"\"\n",
|
||||
"if IS_WORKBENCH_NOTEBOOK:\n",
|
||||
" USER_FLAG = \"--user\"\n",
|
||||
"\n",
|
||||
"! pip3 install --upgrade google-cloud-aiplatform {USER_FLAG} -q\n",
|
||||
"# TODO: Add remaining package installs here. All packages should be on a single pip install to resolve dependencies"
|
||||
"# Install the packages\n",
|
||||
"! pip3 install --user --upgrade google-cloud-aiplatform"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "hhq5zEbGg0XX"
|
||||
"id": "58707a750154"
|
||||
},
|
||||
"source": [
|
||||
"### Restart the kernel\n",
|
||||
"\n",
|
||||
"After you install the additional packages, you need to restart the notebook kernel so it can find the packages."
|
||||
"### Colab only: Uncomment the following cell to restart the kernel."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "EzrelQZ22IZj"
|
||||
"id": "f200f10a1da3"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Automatically restart kernel after installs\n",
|
||||
"import os\n",
|
||||
"# Automatically restart kernel after installs so that your environment can access the new packages\n",
|
||||
"# import IPython\n",
|
||||
"\n",
|
||||
"if not os.getenv(\"IS_TESTING\"):\n",
|
||||
" # Automatically restart kernel after installs\n",
|
||||
" import IPython\n",
|
||||
"\n",
|
||||
" app = IPython.Application.instance()\n",
|
||||
" app.kernel.do_shutdown(True)"
|
||||
"# app = IPython.Application.instance()\n",
|
||||
"# app.kernel.do_shutdown(True)"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -263,16 +201,11 @@
|
||||
"\n",
|
||||
"1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
|
||||
"\n",
|
||||
"1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
|
||||
"2. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
|
||||
"\n",
|
||||
"1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com). {TODO: Update the APIs needed for your tutorial. Edit the API names, and update the link to append the API IDs, separating each one with a comma. For example, container.googleapis.com,cloudbuild.googleapis.com}\n",
|
||||
"3. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com). {TODO: Update the APIs needed for your tutorial. Edit the API names, and update the link to append the API IDs, separating each one with a comma. For example, container.googleapis.com,cloudbuild.googleapis.com}\n",
|
||||
"\n",
|
||||
"1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
|
||||
"\n",
|
||||
"1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
|
||||
"Cloud SDK uses the right project for all the commands in this notebook.\n",
|
||||
"\n",
|
||||
"**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
|
||||
"4. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk)."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -283,7 +216,10 @@
|
||||
"source": [
|
||||
"#### Set your project ID\n",
|
||||
"\n",
|
||||
"**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
|
||||
"**If you don't know your project ID**, try the following:\n",
|
||||
"* Run `gcloud config list`.\n",
|
||||
"* Run `gcloud projects list`.\n",
|
||||
"* See the support page: [Locate the project ID](https://support.google.com/googleapi/answer/7014113)"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -294,57 +230,10 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"PROJECT_ID = \"[your-project-id]\" # @param {type:\"string\"}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "riG_qUokg0XZ"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
|
||||
" # Get your GCP project id from gcloud\n",
|
||||
" shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
|
||||
" PROJECT_ID = shell_output[0]\n",
|
||||
" print(\"Project ID:\", PROJECT_ID)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "set_gcloud_project_id"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gcloud config set project $PROJECT_ID"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "23988890fef6"
|
||||
},
|
||||
"source": [
|
||||
"#### Get your project number {TODO: Include these cells if the notebook uses a project number}\n",
|
||||
"PROJECT_ID = \"[your-project-id]\" # @param {type:\"string\"}\n",
|
||||
"\n",
|
||||
"Now that the project ID is set, you get your corresponding project number."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "2d6950574e1d"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"shell_output = ! gcloud projects list --filter=\"PROJECT_ID:'{PROJECT_ID}'\" --format='value(PROJECT_NUMBER)'\n",
|
||||
"PROJECT_NUMBER = shell_output[0]\n",
|
||||
"print(\"Project Number:\", PROJECT_NUMBER)"
|
||||
"# Set the project id\n",
|
||||
"! gcloud config set project {PROJECT_ID}"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -355,63 +244,18 @@
|
||||
"source": [
|
||||
"#### Region\n",
|
||||
"\n",
|
||||
"You can also change the `REGION` variable, which is used for operations\n",
|
||||
"throughout the rest of this notebook. Below are regions supported for Vertex AI. It is recommended that you choose the region closest to you.\n",
|
||||
"\n",
|
||||
"- Americas: `us-central1`\n",
|
||||
"- Europe: `europe-west4`\n",
|
||||
"- Asia Pacific: `asia-east1`\n",
|
||||
"\n",
|
||||
"You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
|
||||
"\n",
|
||||
"Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
|
||||
"You can also change the `REGION` variable used by Vertex AI. Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 1,
|
||||
"metadata": {
|
||||
"id": "region"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"REGION = \"[your-region]\" # @param {type: \"string\"}\n",
|
||||
"\n",
|
||||
"if REGION == \"[your-region]\":\n",
|
||||
" REGION = \"us-central1\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "06571eb4063b"
|
||||
},
|
||||
"source": [
|
||||
"#### UUID\n",
|
||||
"\n",
|
||||
"If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial.\n",
|
||||
"\n",
|
||||
"{TODO: replace the `TIMESTAMP` with `UUID` in official notebooks}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "697568e92bd6"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import random\n",
|
||||
"import string\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Generate a uuid of a specifed length(default=8)\n",
|
||||
"def generate_uuid(length: int = 8) -> str:\n",
|
||||
" return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"UUID = generate_uuid()"
|
||||
"REGION = \"[your-region]\" # @param {type: \"string\"}"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -422,64 +266,68 @@
|
||||
"source": [
|
||||
"### Authenticate your Google Cloud account\n",
|
||||
"\n",
|
||||
"**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
|
||||
"authenticated. \n",
|
||||
"\n",
|
||||
"**If you are using Colab**, run the cell below and follow the instructions\n",
|
||||
"when prompted to authenticate your account via oAuth.\n",
|
||||
"\n",
|
||||
"**Otherwise**, follow these steps:\n",
|
||||
"\n",
|
||||
"1. In the Cloud Console, go to the [**Create service account key**\n",
|
||||
" page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
|
||||
"\n",
|
||||
"2. Click **Create service account**.\n",
|
||||
"\n",
|
||||
"3. In the **Service account name** field, enter a name, and\n",
|
||||
" click **Create**.\n",
|
||||
"\n",
|
||||
"4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
|
||||
"into the filter box, and select\n",
|
||||
" **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
|
||||
"\n",
|
||||
"5. Click *Create*. A JSON file that contains your key downloads to your\n",
|
||||
"local environment.\n",
|
||||
"\n",
|
||||
"6. Enter the path to your service account key as the\n",
|
||||
"`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
|
||||
"Depending on your Jupyter environment, you may have to manually authenticate. Follow the relevant instructions below."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "74ccc9e52986"
|
||||
},
|
||||
"source": [
|
||||
"**1. Vertex AI Workbench**\n",
|
||||
"* Do nothing as you are already authenticated."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "de775a3773ba"
|
||||
},
|
||||
"source": [
|
||||
"**2. Local JupyterLab instance, uncomment and run:**"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 2,
|
||||
"metadata": {
|
||||
"id": "PyQmSRbKA8r-"
|
||||
"id": "254614fa0c46"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# If you are running this notebook in Colab, run this cell and follow the\n",
|
||||
"# instructions to authenticate your GCP account. This provides access to your\n",
|
||||
"# Cloud Storage bucket and lets you submit training jobs and prediction\n",
|
||||
"# requests.\n",
|
||||
"\n",
|
||||
"import os\n",
|
||||
"import sys\n",
|
||||
"\n",
|
||||
"# If on Vertex AI Workbench, then don't execute this code\n",
|
||||
"IS_COLAB = \"google.colab\" in sys.modules\n",
|
||||
"if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
|
||||
" \"DL_ANACONDA_HOME\"\n",
|
||||
"):\n",
|
||||
" if \"google.colab\" in sys.modules:\n",
|
||||
" from google.colab import auth as google_auth\n",
|
||||
"\n",
|
||||
" google_auth.authenticate_user()\n",
|
||||
"\n",
|
||||
" # If you are running this notebook locally, replace the string below with the\n",
|
||||
" # path to your service account key and run this cell to authenticate your GCP\n",
|
||||
" # account.\n",
|
||||
" elif not os.getenv(\"IS_TESTING\"):\n",
|
||||
" %env GOOGLE_APPLICATION_CREDENTIALS '[your-service-account-key-path]'"
|
||||
"# ! gcloud auth login"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "ef21552ccea8"
|
||||
},
|
||||
"source": [
|
||||
"**3. Colab, uncomment and run:**"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"metadata": {
|
||||
"id": "603adbbf0532"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# from google.colab import auth\n",
|
||||
"# auth.authenticate_user()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "f6b2ccc891ed"
|
||||
},
|
||||
"source": [
|
||||
"**4. Service account or other**\n",
|
||||
"* See all authentication options here: [Google Cloud Platform Jupyter Notebook Authentication Guide](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/notebook_authentication_guide.ipynb)"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -490,20 +338,9 @@
|
||||
"source": [
|
||||
"### Create a Cloud Storage bucket\n",
|
||||
"\n",
|
||||
"**The following steps are required, regardless of your notebook environment.**\n",
|
||||
"Create a storage bucket to store intermediate artifacts such as datasets.\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"{TODO: Adjust wording in the first paragraph to fit your use case - explain how your tutorial uses the Cloud Storage bucket. The example below shows how Vertex AI uses the bucket for training.}\n",
|
||||
"\n",
|
||||
"When you submit a training job using the Vertex AI SDK, you upload a Python package\n",
|
||||
"containing your training code to a Cloud Storage bucket. Vertex AI runs\n",
|
||||
"the code from this package. In this tutorial, Vertex AI also saves the\n",
|
||||
"trained model that results from your job in the same bucket. Using this model artifact, you can then\n",
|
||||
"create Vertex AI model and endpoint resources in order to serve\n",
|
||||
"online predictions.\n",
|
||||
"\n",
|
||||
"Set the name of your Cloud Storage bucket below. It must be unique across all\n",
|
||||
"Cloud Storage buckets."
|
||||
"- *{Note to notebook author: For any user-provided strings that need to be unique (like bucket names or model ID's), append \"-unique\" to the end so proper testing can occur}*"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -514,21 +351,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"BUCKET_NAME = \"[your-bucket-name]\" # @param {type:\"string\"}\n",
|
||||
"BUCKET_URI = f\"gs://{BUCKET_NAME}\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "cf221059d072"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
|
||||
" BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
|
||||
" BUCKET_URI = f\"gs://{BUCKET_NAME}\""
|
||||
"BUCKET_URI = \"gs://your-bucket-name-unique\" # @param {type:\"string\"}"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -554,99 +377,7 @@
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "ucvCsknMCims"
|
||||
},
|
||||
"source": [
|
||||
"Finally, validate access to your Cloud Storage bucket by examining its contents:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "vhOb7YnwClBb"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil ls -al $BUCKET_URI"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "set_service_account"
|
||||
},
|
||||
"source": [
|
||||
"#### Service Account {TODO: Include these cells if the notebook specifies a service account}\n",
|
||||
"\n",
|
||||
"{TODO: What uses service account in the notebook; e.g., You use a service account to create Vertex AI Pipeline jobs.}. If you do not want to use your project's Compute Engine service account, set `SERVICE_ACCOUNT` to another service account ID."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "set_service_account"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"SERVICE_ACCOUNT = \"[your-service-account]\" # @param {type:\"string\"}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "autoset_service_account"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"if (\n",
|
||||
" SERVICE_ACCOUNT == \"\"\n",
|
||||
" or SERVICE_ACCOUNT is None\n",
|
||||
" or SERVICE_ACCOUNT == \"[your-service-account]\"\n",
|
||||
"):\n",
|
||||
" # Get your service account from gcloud\n",
|
||||
" if not IS_COLAB:\n",
|
||||
" shell_output = !gcloud auth list 2>/dev/null\n",
|
||||
" SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
|
||||
"\n",
|
||||
" else: # IS_COLAB:\n",
|
||||
" shell_output = ! gcloud projects describe $PROJECT_ID\n",
|
||||
" project_number = shell_output[-1].split(\":\")[1].strip().replace(\"'\", \"\")\n",
|
||||
" SERVICE_ACCOUNT = f\"{project_number}-compute@developer.gserviceaccount.com\"\n",
|
||||
"\n",
|
||||
" print(\"Service Account:\", SERVICE_ACCOUNT)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "set_service_account:pipelines"
|
||||
},
|
||||
"source": [
|
||||
"#### Set service account access for {TODO; e.g., Vertex AI Pipelines}\n",
|
||||
"\n",
|
||||
"Run the following commands to grant your service account access to {TODO; i.e., read and write pipeline artifacts} in the bucket that you created in the previous step. You only need to run this step once per service account."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "set_service_account:pipelines"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
|
||||
"\n",
|
||||
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "XoEqT2Y4DJmf"
|
||||
"id": "960505627ddf"
|
||||
},
|
||||
"source": [
|
||||
"### Import libraries"
|
||||
@@ -656,13 +387,11 @@
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "pRUOFELefqf1"
|
||||
"id": "PyQmSRbKA8r-"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import google.cloud.aiplatform as aiplatform\n",
|
||||
"\n",
|
||||
"# TODO: import remaining libraries; e.g., tensorflow"
|
||||
"from google.cloud import aiplatform"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -705,21 +434,21 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 1,
|
||||
"metadata": {
|
||||
"id": "sx_vKniMq9ZX"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"\n",
|
||||
"# Delete endpoint resource\n",
|
||||
"! gcloud ai endpoints delete $ENDPOINT_NAME --quiet --region $REGION\n",
|
||||
"# e.g. `endpoint.delete()`\n",
|
||||
"\n",
|
||||
"# Delete model resource\n",
|
||||
"! gcloud ai models delete $MODEL_NAME --quiet\n",
|
||||
"# e.g. `model.delete()`\n",
|
||||
"\n",
|
||||
"# Delete Cloud Storage objects that were created\n",
|
||||
"! gsutil -m rm -r $JOB_DIR\n",
|
||||
"\n",
|
||||
"delete_bucket = False\n",
|
||||
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
|
||||
" ! gsutil -m rm -r $BUCKET_URI"
|
||||
|
||||
Reference in New Issue
Block a user