mirror of
https://github.com/GoogleCloudPlatform/vertex-ai-samples.git
synced 2026-09-26 22:51:56 +00:00
Compare commits
5
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 gericdongandGitHub
|
3ebb74e97a | ||
|
gericdongandGitHub
|
a992a5530d | ||
|
Andrew FerlitschandGitHub
|
24e0e92f8d | ||
|
Andrew Ferlitsch
|
bc4ec36914 | ||
|
Andrew Ferlitsch
|
5387799f32 |
@@ -17,16 +17,13 @@ import concurrent
|
||||
import dataclasses
|
||||
import datetime
|
||||
import functools
|
||||
import json
|
||||
import git
|
||||
import operator
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import subprocess
|
||||
import utils
|
||||
from typing import List, Optional
|
||||
from utils import util
|
||||
|
||||
import execute_notebook_helper
|
||||
import execute_notebook_remote
|
||||
@@ -38,7 +35,6 @@ from utils import NotebookProcessors, util
|
||||
|
||||
# A buffer so that workers finish before the orchestrating job
|
||||
WORKER_TIMEOUT_BUFFER_IN_SECONDS: int = 60 * 60
|
||||
PYTHON_VERSION = "3.9" # Set default python version
|
||||
|
||||
|
||||
def format_timedelta(delta: datetime.timedelta) -> str:
|
||||
@@ -70,7 +66,6 @@ class NotebookExecutionResult:
|
||||
log_url: str
|
||||
output_uri: str
|
||||
build_id: str
|
||||
logs_bucket: str
|
||||
error_message: Optional[str]
|
||||
|
||||
@property
|
||||
@@ -115,33 +110,6 @@ def _process_notebook(
|
||||
nbformat.write(nb, new_file)
|
||||
|
||||
|
||||
def _get_notebook_python_version(notebook_path: str) -> str:
|
||||
"""
|
||||
Get the python version for running the notebook if it is specified in
|
||||
the notebook.
|
||||
"""
|
||||
python_version = PYTHON_VERSION
|
||||
|
||||
# Load the notebook
|
||||
file = open(notebook_path)
|
||||
src = file.read()
|
||||
nb_json = json.loads(src)
|
||||
|
||||
#Iterate over the cells in the ipynb
|
||||
for cell in nb_json['cells']:
|
||||
if cell['cell_type'] == 'markdown':
|
||||
markdown = str.join('', cell['source'])
|
||||
|
||||
# Look for the python version specification pattern
|
||||
re_match = re.search('python version = (\d\.\d)', markdown, flags=re.IGNORECASE)
|
||||
if re_match:
|
||||
# get the version number
|
||||
python_version = re_match.group(1)
|
||||
break
|
||||
|
||||
return python_version
|
||||
|
||||
|
||||
def _create_tag(filepath: str) -> str:
|
||||
tag = os.path.basename(os.path.normpath(filepath))
|
||||
tag = re.sub("[^0-9a-zA-Z_.-]+", "-", tag)
|
||||
@@ -192,7 +160,6 @@ def process_and_execute_notebook(
|
||||
output_uri=notebook_output_uri,
|
||||
log_url="",
|
||||
build_id="",
|
||||
logs_bucket="",
|
||||
error_message=None,
|
||||
)
|
||||
|
||||
@@ -200,10 +167,6 @@ def process_and_execute_notebook(
|
||||
time_start = datetime.datetime.now()
|
||||
operation = None
|
||||
try:
|
||||
# Get the python version for running the notebook if specified
|
||||
notebook_exec_python_version = _get_notebook_python_version(notebook_path=notebook)
|
||||
print(f"Running notebook with python {notebook_exec_python_version}")
|
||||
|
||||
# Pre-process notebook by substituting variable names
|
||||
_process_notebook(
|
||||
notebook_path=notebook,
|
||||
@@ -230,13 +193,11 @@ def process_and_execute_notebook(
|
||||
private_pool_id=private_pool_id,
|
||||
private_pool_region=variable_region,
|
||||
timeout_in_seconds=timeout_in_seconds,
|
||||
python_version=notebook_exec_python_version
|
||||
)
|
||||
|
||||
operation_metadata = BuildOperationMetadata(mapping=operation.metadata)
|
||||
result.build_id = operation_metadata.build.id
|
||||
result.log_url = operation_metadata.build.log_url
|
||||
result.logs_bucket = operation_metadata.build.logs_bucket
|
||||
|
||||
# Block and wait for the result
|
||||
operation_result = operation.result()
|
||||
@@ -378,7 +339,7 @@ def process_and_execute_notebooks(
|
||||
seconds=max(timeout - WORKER_TIMEOUT_BUFFER_IN_SECONDS, 0)
|
||||
)
|
||||
|
||||
if len(notebooks) >= 1:
|
||||
if len(notebooks) > 1:
|
||||
notebook_execution_results: List[NotebookExecutionResult] = []
|
||||
|
||||
print(f"Found {len(notebooks)} modified notebooks: {notebooks}")
|
||||
@@ -443,7 +404,6 @@ def process_and_execute_notebooks(
|
||||
result.log_url,
|
||||
result.output_uri,
|
||||
result.output_uri_web,
|
||||
result.logs_bucket
|
||||
]
|
||||
for result in results_sorted
|
||||
],
|
||||
@@ -454,35 +414,10 @@ def process_and_execute_notebooks(
|
||||
"log_url",
|
||||
"output_uri",
|
||||
"output_uri_web",
|
||||
"logs_bucket"
|
||||
],
|
||||
)
|
||||
)
|
||||
|
||||
if len(notebooks) == 1:
|
||||
print("="*100)
|
||||
print("The notebook execution build log:\n")
|
||||
print("="*100)
|
||||
|
||||
build_id = results_sorted[0].build_id
|
||||
logs_bucket_name = (results_sorted[0].logs_bucket).removeprefix("gs://")
|
||||
log_file_name = f"log-{build_id}.txt"
|
||||
|
||||
log_contents = util.download_blob_into_memory(
|
||||
bucket_name=logs_bucket_name,
|
||||
blob_name=log_file_name,
|
||||
download_as_text=True
|
||||
)
|
||||
|
||||
# Remove extra steps from the log
|
||||
match = re.search("starting Step #4", log_contents, flags=re.IGNORECASE)
|
||||
|
||||
if match is not None:
|
||||
match_index = match.span()[0]
|
||||
print(log_contents[match_index:])
|
||||
else:
|
||||
print(log_contents)
|
||||
|
||||
print("\n=== END RESULTS===\n")
|
||||
|
||||
total_notebook_duration = functools.reduce(
|
||||
@@ -498,5 +433,25 @@ def process_and_execute_notebooks(
|
||||
# Raise error if any notebooks failed
|
||||
if not all([result.is_pass for result in results_sorted]):
|
||||
raise RuntimeError("Notebook failures detected. See logs for details")
|
||||
|
||||
elif len(notebooks) == 1:
|
||||
notebook = notebooks[0]
|
||||
|
||||
# Pre-process notebook by substituting variable names
|
||||
_process_notebook(
|
||||
notebook_path=notebook,
|
||||
variable_project_id=variable_project_id,
|
||||
variable_region=variable_region,
|
||||
variable_service_account=variable_service_account,
|
||||
variable_vpc_network=variable_vpc_network,
|
||||
)
|
||||
|
||||
execute_notebook_helper.execute_notebook(
|
||||
notebook_source=notebook,
|
||||
output_file_or_uri="/".join(
|
||||
[artifacts_bucket, pathlib.Path(notebook).name]
|
||||
),
|
||||
should_log_output=True,
|
||||
)
|
||||
else:
|
||||
print("No notebooks modified in this pull request.")
|
||||
|
||||
@@ -40,7 +40,6 @@ def execute_notebook_remote(
|
||||
private_pool_region: Optional[str],
|
||||
tag: Optional[str],
|
||||
timeout_in_seconds: Optional[int] = None,
|
||||
python_version: Optional[str] = None
|
||||
) -> operation.Operation:
|
||||
"""Create and execute a single notebook on Google Cloud Build"""
|
||||
# Load build steps from YAML
|
||||
@@ -51,12 +50,8 @@ def execute_notebook_remote(
|
||||
"_PYTHON_IMAGE": container_uri,
|
||||
"_NOTEBOOK_GCS_URI": notebook_uri,
|
||||
"_NOTEBOOK_OUTPUT_GCS_URI": notebook_output_uri,
|
||||
"_PYTHON_VERSION" : f"python{python_version}"
|
||||
}
|
||||
|
||||
if python_version is not None:
|
||||
substitutions["_PYTHON_VERSION"] = "python" + python_version
|
||||
|
||||
build = cloudbuild_v1.Build()
|
||||
|
||||
options: Optional[client_options.ClientOptions] = None
|
||||
|
||||
@@ -10,21 +10,21 @@ steps:
|
||||
entrypoint: /bin/sh
|
||||
args:
|
||||
- -c
|
||||
- ${_PYTHON_VERSION} .cloud-build/CheckPythonVersion.py -q
|
||||
- python3 .cloud-build/CheckPythonVersion.py -q
|
||||
# Create a virtual environment
|
||||
- name: ${_PYTHON_IMAGE}
|
||||
entrypoint: /bin/sh
|
||||
args:
|
||||
- -c
|
||||
- ${_PYTHON_VERSION} -m venv workspace/env
|
||||
- python3 -m venv workspace/env
|
||||
# Install Python dependencies
|
||||
- name: ${_PYTHON_IMAGE}
|
||||
entrypoint: /bin/sh
|
||||
args:
|
||||
- -c
|
||||
- . workspace/env/bin/activate &&
|
||||
python -m pip -q install -U pip &&
|
||||
python -m pip -q install -U -r .cloud-build/requirements.txt
|
||||
python3 -m pip -q install -U pip &&
|
||||
python3 -m pip -q install -U -r .cloud-build/requirements.txt
|
||||
# Install Python dependencies and run testing script
|
||||
- name: ${_PYTHON_IMAGE}
|
||||
entrypoint: /bin/sh
|
||||
@@ -32,7 +32,7 @@ steps:
|
||||
- -c
|
||||
- |
|
||||
. workspace/env/bin/activate &&
|
||||
python .cloud-build/execute_notebook_cli.py --notebook_source "${_NOTEBOOK_GCS_URI}" --output_file_or_uri "${_NOTEBOOK_OUTPUT_GCS_URI}"
|
||||
python3 .cloud-build/execute_notebook_cli.py --notebook_source "${_NOTEBOOK_GCS_URI}" --output_file_or_uri "${_NOTEBOOK_OUTPUT_GCS_URI}"
|
||||
env:
|
||||
- 'IS_TESTING=1'
|
||||
timeout: 86400s
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
notebooks/official/vizier/gapic-vizier-multi-objective-optimization.ipynb
|
||||
notebooks/official/pipelines/lightweight_functions_component_io_kfp.ipynb
|
||||
notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-locally-trained-models.ipynb
|
||||
notebooks/official/custom/custom-tabular-bq-managed-dataset.ipynb
|
||||
notebooks/official/matching_engine/sdk_matching_engine_for_indexing.ipynb
|
||||
.cloud-build/tests/python_version_test.ipynb
|
||||
notebooks/official/pipelines/metrics_viz_run_compare_kfp.ipynb
|
||||
notebooks/official/matching_engine/sdk_matching_engine_for_indexing.ipynb
|
||||
@@ -1 +1 @@
|
||||
notebooks/official/custom/custom-tabular-bq-managed-dataset.ipynb
|
||||
notebooks/official/pipelines/metrics_viz_run_compare_kfp.ipynb
|
||||
|
||||
@@ -1,61 +0,0 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "57a3d44ed8a8"
|
||||
},
|
||||
"source": [
|
||||
"### Set up your Google Cloud project\n",
|
||||
"\n",
|
||||
"**_NOTE_**: This notebook has been tested in the following environment:\n",
|
||||
"\n",
|
||||
"* Python version = 3.7\n",
|
||||
"\n",
|
||||
"**The following steps are required, regardless of your notebook environment.**\n",
|
||||
"\n",
|
||||
"1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
|
||||
"\n",
|
||||
"1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
|
||||
"\n",
|
||||
"1. [Enable the Vertex AI API and Compute Engine API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component).\n",
|
||||
"\n",
|
||||
"1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
|
||||
"\n",
|
||||
"1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
|
||||
"Cloud SDK uses the right project for all the commands in this notebook.\n",
|
||||
"\n",
|
||||
"**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "c6516f90311b"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# test if the right python version is being used\n",
|
||||
"import sys\n",
|
||||
"\n",
|
||||
"actual_python_version = f\"{sys.version_info.major}.{sys.version_info.minor}\"\n",
|
||||
"print(f\"Runtime python version: {actual_python_version}\")\n",
|
||||
"\n",
|
||||
"assert actual_python_version == \"3.7\", \"Wrong python version!\""
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"name": "python_version_test.ipynb",
|
||||
"toc_visible": true
|
||||
},
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"name": "python3"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 0
|
||||
}
|
||||
@@ -3,7 +3,7 @@ import subprocess
|
||||
import tarfile
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from typing import Optional, Union
|
||||
from typing import Optional
|
||||
|
||||
from google.auth import credentials as auth_credentials
|
||||
from google.cloud import storage
|
||||
@@ -58,34 +58,3 @@ def archive_code_and_upload(staging_bucket: str):
|
||||
print(f"Uploaded source code archive to {source_archived_file_gcs}")
|
||||
|
||||
return source_archived_file_gcs
|
||||
|
||||
|
||||
def download_blob_into_memory(
|
||||
bucket_name: str,
|
||||
blob_name: str,
|
||||
download_as_text: Optional[bool]=False
|
||||
) -> Union[bytes, str]:
|
||||
"""
|
||||
Downloads a blob into memory as byte or as text if
|
||||
download_as_text is set to True.
|
||||
"""
|
||||
|
||||
storage_client = storage.Client()
|
||||
|
||||
bucket = storage_client.bucket(bucket_name)
|
||||
|
||||
# Construct a client side representation of a blob.
|
||||
blob = bucket.blob(blob_name)
|
||||
|
||||
# Download the blob content
|
||||
if download_as_text:
|
||||
contents = blob.download_as_text()
|
||||
else:
|
||||
contents = blob.download_as_bytes()
|
||||
|
||||
print(
|
||||
f"Downloaded storage object {blob_name} from bucket {bucket_name}."
|
||||
)
|
||||
|
||||
return contents
|
||||
|
||||
|
||||
@@ -6,4 +6,3 @@
|
||||
/sklearn_text_classification_from_script_using_vertex_sdk @maxhardt
|
||||
/pluto_on_workbench @wkharold
|
||||
/cpr-examples @samthrasher
|
||||
/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines @Ark-kun
|
||||
|
||||
-83
@@ -1,83 +0,0 @@
|
||||
name: Train tabular classification logistic regression model using Scikit learn pipeline
|
||||
metadata:
|
||||
annotations:
|
||||
author: Alexey Volkov <alexey.volkov@ark-kun.com>
|
||||
canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_classification_logistic_regression_model_using_Scikit_learn_and_import_to_Vertex_AI/pipeline.component.yaml
|
||||
sdk: https://cloud-pipelines.net/pipeline-editor/
|
||||
implementation:
|
||||
graph:
|
||||
tasks:
|
||||
Download from GCS:
|
||||
componentRef:
|
||||
digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":40,"width":180,"height":40}'
|
||||
Select columns using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: Data
|
||||
taskId: Download from GCS
|
||||
column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":140,"width":180,"height":54}'
|
||||
Fill all missing values using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Select columns using Pandas on CSV data
|
||||
type: CSV
|
||||
replacement_value: '0'
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":250,"width":180,"height":54}'
|
||||
Binarize column using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: d699afd4d7cae862708717cc160f4394ed0c04e536e9515923ef1e8865f01d44
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Fill all missing values using Pandas on CSV data
|
||||
type: CSV
|
||||
column_name: tips
|
||||
predicate: '> 0'
|
||||
new_column_name: class
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":380,"width":180,"height":54}'
|
||||
Train logistic regression model using scikit learn from CSV:
|
||||
componentRef:
|
||||
digest: a864625a822e4b1c8ef6fe4ae1454fd90f15438f70a6712bb4c30e0dda4d35b7
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/cb44b75c9c062fcc40c2b905b2024b4493dbc62b/components/ML_frameworks/Scikit_learn/Train_logistic_regression_model/from_CSV/component.yaml
|
||||
arguments:
|
||||
dataset:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Binarize column using Pandas on CSV data
|
||||
type: CSV
|
||||
label_column_name: class
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":510,"width":180,"height":70}'
|
||||
Upload Scikit learn pickle model to Google Cloud Vertex AI:
|
||||
componentRef:
|
||||
digest: 81c91c8d7d21ec97e0872f669d68bd89edea87279d703685db54aa94743bebcd
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Train logistic regression model using scikit learn from CSV
|
||||
type: ScikitLearnPickleModel
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":660,"width":180,"height":70}'
|
||||
outputValues: {}
|
||||
-73
@@ -1,73 +0,0 @@
|
||||
# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
|
||||
from kfp import components
|
||||
|
||||
# %% Loading components
|
||||
download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml")
|
||||
fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
|
||||
binarize_column_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml")
|
||||
train_logistic_regression_model_using_scikit_learn_from_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/cb44b75c9c062fcc40c2b905b2024b4493dbc62b/components/ML_frameworks/Scikit_learn/Train_logistic_regression_model/from_CSV/component.yaml")
|
||||
upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
|
||||
# %% Pipeline definition
|
||||
def train_tabular_classification_logistic_regression_model_using_Scikit_learn_pipeline():
|
||||
dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
|
||||
feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"] # Excluded "trip_total"
|
||||
label_column = "tips"
|
||||
# Deploying the model might incur additional costs over time
|
||||
deploy_model = False
|
||||
|
||||
classification_label_column = "class"
|
||||
all_columns = [label_column] + feature_columns
|
||||
|
||||
training_data = download_from_gcs_op(
|
||||
gcs_path=dataset_gcs_uri
|
||||
).outputs["Data"]
|
||||
|
||||
training_data = select_columns_using_Pandas_on_CSV_data_op(
|
||||
table=training_data,
|
||||
column_names=all_columns,
|
||||
).outputs["transformed_table"]
|
||||
|
||||
# Cleaning the NaN values.
|
||||
training_data = fill_all_missing_values_using_Pandas_on_CSV_data_op(
|
||||
table=training_data,
|
||||
replacement_value="0",
|
||||
#replacement_type_name="float",
|
||||
).outputs["transformed_table"]
|
||||
|
||||
classification_training_data = binarize_column_using_Pandas_on_CSV_data_op(
|
||||
table=training_data,
|
||||
column_name=label_column,
|
||||
predicate="> 0",
|
||||
new_column_name=classification_label_column,
|
||||
).outputs["transformed_table"]
|
||||
|
||||
model = train_logistic_regression_model_using_scikit_learn_from_CSV_op(
|
||||
dataset=classification_training_data,
|
||||
label_column_name=classification_label_column,
|
||||
# Optional:
|
||||
#penalty="l2",
|
||||
#solver="lbfgs",
|
||||
#max_iterations=100,
|
||||
#multi_class_mode="auto",
|
||||
#random_seed=0,
|
||||
).outputs["model"]
|
||||
|
||||
vertex_model_name = upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op(
|
||||
model=model,
|
||||
).outputs["model_name"]
|
||||
|
||||
# Deploying the model might incur additional costs over time
|
||||
if deploy_model:
|
||||
sklearn_vertex_endpoint_name = deploy_model_to_endpoint_op(
|
||||
model_name=vertex_model_name,
|
||||
).outputs["endpoint_name"]
|
||||
|
||||
pipeline_func = train_tabular_classification_logistic_regression_model_using_Scikit_learn_pipeline
|
||||
|
||||
# %% Pipeline submission
|
||||
if __name__ == '__main__':
|
||||
from google.cloud import aiplatform
|
||||
aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()
|
||||
-114
@@ -1,114 +0,0 @@
|
||||
name: Train tabular classification model using PyTorch pipeline
|
||||
metadata:
|
||||
annotations:
|
||||
author: Alexey Volkov <alexey.volkov@ark-kun.com>
|
||||
canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_classification_model_using_PyTorch_and_import_to_Vertex_AI/pipeline.component.yaml
|
||||
sdk: https://cloud-pipelines.net/pipeline-editor/
|
||||
implementation:
|
||||
graph:
|
||||
tasks:
|
||||
Download from GCS:
|
||||
componentRef:
|
||||
digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
|
||||
annotations:
|
||||
editor.position: '{"x":240,"y":40,"width":180,"height":40}'
|
||||
Select columns using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: Data
|
||||
taskId: Download from GCS
|
||||
column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
|
||||
annotations:
|
||||
editor.position: '{"x":240,"y":140,"width":180,"height":54}'
|
||||
Fill all missing values using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Select columns using Pandas on CSV data
|
||||
type: CSV
|
||||
replacement_value: '0'
|
||||
annotations:
|
||||
editor.position: '{"x":240,"y":250,"width":180,"height":54}'
|
||||
Create fully connected pytorch network:
|
||||
componentRef:
|
||||
digest: d03d8248fd358a0275ec33568ee7dd7dce576cc112b09dfafe2651e4d97e04a9
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1a2ef3eeb77bc278f33cad0dd29008ea2431e191/components/PyTorch/Create_fully_connected_network/component.yaml
|
||||
arguments:
|
||||
input_size: '7'
|
||||
hidden_layer_sizes: '[10]'
|
||||
activation_name: elu
|
||||
output_activation_name: sigmoid
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":360,"width":180,"height":54}'
|
||||
Binarize column using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: d699afd4d7cae862708717cc160f4394ed0c04e536e9515923ef1e8865f01d44
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Fill all missing values using Pandas on CSV data
|
||||
type: CSV
|
||||
column_name: tips
|
||||
predicate: ' > 0'
|
||||
new_column_name: class
|
||||
annotations:
|
||||
editor.position: '{"x":240,"y":360,"width":180,"height":54}'
|
||||
Train pytorch model from csv:
|
||||
componentRef:
|
||||
digest: 40f3185eb61e9727f41a4e0c05dd3d3b44bd802aa0f378cfc31756560033949a
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml
|
||||
arguments:
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Create fully connected pytorch network
|
||||
type: PyTorchScriptModule
|
||||
training_data:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Binarize column using Pandas on CSV data
|
||||
type: CSV
|
||||
label_column_name: class
|
||||
loss_function_name: binary_cross_entropy
|
||||
annotations:
|
||||
editor.position: '{"x":240,"y":490,"width":180,"height":40}'
|
||||
Create PyTorch Model Archive with base handler:
|
||||
componentRef:
|
||||
digest: 8298b5ee1b0f0879f893add4cf352c8dec7cf9e21bb9db134c91a2d046cdb0ec
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/46d51383e6554b7f3ab4fd8cf614d8c2b422fb22/components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml
|
||||
arguments:
|
||||
Model:
|
||||
taskOutput:
|
||||
outputName: trained_model
|
||||
taskId: Train pytorch model from csv
|
||||
type: PyTorchScriptModule
|
||||
Model name: model
|
||||
Model version: '1.0'
|
||||
annotations:
|
||||
editor.position: '{"x":240,"y":590,"width":180,"height":54}'
|
||||
Upload PyTorch model archive to Google Cloud Vertex AI:
|
||||
componentRef:
|
||||
digest: 4450212fae7b9001482aca7eb78b28413c205506eccf08a04e7754a8dfa99004
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
model_archive:
|
||||
taskOutput:
|
||||
outputName: Model archive
|
||||
taskId: Create PyTorch Model Archive with base handler
|
||||
type: PyTorchModelArchive
|
||||
annotations:
|
||||
editor.position: '{"x":240,"y":720,"width":180,"height":70}'
|
||||
outputValues: {}
|
||||
-95
@@ -1,95 +0,0 @@
|
||||
# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
|
||||
from kfp import components
|
||||
|
||||
# %% Loading components
|
||||
download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml")
|
||||
fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
|
||||
binarize_column_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml")
|
||||
create_fully_connected_pytorch_network_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/1a2ef3eeb77bc278f33cad0dd29008ea2431e191/components/PyTorch/Create_fully_connected_network/component.yaml")
|
||||
train_pytorch_model_from_csv_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml")
|
||||
create_pytorch_model_archive_with_base_handler_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/46d51383e6554b7f3ab4fd8cf614d8c2b422fb22/components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml")
|
||||
upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
|
||||
# %% Pipeline definition
|
||||
def train_tabular_classification_model_using_PyTorch_pipeline():
|
||||
dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
|
||||
feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"] # Excluded "trip_total"
|
||||
label_column = "tips"
|
||||
# Deploying the model might incur additional costs over time
|
||||
deploy_model = False
|
||||
|
||||
classification_label_column = "class"
|
||||
all_columns = [label_column] + feature_columns
|
||||
|
||||
training_data = download_from_gcs_op(
|
||||
gcs_path=dataset_gcs_uri
|
||||
).outputs["Data"]
|
||||
|
||||
training_data = select_columns_using_Pandas_on_CSV_data_op(
|
||||
table=training_data,
|
||||
column_names=all_columns,
|
||||
).outputs["transformed_table"]
|
||||
|
||||
# Cleaning the NaN values.
|
||||
training_data = fill_all_missing_values_using_Pandas_on_CSV_data_op(
|
||||
table=training_data,
|
||||
replacement_value="0",
|
||||
#replacement_type_name="float",
|
||||
).outputs["transformed_table"]
|
||||
|
||||
classification_training_data = binarize_column_using_Pandas_on_CSV_data_op(
|
||||
table=training_data,
|
||||
column_name=label_column,
|
||||
predicate=" > 0",
|
||||
new_column_name=classification_label_column,
|
||||
).outputs["transformed_table"]
|
||||
|
||||
network = create_fully_connected_pytorch_network_op(
|
||||
input_size=len(feature_columns),
|
||||
# Optional:
|
||||
hidden_layer_sizes=[10],
|
||||
activation_name="elu",
|
||||
output_activation_name="sigmoid",
|
||||
# output_size=1,
|
||||
).outputs["model"]
|
||||
|
||||
model = train_pytorch_model_from_csv_op(
|
||||
model=network,
|
||||
training_data=classification_training_data,
|
||||
label_column_name=classification_label_column,
|
||||
loss_function_name="binary_cross_entropy",
|
||||
# Optional:
|
||||
#number_of_epochs=1,
|
||||
#learning_rate=0.1,
|
||||
#optimizer_name="Adadelta",
|
||||
#optimizer_parameters={},
|
||||
#batch_size=32,
|
||||
#batch_log_interval=100,
|
||||
#random_seed=0,
|
||||
).outputs["trained_model"]
|
||||
|
||||
model_archive = create_pytorch_model_archive_with_base_handler_op(
|
||||
model=model,
|
||||
# Optional:
|
||||
# model_name="model",
|
||||
# model_version="1.0",
|
||||
).outputs["Model archive"]
|
||||
|
||||
vertex_model_name = upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op(
|
||||
model_archive=model_archive,
|
||||
).outputs["model_name"]
|
||||
|
||||
# Deploying the model might incur additional costs over time
|
||||
if deploy_model:
|
||||
vertex_endpoint_name = deploy_model_to_endpoint_op(
|
||||
model_name=vertex_model_name,
|
||||
).outputs["endpoint_name"]
|
||||
|
||||
pipeline_func=train_tabular_classification_model_using_PyTorch_pipeline
|
||||
|
||||
# %% Pipeline submission
|
||||
if __name__ == '__main__':
|
||||
from google.cloud import aiplatform
|
||||
aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()
|
||||
-132
@@ -1,132 +0,0 @@
|
||||
name: Train tabular classification model using TensorFlow pipeline
|
||||
metadata:
|
||||
annotations:
|
||||
author: Alexey Volkov <alexey.volkov@ark-kun.com>
|
||||
canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_classification_model_using_TensorFlow_and_import_to_Vertex_AI/pipeline.component.yaml
|
||||
sdk: https://cloud-pipelines.net/pipeline-editor/
|
||||
implementation:
|
||||
graph:
|
||||
tasks:
|
||||
Download from GCS:
|
||||
componentRef:
|
||||
digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":40,"width":180,"height":40}'
|
||||
Select columns using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: Data
|
||||
taskId: Download from GCS
|
||||
column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":140,"width":180,"height":54}'
|
||||
Fill all missing values using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Select columns using Pandas on CSV data
|
||||
type: CSV
|
||||
replacement_value: '0'
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":250,"width":180,"height":54}'
|
||||
Binarize column using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: d699afd4d7cae862708717cc160f4394ed0c04e536e9515923ef1e8865f01d44
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Fill all missing values using Pandas on CSV data
|
||||
type: CSV
|
||||
column_name: tips
|
||||
predicate: ' > 0'
|
||||
new_column_name: class
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":370,"width":180,"height":54}'
|
||||
Split rows into subsets:
|
||||
componentRef:
|
||||
digest: a609c3c9196484290f24a1174955f95b27f07a7b458aa5cb8cde28866cb2cb46
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Binarize column using Pandas on CSV data
|
||||
type: CSV
|
||||
fraction_1: '0.8'
|
||||
annotations:
|
||||
editor.position: '{"x":170,"y":500,"width":180,"height":40}'
|
||||
Create fully connected tensorflow network:
|
||||
componentRef:
|
||||
digest: bfcafbc5ce711b1f69cabf1338212d10d50136a73db9f9f7c984de7b80b4bfb0
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/9ca0f9eecf5f896f65b8538bbd809747052617d1/components/tensorflow/Create_fully_connected_network/component.yaml
|
||||
arguments:
|
||||
input_size: '7'
|
||||
hidden_layer_sizes: '[10]'
|
||||
activation_name: elu
|
||||
output_activation_name: sigmoid
|
||||
annotations:
|
||||
editor.position: '{"x":370,"y":500,"width":180,"height":54}'
|
||||
Train model using Keras on CSV:
|
||||
componentRef:
|
||||
digest: 42ae60c889034dbad74815653e95b4f7d576b5f47f803173e8679c7b54984609
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c504a4010348c50eaaf6d4337586ccc008f4dcef/components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml
|
||||
arguments:
|
||||
training_data:
|
||||
taskOutput:
|
||||
outputName: split_1
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Create fully connected tensorflow network
|
||||
type: TensorflowSavedModel
|
||||
label_column_name: class
|
||||
loss_function_name: binary_crossentropy
|
||||
number_of_epochs: '10'
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":620,"width":180,"height":54}'
|
||||
Upload Tensorflow model to Google Cloud Vertex AI:
|
||||
componentRef:
|
||||
digest: 2e45263ff640b1a688e359b6936e27a81b2407749a84f340af2aa5547e0cb92c
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: trained_model
|
||||
taskId: Train model using Keras on CSV
|
||||
type: TensorflowSavedModel
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":750,"width":180,"height":54}'
|
||||
Predict with TensorFlow model on CSV data:
|
||||
componentRef:
|
||||
digest: 921bb1563e93a78233b8acceab87055b9154ccf5595d056028cf0396ca224cd4
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/59c759ce6f543184e30db6817d2a703879bc0f39/components/tensorflow/Predict/on_CSV/component.yaml
|
||||
arguments:
|
||||
dataset:
|
||||
taskOutput:
|
||||
outputName: split_2
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: trained_model
|
||||
taskId: Train model using Keras on CSV
|
||||
type: TensorflowSavedModel
|
||||
label_column_name: class
|
||||
annotations:
|
||||
editor.position: '{"x":240,"y":750,"width":180,"height":54}'
|
||||
outputValues: {}
|
||||
-106
@@ -1,106 +0,0 @@
|
||||
# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
|
||||
from kfp import components
|
||||
|
||||
# %% Loading components
|
||||
download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml")
|
||||
fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
|
||||
binarize_column_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml")
|
||||
split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
|
||||
create_fully_connected_tensorflow_network_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/9ca0f9eecf5f896f65b8538bbd809747052617d1/components/tensorflow/Create_fully_connected_network/component.yaml")
|
||||
train_model_using_Keras_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c504a4010348c50eaaf6d4337586ccc008f4dcef/components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml")
|
||||
predict_with_TensorFlow_model_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/59c759ce6f543184e30db6817d2a703879bc0f39/components/tensorflow/Predict/on_CSV/component.yaml")
|
||||
upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
|
||||
# %% Pipeline definition
|
||||
def train_tabular_classification_model_using_TensorFlow_pipeline():
|
||||
dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
|
||||
feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"] # Excluded "trip_total"
|
||||
label_column = "tips"
|
||||
training_set_fraction = 0.8
|
||||
# Deploying the model might incur additional costs over time
|
||||
deploy_model = False
|
||||
|
||||
classification_label_column = "class"
|
||||
all_columns = [label_column] + feature_columns
|
||||
|
||||
dataset = download_from_gcs_op(
|
||||
gcs_path=dataset_gcs_uri
|
||||
).outputs["Data"]
|
||||
|
||||
dataset = select_columns_using_Pandas_on_CSV_data_op(
|
||||
table=dataset,
|
||||
column_names=all_columns,
|
||||
).outputs["transformed_table"]
|
||||
|
||||
dataset = fill_all_missing_values_using_Pandas_on_CSV_data_op(
|
||||
table=dataset,
|
||||
replacement_value="0",
|
||||
# # Optional:
|
||||
# column_names=None, # =[...]
|
||||
).outputs["transformed_table"]
|
||||
|
||||
classification_dataset = binarize_column_using_Pandas_on_CSV_data_op(
|
||||
table=dataset,
|
||||
column_name=label_column,
|
||||
predicate=" > 0",
|
||||
new_column_name=classification_label_column,
|
||||
).outputs["transformed_table"]
|
||||
|
||||
split_task = split_rows_into_subsets_op(
|
||||
table=classification_dataset,
|
||||
fraction_1=training_set_fraction,
|
||||
)
|
||||
classification_training_data = split_task.outputs["split_1"]
|
||||
classification_testing_data = split_task.outputs["split_2"]
|
||||
|
||||
network = create_fully_connected_tensorflow_network_op(
|
||||
input_size=len(feature_columns),
|
||||
# Optional:
|
||||
hidden_layer_sizes=[10],
|
||||
activation_name="elu",
|
||||
output_activation_name="sigmoid",
|
||||
# output_size=1,
|
||||
).outputs["model"]
|
||||
|
||||
model = train_model_using_Keras_on_CSV_op(
|
||||
training_data=classification_training_data,
|
||||
model=network,
|
||||
label_column_name=classification_label_column,
|
||||
# Optional:
|
||||
loss_function_name="binary_crossentropy",
|
||||
number_of_epochs=10,
|
||||
#learning_rate=0.1,
|
||||
#optimizer_name="Adadelta",
|
||||
#optimizer_parameters={},
|
||||
#batch_size=32,
|
||||
#metric_names=["mean_absolute_error"],
|
||||
#random_seed=0,
|
||||
).outputs["trained_model"]
|
||||
|
||||
predictions = predict_with_TensorFlow_model_on_CSV_data_op(
|
||||
dataset=classification_testing_data,
|
||||
model=model,
|
||||
# label_column_name needs to be set when doing prediction on a dataset that has labels
|
||||
label_column_name=classification_label_column,
|
||||
# Optional:
|
||||
# batch_size=1000,
|
||||
).outputs["predictions"]
|
||||
|
||||
vertex_model_name = upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op(
|
||||
model=model,
|
||||
).outputs["model_name"]
|
||||
|
||||
# Deploying the model might incur additional costs over time
|
||||
if deploy_model:
|
||||
vertex_endpoint_name = deploy_model_to_endpoint_op(
|
||||
model_name=vertex_model_name,
|
||||
).outputs["endpoint_name"]
|
||||
|
||||
pipeline_func = train_tabular_classification_model_using_TensorFlow_pipeline
|
||||
|
||||
# %% Pipeline submission
|
||||
if __name__ == '__main__':
|
||||
from google.cloud import aiplatform
|
||||
aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()
|
||||
-115
@@ -1,115 +0,0 @@
|
||||
name: Train tabular classification model using XGBoost pipeline
|
||||
metadata:
|
||||
annotations:
|
||||
author: Alexey Volkov <alexey.volkov@ark-kun.com>
|
||||
canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_classification_model_using_XGBoost_and_import_to_Vertex_AI/pipeline.component.yaml
|
||||
sdk: https://cloud-pipelines.net/pipeline-editor/
|
||||
implementation:
|
||||
graph:
|
||||
tasks:
|
||||
Download from GCS:
|
||||
componentRef:
|
||||
digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":40,"width":180,"height":40}'
|
||||
Select columns using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: Data
|
||||
taskId: Download from GCS
|
||||
column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":140,"width":180,"height":54}'
|
||||
Fill all missing values using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Select columns using Pandas on CSV data
|
||||
type: CSV
|
||||
replacement_value: '0'
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":250,"width":180,"height":54}'
|
||||
Binarize column using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: d699afd4d7cae862708717cc160f4394ed0c04e536e9515923ef1e8865f01d44
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Fill all missing values using Pandas on CSV data
|
||||
type: CSV
|
||||
column_name: tips
|
||||
predicate: '> 0'
|
||||
new_column_name: class
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":380,"width":180,"height":54}'
|
||||
Split rows into subsets:
|
||||
componentRef:
|
||||
digest: a609c3c9196484290f24a1174955f95b27f07a7b458aa5cb8cde28866cb2cb46
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Binarize column using Pandas on CSV data
|
||||
type: CSV
|
||||
fraction_1: '0.8'
|
||||
annotations:
|
||||
editor.position: '{"x":170,"y":510,"width":180,"height":40}'
|
||||
Train XGBoost model on CSV:
|
||||
componentRef:
|
||||
digest: 538c5a01eb38deaf532d619f0bbeaff4efc550fe1f0f776fc06791097b68ceac
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/58d3a47f904f32a64af8403330ba7e2134cae46d/components/XGBoost/Train/component.yaml
|
||||
arguments:
|
||||
training_data:
|
||||
taskOutput:
|
||||
outputName: split_1
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
label_column_name: class
|
||||
objective: binary:logistic
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":630,"width":180,"height":40}'
|
||||
Upload XGBoost model to Google Cloud Vertex AI:
|
||||
componentRef:
|
||||
digest: 5a5a273c403670743820986c03a4175b7cb4595a556524fefcce403656286977
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Train XGBoost model on CSV
|
||||
type: XGBoostModel
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":750,"width":180,"height":54}'
|
||||
Xgboost predict on CSV:
|
||||
componentRef:
|
||||
digest: 0876233a0c7306fefec188bd70f059b46d1fb5aa57be231799570e3bbbdd0d95
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/4694ec97baccf59284c2a1db4aa2250c22291eab/components/XGBoost/Predict/component.yaml
|
||||
arguments:
|
||||
data:
|
||||
taskOutput:
|
||||
outputName: split_2
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Train XGBoost model on CSV
|
||||
type: XGBoostModel
|
||||
label_column_name: class
|
||||
annotations:
|
||||
editor.position: '{"x":240,"y":750,"width":180,"height":40}'
|
||||
outputValues: {}
|
||||
-94
@@ -1,94 +0,0 @@
|
||||
# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
|
||||
from kfp import components
|
||||
|
||||
# %% Loading components
|
||||
download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml")
|
||||
fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
|
||||
binarize_column_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml")
|
||||
split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
|
||||
train_XGBoost_model_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/58d3a47f904f32a64af8403330ba7e2134cae46d/components/XGBoost/Train/component.yaml")
|
||||
xgboost_predict_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/4694ec97baccf59284c2a1db4aa2250c22291eab/components/XGBoost/Predict/component.yaml")
|
||||
upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
|
||||
# %% Pipeline definition
|
||||
def train_tabular_classification_model_using_XGBoost_pipeline():
|
||||
dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
|
||||
feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"] # Excluded "trip_total"
|
||||
label_column = "tips"
|
||||
training_set_fraction = 0.8
|
||||
# Deploying the model might incur additional costs over time
|
||||
deploy_model = False
|
||||
|
||||
classification_label_column = "class"
|
||||
all_columns = [label_column] + feature_columns
|
||||
|
||||
dataset = download_from_gcs_op(
|
||||
gcs_path=dataset_gcs_uri
|
||||
).outputs["Data"]
|
||||
|
||||
dataset = select_columns_using_Pandas_on_CSV_data_op(
|
||||
table=dataset,
|
||||
column_names=all_columns,
|
||||
).outputs["transformed_table"]
|
||||
|
||||
dataset = fill_all_missing_values_using_Pandas_on_CSV_data_op(
|
||||
table=dataset,
|
||||
replacement_value="0",
|
||||
# # Optional:
|
||||
# column_names=None, # =[...]
|
||||
).outputs["transformed_table"]
|
||||
|
||||
classification_dataset = binarize_column_using_Pandas_on_CSV_data_op(
|
||||
table=dataset,
|
||||
column_name=label_column,
|
||||
predicate="> 0",
|
||||
new_column_name=classification_label_column,
|
||||
).outputs["transformed_table"]
|
||||
|
||||
split_task = split_rows_into_subsets_op(
|
||||
table=classification_dataset,
|
||||
fraction_1=training_set_fraction,
|
||||
)
|
||||
classification_training_data = split_task.outputs["split_1"]
|
||||
classification_testing_data = split_task.outputs["split_2"]
|
||||
|
||||
model = train_XGBoost_model_on_CSV_op(
|
||||
training_data=classification_training_data,
|
||||
label_column_name=classification_label_column,
|
||||
objective="binary:logistic",
|
||||
# Optional:
|
||||
#starting_model=None,
|
||||
#num_iterations=10,
|
||||
#booster_params={},
|
||||
#booster="gbtree",
|
||||
#learning_rate=0.3,
|
||||
#min_split_loss=0,
|
||||
#max_depth=6,
|
||||
).outputs["model"]
|
||||
|
||||
# Predicting on the testing data
|
||||
predictions = xgboost_predict_on_CSV_op(
|
||||
data=classification_testing_data,
|
||||
model=model,
|
||||
# label_column needs to be set when doing prediction on a dataset that has labels
|
||||
label_column_name=classification_label_column,
|
||||
).outputs["predictions"]
|
||||
|
||||
vertex_model_name = upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op(
|
||||
model=model,
|
||||
).outputs["model_name"]
|
||||
|
||||
# Deploying the model might incur additional costs over time
|
||||
if deploy_model:
|
||||
vertex_endpoint_name = deploy_model_to_endpoint_op(
|
||||
model_name=vertex_model_name,
|
||||
).outputs["endpoint_name"]
|
||||
|
||||
pipeline_func = train_tabular_classification_model_using_XGBoost_pipeline
|
||||
|
||||
# %% Pipeline submission
|
||||
if __name__ == '__main__':
|
||||
from google.cloud import aiplatform
|
||||
aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()
|
||||
-257
@@ -1,257 +0,0 @@
|
||||
name: Train tabular classification model using all frameworks pipeline
|
||||
metadata:
|
||||
annotations:
|
||||
author: Alexey Volkov <alexey.volkov@ark-kun.com>
|
||||
canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_classification_model_using_all_frameworks_and_import_to_Vertex_AI/pipeline.component.yaml
|
||||
sdk: https://cloud-pipelines.net/pipeline-editor/
|
||||
implementation:
|
||||
graph:
|
||||
tasks:
|
||||
Download from GCS:
|
||||
componentRef:
|
||||
digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
|
||||
annotations:
|
||||
editor.position: '{"x":550,"y":40,"width":180,"height":40}'
|
||||
Select columns using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: Data
|
||||
taskId: Download from GCS
|
||||
column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
|
||||
annotations:
|
||||
editor.position: '{"x":550,"y":140,"width":180,"height":54}'
|
||||
Fill all missing values using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Select columns using Pandas on CSV data
|
||||
type: CSV
|
||||
replacement_value: '0'
|
||||
annotations:
|
||||
editor.position: '{"x":550,"y":250,"width":180,"height":54}'
|
||||
Binarize column using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: d699afd4d7cae862708717cc160f4394ed0c04e536e9515923ef1e8865f01d44
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Fill all missing values using Pandas on CSV data
|
||||
type: CSV
|
||||
column_name: tips
|
||||
predicate: ' > 0'
|
||||
new_column_name: class
|
||||
annotations:
|
||||
editor.position: '{"x":550,"y":380,"width":180,"height":54}'
|
||||
Split rows into subsets:
|
||||
componentRef:
|
||||
digest: a609c3c9196484290f24a1174955f95b27f07a7b458aa5cb8cde28866cb2cb46
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Binarize column using Pandas on CSV data
|
||||
type: CSV
|
||||
fraction_1: '0.8'
|
||||
annotations:
|
||||
editor.position: '{"x":550,"y":490,"width":180,"height":40}'
|
||||
Create fully connected pytorch network:
|
||||
componentRef:
|
||||
digest: d03d8248fd358a0275ec33568ee7dd7dce576cc112b09dfafe2651e4d97e04a9
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1a2ef3eeb77bc278f33cad0dd29008ea2431e191/components/PyTorch/Create_fully_connected_network/component.yaml
|
||||
arguments:
|
||||
input_size: '7'
|
||||
hidden_layer_sizes: '[10]'
|
||||
activation_name: elu
|
||||
output_activation_name: sigmoid
|
||||
annotations:
|
||||
editor.position: '{"x":380,"y":620,"width":180,"height":54}'
|
||||
Create fully connected tensorflow network:
|
||||
componentRef:
|
||||
digest: bfcafbc5ce711b1f69cabf1338212d10d50136a73db9f9f7c984de7b80b4bfb0
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/9ca0f9eecf5f896f65b8538bbd809747052617d1/components/tensorflow/Create_fully_connected_network/component.yaml
|
||||
arguments:
|
||||
input_size: '7'
|
||||
hidden_layer_sizes: '[10]'
|
||||
activation_name: elu
|
||||
output_activation_name: sigmoid
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":630,"width":180,"height":54}'
|
||||
Train model using Keras on CSV:
|
||||
componentRef:
|
||||
digest: 42ae60c889034dbad74815653e95b4f7d576b5f47f803173e8679c7b54984609
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c504a4010348c50eaaf6d4337586ccc008f4dcef/components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml
|
||||
arguments:
|
||||
training_data:
|
||||
taskOutput:
|
||||
outputName: split_1
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Create fully connected tensorflow network
|
||||
type: TensorflowSavedModel
|
||||
label_column_name: class
|
||||
loss_function_name: binary_crossentropy
|
||||
number_of_epochs: '10'
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":750,"width":180,"height":54}'
|
||||
Train pytorch model from csv:
|
||||
componentRef:
|
||||
digest: 40f3185eb61e9727f41a4e0c05dd3d3b44bd802aa0f378cfc31756560033949a
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml
|
||||
arguments:
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Create fully connected pytorch network
|
||||
type: PyTorchScriptModule
|
||||
training_data:
|
||||
taskOutput:
|
||||
outputName: split_1
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
label_column_name: class
|
||||
loss_function_name: binary_cross_entropy
|
||||
annotations:
|
||||
editor.position: '{"x":380,"y":750,"width":180,"height":40}'
|
||||
Train XGBoost model on CSV:
|
||||
componentRef:
|
||||
digest: 538c5a01eb38deaf532d619f0bbeaff4efc550fe1f0f776fc06791097b68ceac
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/58d3a47f904f32a64af8403330ba7e2134cae46d/components/XGBoost/Train/component.yaml
|
||||
arguments:
|
||||
training_data:
|
||||
taskOutput:
|
||||
outputName: split_1
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
label_column_name: class
|
||||
objective: binary:logistic
|
||||
annotations:
|
||||
editor.position: '{"x":720,"y":750,"width":180,"height":40}'
|
||||
Train logistic regression model using scikit learn from CSV:
|
||||
componentRef:
|
||||
digest: a864625a822e4b1c8ef6fe4ae1454fd90f15438f70a6712bb4c30e0dda4d35b7
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/cb44b75c9c062fcc40c2b905b2024b4493dbc62b/components/ML_frameworks/Scikit_learn/Train_logistic_regression_model/from_CSV/component.yaml
|
||||
arguments:
|
||||
dataset:
|
||||
taskOutput:
|
||||
outputName: split_1
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
label_column_name: class
|
||||
annotations:
|
||||
editor.position: '{"x":1030,"y":750,"width":180,"height":70}'
|
||||
Predict with TensorFlow model on CSV data:
|
||||
componentRef:
|
||||
digest: 921bb1563e93a78233b8acceab87055b9154ccf5595d056028cf0396ca224cd4
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/59c759ce6f543184e30db6817d2a703879bc0f39/components/tensorflow/Predict/on_CSV/component.yaml
|
||||
arguments:
|
||||
dataset:
|
||||
taskOutput:
|
||||
outputName: split_2
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: trained_model
|
||||
taskId: Train model using Keras on CSV
|
||||
type: TensorflowSavedModel
|
||||
label_column_name: class
|
||||
annotations:
|
||||
editor.position: '{"x":160,"y":880,"width":180,"height":54}'
|
||||
Create PyTorch Model Archive with base handler:
|
||||
componentRef:
|
||||
digest: 8298b5ee1b0f0879f893add4cf352c8dec7cf9e21bb9db134c91a2d046cdb0ec
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/46d51383e6554b7f3ab4fd8cf614d8c2b422fb22/components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml
|
||||
arguments:
|
||||
Model:
|
||||
taskOutput:
|
||||
outputName: trained_model
|
||||
taskId: Train pytorch model from csv
|
||||
type: PyTorchScriptModule
|
||||
Model name: model
|
||||
Model version: '1.0'
|
||||
annotations:
|
||||
editor.position: '{"x":380,"y":880,"width":180,"height":54}'
|
||||
Xgboost predict on CSV:
|
||||
componentRef:
|
||||
digest: 0876233a0c7306fefec188bd70f059b46d1fb5aa57be231799570e3bbbdd0d95
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/4694ec97baccf59284c2a1db4aa2250c22291eab/components/XGBoost/Predict/component.yaml
|
||||
arguments:
|
||||
data:
|
||||
taskOutput:
|
||||
outputName: split_2
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Train XGBoost model on CSV
|
||||
type: XGBoostModel
|
||||
label_column_name: class
|
||||
annotations:
|
||||
editor.position: '{"x":810,"y":880,"width":180,"height":40}'
|
||||
Upload Scikit learn pickle model to Google Cloud Vertex AI:
|
||||
componentRef:
|
||||
digest: 81c91c8d7d21ec97e0872f669d68bd89edea87279d703685db54aa94743bebcd
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Train logistic regression model using scikit learn from CSV
|
||||
type: ScikitLearnPickleModel
|
||||
annotations:
|
||||
editor.position: '{"x":1030,"y":880,"width":180,"height":70}'
|
||||
Upload Tensorflow model to Google Cloud Vertex AI:
|
||||
componentRef:
|
||||
digest: 2e45263ff640b1a688e359b6936e27a81b2407749a84f340af2aa5547e0cb92c
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: trained_model
|
||||
taskId: Train model using Keras on CSV
|
||||
type: TensorflowSavedModel
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":1010,"width":180,"height":54}'
|
||||
Upload PyTorch model archive to Google Cloud Vertex AI:
|
||||
componentRef:
|
||||
digest: 4450212fae7b9001482aca7eb78b28413c205506eccf08a04e7754a8dfa99004
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
model_archive:
|
||||
taskOutput:
|
||||
outputName: Model archive
|
||||
taskId: Create PyTorch Model Archive with base handler
|
||||
type: PyTorchModelArchive
|
||||
annotations:
|
||||
editor.position: '{"x":380,"y":1010,"width":180,"height":70}'
|
||||
Upload XGBoost model to Google Cloud Vertex AI:
|
||||
componentRef:
|
||||
digest: 5a5a273c403670743820986c03a4175b7cb4595a556524fefcce403656286977
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Train XGBoost model on CSV
|
||||
type: XGBoostModel
|
||||
annotations:
|
||||
editor.position: '{"x":720,"y":1010,"width":180,"height":54}'
|
||||
outputValues: {}
|
||||
-224
@@ -1,224 +0,0 @@
|
||||
# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
|
||||
from kfp import components
|
||||
|
||||
# %% Loading components
|
||||
download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml")
|
||||
fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
|
||||
binarize_column_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml")
|
||||
split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
|
||||
|
||||
# TensorFlow
|
||||
create_fully_connected_tensorflow_network_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/9ca0f9eecf5f896f65b8538bbd809747052617d1/components/tensorflow/Create_fully_connected_network/component.yaml")
|
||||
train_model_using_Keras_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c504a4010348c50eaaf6d4337586ccc008f4dcef/components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml")
|
||||
predict_with_TensorFlow_model_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/59c759ce6f543184e30db6817d2a703879bc0f39/components/tensorflow/Predict/on_CSV/component.yaml")
|
||||
upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
|
||||
# PyTorch
|
||||
create_fully_connected_pytorch_network_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/1a2ef3eeb77bc278f33cad0dd29008ea2431e191/components/PyTorch/Create_fully_connected_network/component.yaml")
|
||||
train_pytorch_model_from_csv_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml")
|
||||
create_pytorch_model_archive_with_base_handler_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/46d51383e6554b7f3ab4fd8cf614d8c2b422fb22/components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml")
|
||||
upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
|
||||
# XGBoost
|
||||
train_XGBoost_model_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/58d3a47f904f32a64af8403330ba7e2134cae46d/components/XGBoost/Train/component.yaml")
|
||||
xgboost_predict_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/4694ec97baccf59284c2a1db4aa2250c22291eab/components/XGBoost/Predict/component.yaml")
|
||||
upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
|
||||
# Scikit-learn
|
||||
#train_linear_regression_model_using_scikit_learn_from_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/f807e02b54d4886c65a05f40848fd51c72407f40/components/ML_frameworks/Scikit_learn/Train_linear_regression_model/from_CSV/component.yaml")
|
||||
train_logistic_regression_model_using_scikit_learn_from_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/cb44b75c9c062fcc40c2b905b2024b4493dbc62b/components/ML_frameworks/Scikit_learn/Train_logistic_regression_model/from_CSV/component.yaml")
|
||||
upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
|
||||
# Vertex AI
|
||||
deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
|
||||
# %% Pipeline definition
|
||||
def train_tabular_classification_model_using_all_frameworks_pipeline():
|
||||
dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
|
||||
feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"] # Excluded "trip_total"
|
||||
label_column = "tips"
|
||||
training_set_fraction = 0.8
|
||||
# Deploying the model might incur additional costs over time
|
||||
deploy_model = False
|
||||
|
||||
classification_label_column = "class"
|
||||
all_columns = [label_column] + feature_columns
|
||||
|
||||
dataset = download_from_gcs_op(
|
||||
gcs_path=dataset_gcs_uri
|
||||
).outputs["Data"]
|
||||
|
||||
dataset = select_columns_using_Pandas_on_CSV_data_op(
|
||||
table=dataset,
|
||||
column_names=all_columns,
|
||||
).outputs["transformed_table"]
|
||||
|
||||
dataset = fill_all_missing_values_using_Pandas_on_CSV_data_op(
|
||||
table=dataset,
|
||||
replacement_value="0",
|
||||
# # Optional:
|
||||
# column_names=None, # =[...]
|
||||
).outputs["transformed_table"]
|
||||
|
||||
classification_dataset = binarize_column_using_Pandas_on_CSV_data_op(
|
||||
table=dataset,
|
||||
column_name=label_column,
|
||||
predicate=" > 0",
|
||||
new_column_name=classification_label_column,
|
||||
).outputs["transformed_table"]
|
||||
|
||||
split_task = split_rows_into_subsets_op(
|
||||
table=classification_dataset,
|
||||
fraction_1=training_set_fraction,
|
||||
)
|
||||
classification_training_data = split_task.outputs["split_1"]
|
||||
classification_testing_data = split_task.outputs["split_2"]
|
||||
|
||||
# TensorFlow
|
||||
tensorflow_network = create_fully_connected_tensorflow_network_op(
|
||||
input_size=len(feature_columns),
|
||||
# Optional:
|
||||
hidden_layer_sizes=[10],
|
||||
activation_name="elu",
|
||||
output_activation_name="sigmoid",
|
||||
# output_size=1,
|
||||
).outputs["model"]
|
||||
|
||||
tensorflow_model = train_model_using_Keras_on_CSV_op(
|
||||
training_data=classification_training_data,
|
||||
model=tensorflow_network,
|
||||
label_column_name=classification_label_column,
|
||||
# Optional:
|
||||
loss_function_name="binary_crossentropy",
|
||||
number_of_epochs=10,
|
||||
#learning_rate=0.1,
|
||||
#optimizer_name="Adadelta",
|
||||
#optimizer_parameters={},
|
||||
#batch_size=32,
|
||||
#metric_names=["mean_absolute_error"],
|
||||
#random_seed=0,
|
||||
).outputs["trained_model"]
|
||||
|
||||
tensorflow_predictions = predict_with_TensorFlow_model_on_CSV_data_op(
|
||||
dataset=classification_testing_data,
|
||||
model=tensorflow_model,
|
||||
# label_column_name needs to be set when doing prediction on a dataset that has labels
|
||||
label_column_name=classification_label_column,
|
||||
# Optional:
|
||||
# batch_size=1000,
|
||||
).outputs["predictions"]
|
||||
|
||||
tensorflow_vertex_model_name = upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op(
|
||||
model=tensorflow_model,
|
||||
).outputs["model_name"]
|
||||
|
||||
# Deploying the model might incur additional costs over time
|
||||
if deploy_model:
|
||||
tensorflow_vertex_endpoint_name = deploy_model_to_endpoint_op(
|
||||
model_name=tensorflow_vertex_model_name,
|
||||
).outputs["endpoint_name"]
|
||||
|
||||
# PyTorch
|
||||
pytorch_network = create_fully_connected_pytorch_network_op(
|
||||
input_size=len(feature_columns),
|
||||
# Optional:
|
||||
hidden_layer_sizes=[10],
|
||||
activation_name="elu",
|
||||
output_activation_name="sigmoid",
|
||||
# output_size=1,
|
||||
).outputs["model"]
|
||||
|
||||
pytorch_model = train_pytorch_model_from_csv_op(
|
||||
model=pytorch_network,
|
||||
training_data=classification_training_data,
|
||||
label_column_name=classification_label_column,
|
||||
loss_function_name="binary_cross_entropy",
|
||||
# Optional:
|
||||
#number_of_epochs=1,
|
||||
#learning_rate=0.1,
|
||||
#optimizer_name="Adadelta",
|
||||
#optimizer_parameters={},
|
||||
#batch_size=32,
|
||||
#batch_log_interval=100,
|
||||
#random_seed=0,
|
||||
).outputs["trained_model"]
|
||||
|
||||
pytorch_model_archive = create_pytorch_model_archive_with_base_handler_op(
|
||||
model=pytorch_model,
|
||||
# Optional:
|
||||
# model_name="model",
|
||||
# model_version="1.0",
|
||||
).outputs["Model archive"]
|
||||
|
||||
pytorch_vertex_model_name = upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op(
|
||||
model_archive=pytorch_model_archive,
|
||||
).outputs["model_name"]
|
||||
|
||||
# Deploying the model might incur additional costs over time
|
||||
if deploy_model:
|
||||
pytorch_vertex_endpoint_name = deploy_model_to_endpoint_op(
|
||||
model_name=pytorch_vertex_model_name,
|
||||
).outputs["endpoint_name"]
|
||||
|
||||
# XGBoost
|
||||
xgboost_model = train_XGBoost_model_on_CSV_op(
|
||||
training_data=classification_training_data,
|
||||
label_column_name=classification_label_column,
|
||||
objective="binary:logistic",
|
||||
# Optional:
|
||||
#starting_model=None,
|
||||
#num_iterations=10,
|
||||
#booster_params={},
|
||||
#booster="gbtree",
|
||||
#learning_rate=0.3,
|
||||
#min_split_loss=0,
|
||||
#max_depth=6,
|
||||
).outputs["model"]
|
||||
|
||||
# Predicting on the testing data
|
||||
xgboost_predictions = xgboost_predict_on_CSV_op(
|
||||
data=classification_testing_data,
|
||||
model=xgboost_model,
|
||||
# label_column needs to be set when doing prediction on a dataset that has labels
|
||||
label_column_name=classification_label_column,
|
||||
).outputs["predictions"]
|
||||
|
||||
xgboost_vertex_model_name = upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op(
|
||||
model=xgboost_model,
|
||||
).outputs["model_name"]
|
||||
|
||||
# Deploying the model might incur additional costs over time
|
||||
if deploy_model:
|
||||
xgboost_vertex_endpoint_name = deploy_model_to_endpoint_op(
|
||||
model_name=xgboost_vertex_model_name,
|
||||
).outputs["endpoint_name"]
|
||||
|
||||
# Scikit-learn
|
||||
sklearn_model = train_logistic_regression_model_using_scikit_learn_from_CSV_op(
|
||||
dataset=classification_training_data,
|
||||
label_column_name=classification_label_column,
|
||||
# Optional:
|
||||
#penalty="l2",
|
||||
#solver="lbfgs",
|
||||
#max_iterations=100,
|
||||
#multi_class_mode="auto",
|
||||
#random_seed=0,
|
||||
).outputs["model"]
|
||||
|
||||
sklearn_vertex_model_name = upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op(
|
||||
model=sklearn_model,
|
||||
).outputs["model_name"]
|
||||
|
||||
# Deploying the model might incur additional costs over time
|
||||
if deploy_model:
|
||||
sklearn_vertex_endpoint_name = deploy_model_to_endpoint_op(
|
||||
model_name=sklearn_vertex_model_name,
|
||||
).outputs["endpoint_name"]
|
||||
|
||||
pipeline_func=train_tabular_classification_model_using_all_frameworks_pipeline
|
||||
|
||||
# %% Pipeline submission
|
||||
if __name__ == '__main__':
|
||||
from google.cloud import aiplatform
|
||||
aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()
|
||||
-68
@@ -1,68 +0,0 @@
|
||||
name: Train tabular regression linear model using Scikit learn pipeline
|
||||
metadata:
|
||||
annotations:
|
||||
author: Alexey Volkov <alexey.volkov@ark-kun.com>
|
||||
canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_regression_linear_model_using_Scikit_learn_and_import_to_Vertex_AI/pipeline.component.yaml
|
||||
sdk: https://cloud-pipelines.net/pipeline-editor/
|
||||
implementation:
|
||||
graph:
|
||||
tasks:
|
||||
Download from GCS:
|
||||
componentRef:
|
||||
digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":40,"width":180,"height":40}'
|
||||
Select columns using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: Data
|
||||
taskId: Download from GCS
|
||||
column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":140,"width":180,"height":54}'
|
||||
Fill all missing values using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Select columns using Pandas on CSV data
|
||||
type: CSV
|
||||
replacement_value: '0'
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":250,"width":180,"height":54}'
|
||||
Train linear regression model using scikit learn from CSV:
|
||||
componentRef:
|
||||
digest: c7fe7912ab0d1fb45d201d452e9ce6be5544e7d8c6d229db7a4b931ff58560f3
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/f807e02b54d4886c65a05f40848fd51c72407f40/components/ML_frameworks/Scikit_learn/Train_linear_regression_model/from_CSV/component.yaml
|
||||
arguments:
|
||||
dataset:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Fill all missing values using Pandas on CSV data
|
||||
type: CSV
|
||||
label_column_name: tips
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":360,"width":180,"height":54}'
|
||||
Upload Scikit learn pickle model to Google Cloud Vertex AI:
|
||||
componentRef:
|
||||
digest: 81c91c8d7d21ec97e0872f669d68bd89edea87279d703685db54aa94743bebcd
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Train linear regression model using scikit learn from CSV
|
||||
type: ScikitLearnPickleModel
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":490,"width":180,"height":70}'
|
||||
outputValues: {}
|
||||
-57
@@ -1,57 +0,0 @@
|
||||
# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
|
||||
from kfp import components
|
||||
|
||||
# %% Loading components
|
||||
download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml")
|
||||
fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
|
||||
train_linear_regression_model_using_scikit_learn_from_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/f807e02b54d4886c65a05f40848fd51c72407f40/components/ML_frameworks/Scikit_learn/Train_linear_regression_model/from_CSV/component.yaml")
|
||||
upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
|
||||
# %% Pipeline definition
|
||||
def train_tabular_regression_linear_model_using_Scikit_learn_pipeline():
|
||||
dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
|
||||
feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"] # Excluded "trip_total"
|
||||
label_column = "tips"
|
||||
all_columns = [label_column] + feature_columns
|
||||
# Deploying the model might incur additional costs over time
|
||||
deploy_model = False
|
||||
|
||||
training_data = download_from_gcs_op(
|
||||
gcs_path=dataset_gcs_uri
|
||||
).outputs["Data"]
|
||||
|
||||
training_data = select_columns_using_Pandas_on_CSV_data_op(
|
||||
table=training_data,
|
||||
column_names=all_columns,
|
||||
).outputs["transformed_table"]
|
||||
|
||||
# Cleaning the NaN values.
|
||||
training_data = fill_all_missing_values_using_Pandas_on_CSV_data_op(
|
||||
table=training_data,
|
||||
replacement_value="0",
|
||||
#replacement_type_name="float",
|
||||
).outputs["transformed_table"]
|
||||
|
||||
model = train_linear_regression_model_using_scikit_learn_from_CSV_op(
|
||||
dataset=training_data,
|
||||
label_column_name=label_column,
|
||||
).outputs["model"]
|
||||
|
||||
vertex_model_name = upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op(
|
||||
model=model,
|
||||
).outputs["model_name"]
|
||||
|
||||
# Deploying the model might incur additional costs over time
|
||||
if deploy_model:
|
||||
sklearn_vertex_endpoint_name = deploy_model_to_endpoint_op(
|
||||
model_name=vertex_model_name,
|
||||
).outputs["endpoint_name"]
|
||||
|
||||
pipeline_func = train_tabular_regression_linear_model_using_Scikit_learn_pipeline
|
||||
|
||||
# %% Pipeline submission
|
||||
if __name__ == '__main__':
|
||||
from google.cloud import aiplatform
|
||||
aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()
|
||||
-97
@@ -1,97 +0,0 @@
|
||||
name: Train tabular regression model using PyTorch pipeline
|
||||
metadata:
|
||||
annotations:
|
||||
author: Alexey Volkov <alexey.volkov@ark-kun.com>
|
||||
canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_regression_model_using_PyTorch_and_import_to_Vertex_AI/pipeline.component.yaml
|
||||
sdk: https://cloud-pipelines.net/pipeline-editor/
|
||||
implementation:
|
||||
graph:
|
||||
tasks:
|
||||
Download from GCS:
|
||||
componentRef:
|
||||
digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
|
||||
annotations:
|
||||
editor.position: '{"x":240,"y":40,"width":180,"height":40}'
|
||||
Select columns using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: Data
|
||||
taskId: Download from GCS
|
||||
column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
|
||||
annotations:
|
||||
editor.position: '{"x":240,"y":130,"width":180,"height":54}'
|
||||
Create fully connected pytorch network:
|
||||
componentRef:
|
||||
digest: d03d8248fd358a0275ec33568ee7dd7dce576cc112b09dfafe2651e4d97e04a9
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1a2ef3eeb77bc278f33cad0dd29008ea2431e191/components/PyTorch/Create_fully_connected_network/component.yaml
|
||||
arguments:
|
||||
input_size: '7'
|
||||
hidden_layer_sizes: '[10]'
|
||||
activation_name: elu
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":240,"width":180,"height":54}'
|
||||
Fill all missing values using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Select columns using Pandas on CSV data
|
||||
type: CSV
|
||||
replacement_value: '0'
|
||||
annotations:
|
||||
editor.position: '{"x":240,"y":240,"width":180,"height":54}'
|
||||
Train pytorch model from csv:
|
||||
componentRef:
|
||||
digest: 40f3185eb61e9727f41a4e0c05dd3d3b44bd802aa0f378cfc31756560033949a
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml
|
||||
arguments:
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Create fully connected pytorch network
|
||||
type: PyTorchScriptModule
|
||||
training_data:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Fill all missing values using Pandas on CSV data
|
||||
type: CSV
|
||||
label_column_name: tips
|
||||
annotations:
|
||||
editor.position: '{"x":240,"y":380,"width":180,"height":40}'
|
||||
Create PyTorch Model Archive with base handler:
|
||||
componentRef:
|
||||
digest: 8298b5ee1b0f0879f893add4cf352c8dec7cf9e21bb9db134c91a2d046cdb0ec
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/46d51383e6554b7f3ab4fd8cf614d8c2b422fb22/components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml
|
||||
arguments:
|
||||
Model:
|
||||
taskOutput:
|
||||
outputName: trained_model
|
||||
taskId: Train pytorch model from csv
|
||||
type: PyTorchScriptModule
|
||||
Model name: model
|
||||
Model version: '1.0'
|
||||
annotations:
|
||||
editor.position: '{"x":240,"y":500,"width":180,"height":54}'
|
||||
Upload PyTorch model archive to Google Cloud Vertex AI:
|
||||
componentRef:
|
||||
digest: 4450212fae7b9001482aca7eb78b28413c205506eccf08a04e7754a8dfa99004
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
model_archive:
|
||||
taskOutput:
|
||||
outputName: Model archive
|
||||
taskId: Create PyTorch Model Archive with base handler
|
||||
type: PyTorchModelArchive
|
||||
annotations:
|
||||
editor.position: '{"x":240,"y":630,"width":180,"height":70}'
|
||||
outputValues: {}
|
||||
-85
@@ -1,85 +0,0 @@
|
||||
# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
|
||||
from kfp import components
|
||||
|
||||
# %% Loading components
|
||||
download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml")
|
||||
fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
|
||||
create_fully_connected_pytorch_network_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/1a2ef3eeb77bc278f33cad0dd29008ea2431e191/components/PyTorch/Create_fully_connected_network/component.yaml")
|
||||
train_pytorch_model_from_csv_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml")
|
||||
create_pytorch_model_archive_with_base_handler_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/46d51383e6554b7f3ab4fd8cf614d8c2b422fb22/components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml")
|
||||
upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
|
||||
# %% Pipeline definition
|
||||
def train_tabular_regression_model_using_PyTorch_pipeline():
|
||||
dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
|
||||
feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"] # Excluded "trip_total"
|
||||
label_column = "tips"
|
||||
all_columns = [label_column] + feature_columns
|
||||
# Deploying the model might incur additional costs over time
|
||||
deploy_model = False
|
||||
|
||||
training_data = download_from_gcs_op(
|
||||
gcs_path=dataset_gcs_uri
|
||||
).outputs["Data"]
|
||||
|
||||
training_data = select_columns_using_Pandas_on_CSV_data_op(
|
||||
table=training_data,
|
||||
column_names=all_columns,
|
||||
).outputs["transformed_table"]
|
||||
|
||||
# Cleaning the NaN values.
|
||||
training_data = fill_all_missing_values_using_Pandas_on_CSV_data_op(
|
||||
table=training_data,
|
||||
replacement_value="0",
|
||||
#replacement_type_name="float",
|
||||
).outputs["transformed_table"]
|
||||
|
||||
network = create_fully_connected_pytorch_network_op(
|
||||
input_size=len(feature_columns),
|
||||
# Optional:
|
||||
hidden_layer_sizes=[10],
|
||||
activation_name="elu",
|
||||
# output_activation_name=None,
|
||||
# output_size=1,
|
||||
).outputs["model"]
|
||||
|
||||
model = train_pytorch_model_from_csv_op(
|
||||
model=network,
|
||||
training_data=training_data,
|
||||
label_column_name=label_column,
|
||||
# Optional:
|
||||
#loss_function_name="mse_loss",
|
||||
#number_of_epochs=1,
|
||||
#learning_rate=0.1,
|
||||
#optimizer_name="Adadelta",
|
||||
#optimizer_parameters={},
|
||||
#batch_size=32,
|
||||
#batch_log_interval=100,
|
||||
#random_seed=0,
|
||||
).outputs["trained_model"]
|
||||
|
||||
model_archive = create_pytorch_model_archive_with_base_handler_op(
|
||||
model=model,
|
||||
# Optional:
|
||||
# model_name="model",
|
||||
# model_version="1.0",
|
||||
).outputs["Model archive"]
|
||||
|
||||
vertex_model_name = upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op(
|
||||
model_archive=model_archive,
|
||||
).outputs["model_name"]
|
||||
|
||||
# Deploying the model might incur additional costs over time
|
||||
if deploy_model:
|
||||
vertex_endpoint_name = deploy_model_to_endpoint_op(
|
||||
model_name=vertex_model_name,
|
||||
).outputs["endpoint_name"]
|
||||
|
||||
pipeline_func=train_tabular_regression_model_using_PyTorch_pipeline
|
||||
|
||||
# %% Pipeline submission
|
||||
if __name__ == '__main__':
|
||||
from google.cloud import aiplatform
|
||||
aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()
|
||||
-116
@@ -1,116 +0,0 @@
|
||||
name: Train tabular regression model using Tensorflow pipeline
|
||||
metadata:
|
||||
annotations:
|
||||
author: Alexey Volkov <alexey.volkov@ark-kun.com>
|
||||
canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_regression_model_using_TensorFlow_and_import_to_Vertex_AI/pipeline.component.yaml
|
||||
sdk: https://cloud-pipelines.net/pipeline-editor/
|
||||
implementation:
|
||||
graph:
|
||||
tasks:
|
||||
Download from GCS:
|
||||
componentRef:
|
||||
digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":40,"width":180,"height":40}'
|
||||
Select columns using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: Data
|
||||
taskId: Download from GCS
|
||||
column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":140,"width":180,"height":54}'
|
||||
Fill all missing values using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Select columns using Pandas on CSV data
|
||||
type: CSV
|
||||
replacement_value: '0'
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":250,"width":180,"height":54}'
|
||||
Split rows into subsets:
|
||||
componentRef:
|
||||
digest: a609c3c9196484290f24a1174955f95b27f07a7b458aa5cb8cde28866cb2cb46
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Fill all missing values using Pandas on CSV data
|
||||
type: CSV
|
||||
fraction_1: '0.8'
|
||||
annotations:
|
||||
editor.position: '{"x":170,"y":380,"width":180,"height":40}'
|
||||
Create fully connected tensorflow network:
|
||||
componentRef:
|
||||
digest: bfcafbc5ce711b1f69cabf1338212d10d50136a73db9f9f7c984de7b80b4bfb0
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/9ca0f9eecf5f896f65b8538bbd809747052617d1/components/tensorflow/Create_fully_connected_network/component.yaml
|
||||
arguments:
|
||||
input_size: '7'
|
||||
hidden_layer_sizes: '[10]'
|
||||
activation_name: elu
|
||||
annotations:
|
||||
editor.position: '{"x":370,"y":380,"width":180,"height":54}'
|
||||
Train model using Keras on CSV:
|
||||
componentRef:
|
||||
digest: 42ae60c889034dbad74815653e95b4f7d576b5f47f803173e8679c7b54984609
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c504a4010348c50eaaf6d4337586ccc008f4dcef/components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml
|
||||
arguments:
|
||||
training_data:
|
||||
taskOutput:
|
||||
outputName: split_1
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Create fully connected tensorflow network
|
||||
type: TensorflowSavedModel
|
||||
label_column_name: tips
|
||||
number_of_epochs: '10'
|
||||
metric_names: '["mean_absolute_error"]'
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":500,"width":180,"height":54}'
|
||||
Upload Tensorflow model to Google Cloud Vertex AI:
|
||||
componentRef:
|
||||
digest: 2e45263ff640b1a688e359b6936e27a81b2407749a84f340af2aa5547e0cb92c
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: trained_model
|
||||
taskId: Train model using Keras on CSV
|
||||
type: TensorflowSavedModel
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":630,"width":180,"height":54}'
|
||||
Predict with TensorFlow model on CSV data:
|
||||
componentRef:
|
||||
digest: 921bb1563e93a78233b8acceab87055b9154ccf5595d056028cf0396ca224cd4
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/59c759ce6f543184e30db6817d2a703879bc0f39/components/tensorflow/Predict/on_CSV/component.yaml
|
||||
arguments:
|
||||
dataset:
|
||||
taskOutput:
|
||||
outputName: split_2
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: trained_model
|
||||
taskId: Train model using Keras on CSV
|
||||
type: TensorflowSavedModel
|
||||
label_column_name: tips
|
||||
annotations:
|
||||
editor.position: '{"x":240,"y":630,"width":180,"height":54}'
|
||||
outputValues: {}
|
||||
-97
@@ -1,97 +0,0 @@
|
||||
# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
|
||||
from kfp import components
|
||||
|
||||
# %% Loading components
|
||||
download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml")
|
||||
fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
|
||||
split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
|
||||
create_fully_connected_tensorflow_network_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/9ca0f9eecf5f896f65b8538bbd809747052617d1/components/tensorflow/Create_fully_connected_network/component.yaml")
|
||||
train_model_using_Keras_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c504a4010348c50eaaf6d4337586ccc008f4dcef/components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml")
|
||||
predict_with_TensorFlow_model_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/59c759ce6f543184e30db6817d2a703879bc0f39/components/tensorflow/Predict/on_CSV/component.yaml")
|
||||
upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
|
||||
# %% Pipeline definition
|
||||
def train_tabular_regression_model_using_Tensorflow_pipeline():
|
||||
dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
|
||||
feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"] # Excluded "trip_total"
|
||||
label_column = "tips"
|
||||
training_set_fraction = 0.8
|
||||
# Deploying the model might incur additional costs over time
|
||||
deploy_model = False
|
||||
|
||||
all_columns = [label_column] + feature_columns
|
||||
|
||||
dataset = download_from_gcs_op(
|
||||
gcs_path=dataset_gcs_uri
|
||||
).outputs["Data"]
|
||||
|
||||
dataset = select_columns_using_Pandas_on_CSV_data_op(
|
||||
table=dataset,
|
||||
column_names=all_columns,
|
||||
).outputs["transformed_table"]
|
||||
|
||||
dataset = fill_all_missing_values_using_Pandas_on_CSV_data_op(
|
||||
table=dataset,
|
||||
replacement_value="0",
|
||||
# # Optional:
|
||||
# column_names=None, # =[...]
|
||||
).outputs["transformed_table"]
|
||||
|
||||
split_task = split_rows_into_subsets_op(
|
||||
table=dataset,
|
||||
fraction_1=training_set_fraction,
|
||||
)
|
||||
training_data = split_task.outputs["split_1"]
|
||||
testing_data = split_task.outputs["split_2"]
|
||||
|
||||
network = create_fully_connected_tensorflow_network_op(
|
||||
input_size=len(feature_columns),
|
||||
# Optional:
|
||||
hidden_layer_sizes=[10],
|
||||
activation_name="elu",
|
||||
# output_activation_name=None,
|
||||
# output_size=1,
|
||||
).outputs["model"]
|
||||
|
||||
model = train_model_using_Keras_on_CSV_op(
|
||||
training_data=training_data,
|
||||
model=network,
|
||||
label_column_name=label_column,
|
||||
# Optional:
|
||||
#loss_function_name="mean_squared_error",
|
||||
number_of_epochs=10,
|
||||
#learning_rate=0.1,
|
||||
#optimizer_name="Adadelta",
|
||||
#optimizer_parameters={},
|
||||
#batch_size=32,
|
||||
metric_names=["mean_absolute_error"],
|
||||
#random_seed=0,
|
||||
).outputs["trained_model"]
|
||||
|
||||
predictions = predict_with_TensorFlow_model_on_CSV_data_op(
|
||||
dataset=testing_data,
|
||||
model=model,
|
||||
# label_column_name needs to be set when doing prediction on a dataset that has labels
|
||||
label_column_name=label_column,
|
||||
# Optional:
|
||||
# batch_size=1000,
|
||||
).outputs["predictions"]
|
||||
|
||||
vertex_model_name = upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op(
|
||||
model=model,
|
||||
).outputs["model_name"]
|
||||
|
||||
# Deploying the model might incur additional costs over time
|
||||
if deploy_model:
|
||||
vertex_endpoint_name = deploy_model_to_endpoint_op(
|
||||
model_name=vertex_model_name,
|
||||
).outputs["endpoint_name"]
|
||||
|
||||
pipeline_func=train_tabular_regression_model_using_Tensorflow_pipeline
|
||||
|
||||
# %% Pipeline submission
|
||||
if __name__ == '__main__':
|
||||
from google.cloud import aiplatform
|
||||
aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()
|
||||
-99
@@ -1,99 +0,0 @@
|
||||
name: Train tabular regression model using XGBoost pipeline
|
||||
metadata:
|
||||
annotations:
|
||||
author: Alexey Volkov <alexey.volkov@ark-kun.com>
|
||||
canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_regression_model_using_XGBoost_and_import_to_Vertex_AI/pipeline.component.yaml
|
||||
sdk: https://cloud-pipelines.net/pipeline-editor/
|
||||
implementation:
|
||||
graph:
|
||||
tasks:
|
||||
Download from GCS:
|
||||
componentRef:
|
||||
digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":40,"width":180,"height":40}'
|
||||
Select columns using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: Data
|
||||
taskId: Download from GCS
|
||||
column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":140,"width":180,"height":54}'
|
||||
Fill all missing values using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Select columns using Pandas on CSV data
|
||||
type: CSV
|
||||
replacement_value: '0'
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":250,"width":180,"height":54}'
|
||||
Split rows into subsets:
|
||||
componentRef:
|
||||
digest: a609c3c9196484290f24a1174955f95b27f07a7b458aa5cb8cde28866cb2cb46
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Fill all missing values using Pandas on CSV data
|
||||
type: CSV
|
||||
fraction_1: '0.8'
|
||||
annotations:
|
||||
editor.position: '{"x":170,"y":360,"width":180,"height":40}'
|
||||
Train XGBoost model on CSV:
|
||||
componentRef:
|
||||
digest: 538c5a01eb38deaf532d619f0bbeaff4efc550fe1f0f776fc06791097b68ceac
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/58d3a47f904f32a64af8403330ba7e2134cae46d/components/XGBoost/Train/component.yaml
|
||||
arguments:
|
||||
training_data:
|
||||
taskOutput:
|
||||
outputName: split_1
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
label_column_name: tips
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":480,"width":180,"height":40}'
|
||||
Upload XGBoost model to Google Cloud Vertex AI:
|
||||
componentRef:
|
||||
digest: 5a5a273c403670743820986c03a4175b7cb4595a556524fefcce403656286977
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Train XGBoost model on CSV
|
||||
type: XGBoostModel
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":600,"width":180,"height":54}'
|
||||
Xgboost predict on CSV:
|
||||
componentRef:
|
||||
digest: 0876233a0c7306fefec188bd70f059b46d1fb5aa57be231799570e3bbbdd0d95
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/4694ec97baccf59284c2a1db4aa2250c22291eab/components/XGBoost/Predict/component.yaml
|
||||
arguments:
|
||||
data:
|
||||
taskOutput:
|
||||
outputName: split_2
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Train XGBoost model on CSV
|
||||
type: XGBoostModel
|
||||
label_column_name: tips
|
||||
annotations:
|
||||
editor.position: '{"x":240,"y":600,"width":180,"height":40}'
|
||||
outputValues: {}
|
||||
-85
@@ -1,85 +0,0 @@
|
||||
# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
|
||||
from kfp import components
|
||||
|
||||
# %% Loading components
|
||||
download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml")
|
||||
fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
|
||||
split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
|
||||
train_XGBoost_model_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/58d3a47f904f32a64af8403330ba7e2134cae46d/components/XGBoost/Train/component.yaml")
|
||||
xgboost_predict_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/4694ec97baccf59284c2a1db4aa2250c22291eab/components/XGBoost/Predict/component.yaml")
|
||||
upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
|
||||
# %% Pipeline definition
|
||||
def train_tabular_regression_model_using_XGBoost_pipeline():
|
||||
dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
|
||||
feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"] # Excluded "trip_total"
|
||||
label_column = "tips"
|
||||
training_set_fraction = 0.8
|
||||
# Deploying the model might incur additional costs over time
|
||||
deploy_model = False
|
||||
|
||||
all_columns = [label_column] + feature_columns
|
||||
|
||||
dataset = download_from_gcs_op(
|
||||
gcs_path=dataset_gcs_uri
|
||||
).outputs["Data"]
|
||||
|
||||
dataset = select_columns_using_Pandas_on_CSV_data_op(
|
||||
table=dataset,
|
||||
column_names=all_columns,
|
||||
).outputs["transformed_table"]
|
||||
|
||||
dataset = fill_all_missing_values_using_Pandas_on_CSV_data_op(
|
||||
table=dataset,
|
||||
replacement_value="0",
|
||||
# # Optional:
|
||||
# column_names=None, # =[...]
|
||||
).outputs["transformed_table"]
|
||||
|
||||
split_task = split_rows_into_subsets_op(
|
||||
table=dataset,
|
||||
fraction_1=training_set_fraction,
|
||||
)
|
||||
training_data = split_task.outputs["split_1"]
|
||||
testing_data = split_task.outputs["split_2"]
|
||||
|
||||
model = train_XGBoost_model_on_CSV_op(
|
||||
training_data=training_data,
|
||||
label_column_name=label_column,
|
||||
# Optional:
|
||||
#starting_model=None,
|
||||
#num_iterations=10,
|
||||
#booster_params={},
|
||||
#objective="reg:squarederror",
|
||||
#booster="gbtree",
|
||||
#learning_rate=0.3,
|
||||
#min_split_loss=0,
|
||||
#max_depth=6,
|
||||
).outputs["model"]
|
||||
|
||||
# Predicting on the testing data
|
||||
predictions = xgboost_predict_on_CSV_op(
|
||||
data=testing_data,
|
||||
model=model,
|
||||
# label_column needs to be set when doing prediction on a dataset that has labels
|
||||
label_column_name=label_column,
|
||||
).outputs["predictions"]
|
||||
|
||||
vertex_model_name = upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op(
|
||||
model=model,
|
||||
).outputs["model_name"]
|
||||
|
||||
# Deploying the model might incur additional costs over time
|
||||
if deploy_model:
|
||||
vertex_endpoint_name = deploy_model_to_endpoint_op(
|
||||
model_name=vertex_model_name,
|
||||
).outputs["endpoint_name"]
|
||||
|
||||
pipeline_func = train_tabular_regression_model_using_XGBoost_pipeline
|
||||
|
||||
# %% Pipeline submission
|
||||
if __name__ == '__main__':
|
||||
from google.cloud import aiplatform
|
||||
aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()
|
||||
-238
@@ -1,238 +0,0 @@
|
||||
name: Train tabular regression model using all frameworks pipeline
|
||||
metadata:
|
||||
annotations:
|
||||
author: Alexey Volkov <alexey.volkov@ark-kun.com>
|
||||
canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_regression_model_using_all_frameworks_and_import_to_Vertex_AI/pipeline.component.yaml
|
||||
sdk: https://cloud-pipelines.net/pipeline-editor/
|
||||
implementation:
|
||||
graph:
|
||||
tasks:
|
||||
Download from GCS:
|
||||
componentRef:
|
||||
digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
|
||||
annotations:
|
||||
editor.position: '{"x":550,"y":40,"width":180,"height":40}'
|
||||
Select columns using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: Data
|
||||
taskId: Download from GCS
|
||||
column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
|
||||
annotations:
|
||||
editor.position: '{"x":550,"y":140,"width":180,"height":54}'
|
||||
Fill all missing values using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Select columns using Pandas on CSV data
|
||||
type: CSV
|
||||
replacement_value: '0'
|
||||
annotations:
|
||||
editor.position: '{"x":550,"y":250,"width":180,"height":54}'
|
||||
Split rows into subsets:
|
||||
componentRef:
|
||||
digest: a609c3c9196484290f24a1174955f95b27f07a7b458aa5cb8cde28866cb2cb46
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Fill all missing values using Pandas on CSV data
|
||||
type: CSV
|
||||
fraction_1: '0.8'
|
||||
annotations:
|
||||
editor.position: '{"x":550,"y":360,"width":180,"height":40}'
|
||||
Create fully connected pytorch network:
|
||||
componentRef:
|
||||
digest: d03d8248fd358a0275ec33568ee7dd7dce576cc112b09dfafe2651e4d97e04a9
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1a2ef3eeb77bc278f33cad0dd29008ea2431e191/components/PyTorch/Create_fully_connected_network/component.yaml
|
||||
arguments:
|
||||
input_size: '7'
|
||||
hidden_layer_sizes: '[10]'
|
||||
activation_name: elu
|
||||
annotations:
|
||||
editor.position: '{"x":380,"y":490,"width":180,"height":54}'
|
||||
Create fully connected tensorflow network:
|
||||
componentRef:
|
||||
digest: bfcafbc5ce711b1f69cabf1338212d10d50136a73db9f9f7c984de7b80b4bfb0
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/9ca0f9eecf5f896f65b8538bbd809747052617d1/components/tensorflow/Create_fully_connected_network/component.yaml
|
||||
arguments:
|
||||
input_size: '7'
|
||||
hidden_layer_sizes: '[10]'
|
||||
activation_name: elu
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":500,"width":180,"height":54}'
|
||||
Train model using Keras on CSV:
|
||||
componentRef:
|
||||
digest: 42ae60c889034dbad74815653e95b4f7d576b5f47f803173e8679c7b54984609
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c504a4010348c50eaaf6d4337586ccc008f4dcef/components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml
|
||||
arguments:
|
||||
training_data:
|
||||
taskOutput:
|
||||
outputName: split_1
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Create fully connected tensorflow network
|
||||
type: TensorflowSavedModel
|
||||
label_column_name: tips
|
||||
number_of_epochs: '10'
|
||||
metric_names: '["mean_absolute_error"]'
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":620,"width":180,"height":54}'
|
||||
Train pytorch model from csv:
|
||||
componentRef:
|
||||
digest: 40f3185eb61e9727f41a4e0c05dd3d3b44bd802aa0f378cfc31756560033949a
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml
|
||||
arguments:
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Create fully connected pytorch network
|
||||
type: PyTorchScriptModule
|
||||
training_data:
|
||||
taskOutput:
|
||||
outputName: split_1
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
label_column_name: tips
|
||||
annotations:
|
||||
editor.position: '{"x":380,"y":620,"width":180,"height":40}'
|
||||
Train XGBoost model on CSV:
|
||||
componentRef:
|
||||
digest: 538c5a01eb38deaf532d619f0bbeaff4efc550fe1f0f776fc06791097b68ceac
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/58d3a47f904f32a64af8403330ba7e2134cae46d/components/XGBoost/Train/component.yaml
|
||||
arguments:
|
||||
training_data:
|
||||
taskOutput:
|
||||
outputName: split_1
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
label_column_name: tips
|
||||
annotations:
|
||||
editor.position: '{"x":720,"y":620,"width":180,"height":40}'
|
||||
Train linear regression model using scikit learn from CSV:
|
||||
componentRef:
|
||||
digest: c7fe7912ab0d1fb45d201d452e9ce6be5544e7d8c6d229db7a4b931ff58560f3
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/f807e02b54d4886c65a05f40848fd51c72407f40/components/ML_frameworks/Scikit_learn/Train_linear_regression_model/from_CSV/component.yaml
|
||||
arguments:
|
||||
dataset:
|
||||
taskOutput:
|
||||
outputName: split_1
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
label_column_name: tips
|
||||
annotations:
|
||||
editor.position: '{"x":1030,"y":620,"width":180,"height":54}'
|
||||
Predict with TensorFlow model on CSV data:
|
||||
componentRef:
|
||||
digest: 921bb1563e93a78233b8acceab87055b9154ccf5595d056028cf0396ca224cd4
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/59c759ce6f543184e30db6817d2a703879bc0f39/components/tensorflow/Predict/on_CSV/component.yaml
|
||||
arguments:
|
||||
dataset:
|
||||
taskOutput:
|
||||
outputName: split_2
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: trained_model
|
||||
taskId: Train model using Keras on CSV
|
||||
type: TensorflowSavedModel
|
||||
label_column_name: tips
|
||||
annotations:
|
||||
editor.position: '{"x":160,"y":750,"width":180,"height":54}'
|
||||
Create PyTorch Model Archive with base handler:
|
||||
componentRef:
|
||||
digest: 8298b5ee1b0f0879f893add4cf352c8dec7cf9e21bb9db134c91a2d046cdb0ec
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/46d51383e6554b7f3ab4fd8cf614d8c2b422fb22/components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml
|
||||
arguments:
|
||||
Model:
|
||||
taskOutput:
|
||||
outputName: trained_model
|
||||
taskId: Train pytorch model from csv
|
||||
type: PyTorchScriptModule
|
||||
Model name: model
|
||||
Model version: '1.0'
|
||||
annotations:
|
||||
editor.position: '{"x":380,"y":750,"width":180,"height":54}'
|
||||
Xgboost predict on CSV:
|
||||
componentRef:
|
||||
digest: 0876233a0c7306fefec188bd70f059b46d1fb5aa57be231799570e3bbbdd0d95
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/4694ec97baccf59284c2a1db4aa2250c22291eab/components/XGBoost/Predict/component.yaml
|
||||
arguments:
|
||||
data:
|
||||
taskOutput:
|
||||
outputName: split_2
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Train XGBoost model on CSV
|
||||
type: XGBoostModel
|
||||
label_column_name: tips
|
||||
annotations:
|
||||
editor.position: '{"x":810,"y":750,"width":180,"height":40}'
|
||||
Upload Scikit learn pickle model to Google Cloud Vertex AI:
|
||||
componentRef:
|
||||
digest: 81c91c8d7d21ec97e0872f669d68bd89edea87279d703685db54aa94743bebcd
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Train linear regression model using scikit learn from CSV
|
||||
type: ScikitLearnPickleModel
|
||||
annotations:
|
||||
editor.position: '{"x":1030,"y":750,"width":180,"height":70}'
|
||||
Upload Tensorflow model to Google Cloud Vertex AI:
|
||||
componentRef:
|
||||
digest: 2e45263ff640b1a688e359b6936e27a81b2407749a84f340af2aa5547e0cb92c
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: trained_model
|
||||
taskId: Train model using Keras on CSV
|
||||
type: TensorflowSavedModel
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":880,"width":180,"height":54}'
|
||||
Upload PyTorch model archive to Google Cloud Vertex AI:
|
||||
componentRef:
|
||||
digest: 4450212fae7b9001482aca7eb78b28413c205506eccf08a04e7754a8dfa99004
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
model_archive:
|
||||
taskOutput:
|
||||
outputName: Model archive
|
||||
taskId: Create PyTorch Model Archive with base handler
|
||||
type: PyTorchModelArchive
|
||||
annotations:
|
||||
editor.position: '{"x":380,"y":880,"width":180,"height":70}'
|
||||
Upload XGBoost model to Google Cloud Vertex AI:
|
||||
componentRef:
|
||||
digest: 5a5a273c403670743820986c03a4175b7cb4595a556524fefcce403656286977
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Train XGBoost model on CSV
|
||||
type: XGBoostModel
|
||||
annotations:
|
||||
editor.position: '{"x":720,"y":880,"width":180,"height":54}'
|
||||
outputValues: {}
|
||||
-208
@@ -1,208 +0,0 @@
|
||||
# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
|
||||
from kfp import components
|
||||
|
||||
# %% Loading components
|
||||
download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml")
|
||||
fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
|
||||
split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
|
||||
|
||||
# TensorFlow
|
||||
create_fully_connected_tensorflow_network_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/9ca0f9eecf5f896f65b8538bbd809747052617d1/components/tensorflow/Create_fully_connected_network/component.yaml")
|
||||
train_model_using_Keras_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c504a4010348c50eaaf6d4337586ccc008f4dcef/components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml")
|
||||
predict_with_TensorFlow_model_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/59c759ce6f543184e30db6817d2a703879bc0f39/components/tensorflow/Predict/on_CSV/component.yaml")
|
||||
upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
|
||||
# PyTorch
|
||||
create_fully_connected_pytorch_network_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/1a2ef3eeb77bc278f33cad0dd29008ea2431e191/components/PyTorch/Create_fully_connected_network/component.yaml")
|
||||
train_pytorch_model_from_csv_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml")
|
||||
create_pytorch_model_archive_with_base_handler_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/46d51383e6554b7f3ab4fd8cf614d8c2b422fb22/components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml")
|
||||
upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
|
||||
# XGBoost
|
||||
train_XGBoost_model_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/58d3a47f904f32a64af8403330ba7e2134cae46d/components/XGBoost/Train/component.yaml")
|
||||
xgboost_predict_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/4694ec97baccf59284c2a1db4aa2250c22291eab/components/XGBoost/Predict/component.yaml")
|
||||
upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
|
||||
# Scikit-learn
|
||||
train_linear_regression_model_using_scikit_learn_from_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/f807e02b54d4886c65a05f40848fd51c72407f40/components/ML_frameworks/Scikit_learn/Train_linear_regression_model/from_CSV/component.yaml")
|
||||
upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
|
||||
# Vertex AI
|
||||
deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
|
||||
# %% Pipeline definition
|
||||
def train_tabular_regression_model_using_all_frameworks_pipeline():
|
||||
dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
|
||||
feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"] # Excluded "trip_total"
|
||||
label_column = "tips"
|
||||
training_set_fraction = 0.8
|
||||
# Deploying the model might incur additional costs over time
|
||||
deploy_model = False
|
||||
|
||||
all_columns = [label_column] + feature_columns
|
||||
|
||||
dataset = download_from_gcs_op(
|
||||
gcs_path=dataset_gcs_uri
|
||||
).outputs["Data"]
|
||||
|
||||
dataset = select_columns_using_Pandas_on_CSV_data_op(
|
||||
table=dataset,
|
||||
column_names=all_columns,
|
||||
).outputs["transformed_table"]
|
||||
|
||||
dataset = fill_all_missing_values_using_Pandas_on_CSV_data_op(
|
||||
table=dataset,
|
||||
replacement_value="0",
|
||||
# # Optional:
|
||||
# column_names=None, # =[...]
|
||||
).outputs["transformed_table"]
|
||||
|
||||
split_task = split_rows_into_subsets_op(
|
||||
table=dataset,
|
||||
fraction_1=training_set_fraction,
|
||||
)
|
||||
training_data = split_task.outputs["split_1"]
|
||||
testing_data = split_task.outputs["split_2"]
|
||||
|
||||
# TensorFlow
|
||||
tensorflow_network = create_fully_connected_tensorflow_network_op(
|
||||
input_size=len(feature_columns),
|
||||
# Optional:
|
||||
hidden_layer_sizes=[10],
|
||||
activation_name="elu",
|
||||
# output_activation_name=None,
|
||||
# output_size=1,
|
||||
).outputs["model"]
|
||||
|
||||
tensorflow_model = train_model_using_Keras_on_CSV_op(
|
||||
training_data=training_data,
|
||||
model=tensorflow_network,
|
||||
label_column_name=label_column,
|
||||
# Optional:
|
||||
#loss_function_name="mean_squared_error",
|
||||
number_of_epochs=10,
|
||||
#learning_rate=0.1,
|
||||
#optimizer_name="Adadelta",
|
||||
#optimizer_parameters={},
|
||||
#batch_size=32,
|
||||
metric_names=["mean_absolute_error"],
|
||||
#random_seed=0,
|
||||
).outputs["trained_model"]
|
||||
|
||||
tensorflow_predictions = predict_with_TensorFlow_model_on_CSV_data_op(
|
||||
dataset=testing_data,
|
||||
model=tensorflow_model,
|
||||
# label_column_name needs to be set when doing prediction on a dataset that has labels
|
||||
label_column_name=label_column,
|
||||
# Optional:
|
||||
# batch_size=1000,
|
||||
).outputs["predictions"]
|
||||
|
||||
tensorflow_vertex_model_name = upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op(
|
||||
model=tensorflow_model,
|
||||
).outputs["model_name"]
|
||||
|
||||
# Deploying the model might incur additional costs over time
|
||||
if deploy_model:
|
||||
tensorflow_vertex_endpoint_name = deploy_model_to_endpoint_op(
|
||||
model_name=tensorflow_vertex_model_name,
|
||||
).outputs["endpoint_name"]
|
||||
|
||||
# PyTorch
|
||||
pytorch_network = create_fully_connected_pytorch_network_op(
|
||||
input_size=len(feature_columns),
|
||||
# Optional:
|
||||
hidden_layer_sizes=[10],
|
||||
activation_name="elu",
|
||||
# output_activation_name=None,
|
||||
# output_size=1,
|
||||
).outputs["model"]
|
||||
|
||||
pytorch_model = train_pytorch_model_from_csv_op(
|
||||
model=pytorch_network,
|
||||
training_data=training_data,
|
||||
label_column_name=label_column,
|
||||
# Optional:
|
||||
#loss_function_name="mse_loss",
|
||||
#number_of_epochs=1,
|
||||
#learning_rate=0.1,
|
||||
#optimizer_name="Adadelta",
|
||||
#optimizer_parameters={},
|
||||
#batch_size=32,
|
||||
#batch_log_interval=100,
|
||||
#random_seed=0,
|
||||
).outputs["trained_model"]
|
||||
|
||||
pytorch_model_archive = create_pytorch_model_archive_with_base_handler_op(
|
||||
model=pytorch_model,
|
||||
# Optional:
|
||||
# model_name="model",
|
||||
# model_version="1.0",
|
||||
).outputs["Model archive"]
|
||||
|
||||
pytorch_vertex_model_name = upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op(
|
||||
model_archive=pytorch_model_archive,
|
||||
).outputs["model_name"]
|
||||
|
||||
# Deploying the model might incur additional costs over time
|
||||
if deploy_model:
|
||||
pytorch_vertex_endpoint_name = deploy_model_to_endpoint_op(
|
||||
model_name=pytorch_vertex_model_name,
|
||||
).outputs["endpoint_name"]
|
||||
|
||||
# XGBoost
|
||||
xgboost_model = train_XGBoost_model_on_CSV_op(
|
||||
training_data=training_data,
|
||||
label_column_name=label_column,
|
||||
# Optional:
|
||||
#starting_model=None,
|
||||
#num_iterations=10,
|
||||
#booster_params={},
|
||||
#objective="reg:squarederror",
|
||||
#booster="gbtree",
|
||||
#learning_rate=0.3,
|
||||
#min_split_loss=0,
|
||||
#max_depth=6,
|
||||
).outputs["model"]
|
||||
|
||||
# Predicting on the testing data
|
||||
xgboost_predictions = xgboost_predict_on_CSV_op(
|
||||
data=testing_data,
|
||||
model=xgboost_model,
|
||||
# label_column needs to be set when doing prediction on a dataset that has labels
|
||||
label_column_name=label_column,
|
||||
).outputs["predictions"]
|
||||
|
||||
xgboost_vertex_model_name = upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op(
|
||||
model=xgboost_model,
|
||||
).outputs["model_name"]
|
||||
|
||||
# Deploying the model might incur additional costs over time
|
||||
if deploy_model:
|
||||
xgboost_vertex_endpoint_name = deploy_model_to_endpoint_op(
|
||||
model_name=xgboost_vertex_model_name,
|
||||
).outputs["endpoint_name"]
|
||||
|
||||
# Scikit-learn
|
||||
sklearn_model = train_linear_regression_model_using_scikit_learn_from_CSV_op(
|
||||
dataset=training_data,
|
||||
label_column_name=label_column,
|
||||
).outputs["model"]
|
||||
|
||||
sklearn_vertex_model_name = upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op(
|
||||
model=sklearn_model,
|
||||
).outputs["model_name"]
|
||||
|
||||
# Deploying the model might incur additional costs over time
|
||||
if deploy_model:
|
||||
sklearn_vertex_endpoint_name = deploy_model_to_endpoint_op(
|
||||
model_name=sklearn_vertex_model_name,
|
||||
).outputs["endpoint_name"]
|
||||
|
||||
pipeline_func=train_tabular_regression_model_using_all_frameworks_pipeline
|
||||
|
||||
# %% Pipeline submission
|
||||
if __name__ == '__main__':
|
||||
from google.cloud import aiplatform
|
||||
aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()
|
||||
@@ -7,7 +7,7 @@
|
||||
/gapic @andrewferlitsch
|
||||
/gapic/custom/showcase_custom_image_classification_online_explain_example_based_api.ipynb @inardini
|
||||
/ml_ops @andrewferlitsch
|
||||
/model_monitoring/* @andrewferlitsch
|
||||
/model_monitoring/* @mco-gh
|
||||
/structured_data/rapid_prototyping_* @rafael-carvalho
|
||||
|
||||
/managed_notebooks/
|
||||
@@ -31,6 +31,3 @@
|
||||
/notebooks/community/pipelines/google_cloud_pipeline_components_bqml_pipeline_demand_forecasting.ipynb @inardini
|
||||
/notebooks/community/ml_ops/stage2/get_started_vertex_hpt_r_kernel.ipynb @fhirschmann
|
||||
/notebooks/community/ml_ops/stage2/get_started_vertex_training_r_using_r_kernel.ipynb @fhirschmann
|
||||
/notebooks/community/vertex-ai-samples/notebooks/community/model_registry/vertex_ai_model_registry_bqml_custom_model_versioning.ipynb @inardini
|
||||
/notebooks/community/vertex-ai-samples/notebooks/community/model_registry/vertex_ai_model_registry_automl_model_versioning.ipynb @inardini
|
||||
/notebooks/community/vizier/conversions_vertex_vizier_and_open_source_vizier.ipynb @halio-g
|
||||
|
||||
@@ -14,5 +14,5 @@ The purpose of this set of notebooks and markdown files is to demonstrate Google
|
||||
4. [Evaluation](stage4)
|
||||
5. [Deployment](stage5)
|
||||
6. [Serving](stage6)
|
||||
7. Monitoring(stage7)
|
||||
7. Monitoring
|
||||
8. Continuous Training
|
||||
|
||||
@@ -1,43 +0,0 @@
|
||||
## Before you begin
|
||||
|
||||
### Set up your Google Cloud project
|
||||
|
||||
**The following steps are required, regardless of your notebook environment.**
|
||||
|
||||
1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.
|
||||
|
||||
1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).
|
||||
|
||||
1. [Enable the Vertex AI, BigQuery, Compute Engine and Cloud Storage APIs](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,bigquery,compute_component,storage_component).
|
||||
|
||||
1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).
|
||||
|
||||
1. Enter your project ID in the cell below. Then run the cell to make sure the
|
||||
Cloud SDK uses the right project for all the commands in this notebook.
|
||||
|
||||
**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands.
|
||||
|
||||
### Set up your local development environment
|
||||
|
||||
**If you are using Colab or Vertex AI Workbench Notebooks**, your environment already meets all the requirements to run this notebook. You can skip this step.
|
||||
|
||||
**Otherwise**, make sure your environment meets this notebook's requirements. You need the following:
|
||||
|
||||
- The Cloud Storage SDK
|
||||
- Python 3
|
||||
- virtualenv
|
||||
- Jupyter notebook running in a virtual environment with Python 3
|
||||
|
||||
The Cloud Storage guide to [Setting up a Python development environment](https://cloud.google.com/python/setup) and the [Jupyter installation guide](https://jupyter.org/install) provide detailed instructions for meeting these requirements. The following steps provide a condensed set of instructions:
|
||||
|
||||
1. [Install and initialize the SDK](https://cloud.google.com/sdk/docs/).
|
||||
|
||||
2. [Install Python 3](https://cloud.google.com/python/setup#installing_python).
|
||||
|
||||
3. [Install virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv) and create a virtual environment that uses Python 3. Activate the virtual environment.
|
||||
|
||||
4. To install Jupyter, run `pip3 install jupyter` on the command-line in a terminal shell.
|
||||
|
||||
5. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.
|
||||
|
||||
6. Open this notebook in the Jupyter Notebook Dashboard.
|
||||
@@ -1,112 +0,0 @@
|
||||
import os
|
||||
import sys
|
||||
import argparse
|
||||
import subprocess
|
||||
import random
|
||||
import string
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--bucket', dest='bucket_required', action='store_true',
|
||||
default=False, help='Bucket required')
|
||||
parser.add_argument('--email', dest='email_required', action='store_true',
|
||||
default=False, help='Email required')
|
||||
parser.add_argument('--sa', dest='sa_required', action='store_true',
|
||||
default=False, help='Service account required')
|
||||
parser.add_argument('--packages', dest='extra_packages',
|
||||
default='', type=str, help='additional required packages')
|
||||
args = parser.parse_args()
|
||||
|
||||
extra_pkgs = args.extra_packages
|
||||
|
||||
|
||||
# Installation
|
||||
|
||||
|
||||
# The Vertex AI Workbench Notebook product has specific requirements
|
||||
IS_WORKBENCH_NOTEBOOK = os.getenv("DL_ANACONDA_HOME")
|
||||
IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(
|
||||
"/opt/deeplearning/metadata/env_version"
|
||||
)
|
||||
IS_COLAB = "google.colab" in sys.modules
|
||||
|
||||
# Vertex AI Notebook requires dependencies to be installed with '--user'
|
||||
USER_FLAG = ""
|
||||
if IS_WORKBENCH_NOTEBOOK:
|
||||
USER_FLAG = "--user"
|
||||
|
||||
# not used
|
||||
'''
|
||||
print("Installing packages")
|
||||
os.system(f"pip3 install --upgrade --quiet {USER_FLAG} google-cloud-aiplatform {args.extra_packages}")
|
||||
print("Done installation")
|
||||
'''
|
||||
|
||||
# Authenticate
|
||||
if IS_COLAB:
|
||||
from google.colab import auth as google_auth
|
||||
|
||||
google_auth.authenticate_user()
|
||||
|
||||
|
||||
# project ID
|
||||
if IS_WORKBENCH_NOTEBOOK:
|
||||
shell_output = subprocess.check_output("gcloud config list --format 'value(core.project)' 2>/dev/null", shell=True)
|
||||
PROJECT_ID = shell_output[0:-1].decode('utf-8')
|
||||
print("PROJECT ID: ", PROJECT_ID)
|
||||
else:
|
||||
PROJECT_ID = input("Enter PROJECT_ID: ")
|
||||
os.system(f"gcloud config set project {PROJECT_ID}")
|
||||
|
||||
# email
|
||||
if args.email_required:
|
||||
shell_output = subprocess.check_output("gcloud config list --format 'value(core.account)' 2>/dev/null", shell=True)
|
||||
EMAIL_ADDR = shell_output[0:-1].decode('utf-8')
|
||||
if EMAIL_ADDR == '':
|
||||
EMAIL_ADDR = input("Enter Email Address: ")
|
||||
print("EMAIL_ADDR: ", EMAIL_ADDR)
|
||||
|
||||
# region
|
||||
shell_output = subprocess.check_output("gcloud config list --format 'value(ai.region)'", shell=True)
|
||||
REGION = shell_output[0:-1].decode('utf-8')
|
||||
if REGION == '':
|
||||
REGION = input("Enter REGION: ")
|
||||
print("REGION: ", REGION)
|
||||
|
||||
# multi-region
|
||||
MULTI_REGION = REGION.split('-')[0]
|
||||
|
||||
|
||||
# UUID
|
||||
# Generate a uuid of a specifed length(default=8)
|
||||
def generate_uuid(length: int = 8) -> str:
|
||||
return "".join(random.choices(string.ascii_lowercase + string.digits, k=length))
|
||||
|
||||
|
||||
UUID = generate_uuid()
|
||||
print("UUID", UUID)
|
||||
|
||||
# Bucket
|
||||
if args.bucket_required:
|
||||
BUCKET_NAME = PROJECT_ID + "aip-" + UUID
|
||||
BUCKET_URI = f"gs://{BUCKET_NAME}"
|
||||
os.system(f"gsutil mb -l {REGION} {BUCKET_URI}")
|
||||
print("BUCKET_URI", BUCKET_URI)
|
||||
|
||||
|
||||
# Project Number
|
||||
if args.sa_required:
|
||||
if IS_WORKBENCH_NOTEBOOK:
|
||||
shell_output = subprocess.check_output("gcloud auth list 2>/dev/null", shell=True)
|
||||
SERVICE_ACCOUNT = shell_output[:-1].decode('utf-8').split('\n')[2].strip()
|
||||
PROJECT_NUMBER = SERVICE_ACCOUNT.split('-')[0]
|
||||
else:
|
||||
shell_output = subprocess.check_output(f"gcloud projects describe {PROJECT_ID}", shell=True)
|
||||
try:
|
||||
PROJECT_NUMBER = shell_output[:-1].decode('utf-8').split('\n')[7].split(':')[-1].strip().replace("'", "")
|
||||
SERVICE_ACCOUNT = f"{PROJECT_NUMBER}-compute@developer.gserviceaccount.com"
|
||||
except:
|
||||
PROJECT_NUMBER = input("Enter project number: ")
|
||||
SERVICE_ACCOUNT = f"{PROJECT_NUMBER}-compute@developer.gserviceaccount.com"
|
||||
|
||||
print("SERVICE_ACCOUNT", SERVICE_ACCOUNT)
|
||||
print("PROJECT_NUMBER", PROJECT_NUMBER)
|
||||
@@ -28,46 +28,50 @@ The first stage in MLOps is the collection and preparation for the purpose of de
|
||||
|
||||
### Get Started
|
||||
|
||||
[Get started with Dataflow](community/ml_ops/stage1/get_started_dataflow.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use `Dataflow` for training with `Vertex AI`.
|
||||
[Get started with Vertex AI datasets](get_started_vertex_datasets.ipynb)
|
||||
|
||||
```
|
||||
The steps performed include:
|
||||
|
||||
- Offline preprocessing of data:
|
||||
- Serially - w/o dataflow
|
||||
- Parallel - with dataflow
|
||||
- Upstream preprocessing of data:
|
||||
- tabular data
|
||||
- image data
|
||||
|
||||
[Get started with Vertex AI datasets](community/ml_ops/stage1/get_started_vertex_datasets.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use `Vertex AI Dataset` for training with `Vertex AI`.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Create a Vertex AI `Dataset` resource for:
|
||||
- image data
|
||||
- text data
|
||||
- video data
|
||||
- tabular data
|
||||
- forecasting data
|
||||
|
||||
|
||||
- Search `Dataset` resources using a filter.
|
||||
- Read a sample of a `BigQuery` dataset into a dataframe.
|
||||
- Generate statistics and data schema using TensorFlow Data Validation from the samples in the dataframe.
|
||||
- Detect anomalies in new data using TensorFlow Data Validation.
|
||||
- Generate a TFRecord feature specification using TensorFlow Transform from the data schema.
|
||||
- Export a dataset and convert to TFRecords.
|
||||
```
|
||||
|
||||
[Get started with BigQuery datasets](community/ml_ops/stage1/get_started_bq_datasets.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use `BigQuery` as a dataset for training with `Vertex AI`.
|
||||
[Get started with Dataflow](get_started_dataflow.ipynb)
|
||||
|
||||
```
|
||||
The steps performed include:
|
||||
- Offline preprocessing of data:
|
||||
- Serially - w/o dataflow
|
||||
- Parallel - with dataflow
|
||||
- Upstream preprocessing of data:
|
||||
- tabular data
|
||||
- image data
|
||||
```
|
||||
|
||||
[Create an unlabelled Vertex AI AutoML text entity extraction dataset from pdfs using Vision API](get_started_with_visionapi_and_vertex_datasets.ipynb)
|
||||
|
||||
```
|
||||
The steps performed include:
|
||||
1. Using `Vision API` to perform Optical Character Recognition (OCR) to extract text from PDF files.
|
||||
2. Processing the results and saving them to text files.
|
||||
3. Generating a `Vertex AI Dataset` import file.
|
||||
4. Creating a new unlabelled text entity extraction `Vertex AI Dataset` resource in `Vertex AI`.
|
||||
```
|
||||
|
||||
[Get started with BigQuery datasets](get_started_bq_datasets.ipynb)
|
||||
|
||||
```
|
||||
The steps performed include:
|
||||
- Create a Vertex AI `Dataset` resource from `BigQuery` table -- compatible for `AutoML` training.
|
||||
- Extract a copy of the dataset from `BigQuery` to a CSV file in Cloud Storage -- compatible for `AutoML` or custom training.
|
||||
- Select rows from a `BigQuery` dataset into a `pandas` dataframe -- compatible for custom training.
|
||||
@@ -75,32 +79,19 @@ The steps performed include:
|
||||
- Select rows from extracted CSV files into a `tf.data.Dataset` -- compatible for custom training `TensorFlow` models.
|
||||
- Create a `BigQuery` dataset from CSV files.
|
||||
- Extract data from `BigQuery` table into a `DMatrix` -- compatible for custom training `XGBoost` models.
|
||||
```
|
||||
|
||||
[Get started with Vertex AI Data Labeling](community/ml_ops/stage1/get_started_with_data_labeling.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use the `Vertex AI Data Labeling` service.
|
||||
[Get started with Vertex AI data labeling](get_started_with_data_labeling.ipynb)
|
||||
|
||||
```
|
||||
The steps performed include:
|
||||
|
||||
- Create a Specialist Pool for data labelers.
|
||||
- Create a data labeling job.
|
||||
- Submit the data labeling job.
|
||||
- List data labeling jobs.
|
||||
- Cancel a data labeling job.
|
||||
|
||||
|
||||
|
||||
[Create an unlabelled Vertex AI AutoML text entity extraction dataset from PDFs using Vision API](community/ml_ops/stage1/get_started_with_visionapi_and_vertex_datasets.ipynb)
|
||||
|
||||
In this tutorial, you learn to use `Vision API` to extract text from PDF files stored on a Cloud Storage bucket. You then process the results and create an unlabelled `Vertex AI Dataset`, compatible with `AutoML`, for text entity extraction.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
1. Using `Vision API` to perform Optical Character Recognition (OCR) to extract text from PDF files.
|
||||
2. Processing the results and saving them to text files.
|
||||
3. Generating a `Vertex AI Dataset` import file.
|
||||
4. Creating a new unlabelled text entity extraction `Vertex AI Dataset` resource in `Vertex AI`.
|
||||
|
||||
```
|
||||
|
||||
### E2E Stage Example
|
||||
|
||||
|
||||
@@ -120,7 +120,7 @@
|
||||
" - XGBoost model training:\n",
|
||||
" - Use BigQuery ML built-in XGBoost training.\n",
|
||||
" - Alternatively, create a DMatrix generator from CSV files extracted from BigQuery table.\n",
|
||||
" - PyTorch model training:\n",
|
||||
" - Pytorch model training:\n",
|
||||
" - Extract the BigQuery to a pandas dataframe.\n",
|
||||
" - Preprocess the data in the dataframe.\n",
|
||||
" - Create a DataLoader generator from the pandas dataframe.\n",
|
||||
@@ -191,8 +191,13 @@
|
||||
"if IS_WORKBENCH_NOTEBOOK:\n",
|
||||
" USER_FLAG = \"--user\"\n",
|
||||
"\n",
|
||||
"extra_pkgs = \"tensorflow tensorflow-io==0.18 pyarrow xgboost google-cloud-bigquery\"\n",
|
||||
"! pip3 install --upgrade --quiet {USER_FLAG} google-cloud-aiplatform $extra_pkgs"
|
||||
"# Install the packages\n",
|
||||
"! pip3 install --upgrade pyarrow $USER_FLAG -q\n",
|
||||
"! pip3 install --upgrade google-cloud-bigquery $USER_FLAG -q\n",
|
||||
"! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG -q\n",
|
||||
"! pip3 install -U xgboost $USER_FLAG -q\n",
|
||||
"! pip3 install -U tensorflow $USER_FLAG -q\n",
|
||||
"! pip3 install -U tensorflow-io==0.18 $USER_FLAG -q"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -214,9 +219,9 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import sys\n",
|
||||
"import os\n",
|
||||
"\n",
|
||||
"if \"google.colab\" in sys.modules:\n",
|
||||
"if not os.getenv(\"IS_TESTING\"):\n",
|
||||
" # Automatically restart kernel after installs\n",
|
||||
" import IPython\n",
|
||||
"\n",
|
||||
@@ -227,42 +232,274 @@
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "fc8fb52b5cca"
|
||||
"id": "84cd83853240"
|
||||
},
|
||||
"source": [
|
||||
"### Common setup\n",
|
||||
"## Before you begin\n",
|
||||
"\n",
|
||||
"Now, execute the common setup for the notebook tutorials."
|
||||
"### Set up your Google Cloud project\n",
|
||||
"\n",
|
||||
"**The following steps are required, regardless of your notebook environment.**\n",
|
||||
"\n",
|
||||
"1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
|
||||
"\n",
|
||||
"1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
|
||||
"\n",
|
||||
"1. [Enable the Vertex AI, BigQuery, Compute Engine and Cloud Storage APIs](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,bigquery,compute_component,storage_component).\n",
|
||||
"\n",
|
||||
"1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
|
||||
"\n",
|
||||
"1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
|
||||
"Cloud SDK uses the right project for all the commands in this notebook.\n",
|
||||
"\n",
|
||||
"**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "project_id"
|
||||
},
|
||||
"source": [
|
||||
"#### Set your project ID\n",
|
||||
"\n",
|
||||
"**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "001a0fcd5d78"
|
||||
"id": "set_project_id"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Common code setup for notebook tutorials\n",
|
||||
"\n",
|
||||
"! wget https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/setup.py -O setup.py\n",
|
||||
"\n",
|
||||
"%run setup.py --bucket"
|
||||
"PROJECT_ID = \"[your-project-id]\" # @param {type:\"string\"}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "d809f07a8935"
|
||||
"id": "autoset_project_id"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Other Common setup instructions for notebook tutorials\n",
|
||||
"if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
|
||||
" # Get your GCP project id from gcloud\n",
|
||||
" shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
|
||||
" PROJECT_ID = shell_output[0]\n",
|
||||
" print(\"Project ID:\", PROJECT_ID)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "set_gcloud_project_id"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gcloud config set project $PROJECT_ID"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "region"
|
||||
},
|
||||
"source": [
|
||||
"#### Region\n",
|
||||
"\n",
|
||||
"! wget https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/setup.md -O setup.md\n",
|
||||
"You can also change the `REGION` variable, which is used for operations\n",
|
||||
"throughout the rest of this notebook. Below are regions supported for Vertex AI. We recommend that you choose the region closest to you.\n",
|
||||
"\n",
|
||||
"%load setup.md"
|
||||
"- Americas: `us-central1`\n",
|
||||
"- Europe: `europe-west4`\n",
|
||||
"- Asia Pacific: `asia-east1`\n",
|
||||
"\n",
|
||||
"You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
|
||||
"\n",
|
||||
"Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "region"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"REGION = \"[your-region]\" # @param {type: \"string\"}\n",
|
||||
"\n",
|
||||
"if REGION == \"[your-region]\":\n",
|
||||
" REGION = \"us-central1\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "timestamp"
|
||||
},
|
||||
"source": [
|
||||
"#### Timestamp\n",
|
||||
"\n",
|
||||
"If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "timestamp"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from datetime import datetime\n",
|
||||
"\n",
|
||||
"TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "77c385f0db59"
|
||||
},
|
||||
"source": [
|
||||
"### Authenticate your Google Cloud account\n",
|
||||
"\n",
|
||||
"**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
|
||||
"\n",
|
||||
"**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
|
||||
"\n",
|
||||
"**Otherwise**, follow these steps:\n",
|
||||
"\n",
|
||||
"In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
|
||||
"\n",
|
||||
"1. **Click Create service account**.\n",
|
||||
"\n",
|
||||
"2. In the **Service account name** field, enter a name, and click **Create**.\n",
|
||||
"\n",
|
||||
"3. In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex AI\" into the filter box, and select **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
|
||||
"\n",
|
||||
"4. Click Create. A JSON file that contains your key downloads to your local environment.\n",
|
||||
"\n",
|
||||
"5. Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "535223fa4b84"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# If you are running this notebook in Colab, run this cell and follow the\n",
|
||||
"# instructions to authenticate your GCP account. This provides access to your\n",
|
||||
"# Cloud Storage bucket and lets you submit training jobs and prediction\n",
|
||||
"# requests.\n",
|
||||
"\n",
|
||||
"import os\n",
|
||||
"import sys\n",
|
||||
"\n",
|
||||
"# If on Vertex AI Workbench, then don't execute this code\n",
|
||||
"IS_COLAB = False\n",
|
||||
"if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
|
||||
" \"DL_ANACONDA_HOME\"\n",
|
||||
"):\n",
|
||||
" if \"google.colab\" in sys.modules:\n",
|
||||
" IS_COLAB = True\n",
|
||||
" from google.colab import auth as google_auth\n",
|
||||
"\n",
|
||||
" google_auth.authenticate_user()\n",
|
||||
"\n",
|
||||
" # If you are running this notebook locally, replace the string below with the\n",
|
||||
" # path to your service account key and run this cell to authenticate your GCP\n",
|
||||
" # account.\n",
|
||||
" elif not os.getenv(\"IS_TESTING\"):\n",
|
||||
" %env GOOGLE_APPLICATION_CREDENTIALS ''"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "bucket:custom"
|
||||
},
|
||||
"source": [
|
||||
"### Create a Cloud Storage bucket\n",
|
||||
"\n",
|
||||
"**The following steps are required, regardless of your notebook environment.**\n",
|
||||
"\n",
|
||||
"When you create a dataset resource using the Vertex SDK, you can provide a Cloud Storage bucket that contains the data. Vertex AI creates the dataset resource from the data. In this tutorial, Vertex AI also creates a dataset resource from your data in the Cloud Storage bucket.\n",
|
||||
"\n",
|
||||
"Set the name of your Cloud Storage bucket below. Bucket names must be globally unique across all Google Cloud projects, including those outside of your organization."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "bucket"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"BUCKET_NAME = \"[your-bucket-name]\" # @param {type:\"string\"}\n",
|
||||
"BUCKET_URI = f\"gs://{BUCKET_NAME}\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "autoset_bucket"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
|
||||
" BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
|
||||
" BUCKET_URI = \"gs://\" + BUCKET_NAME"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "create_bucket"
|
||||
},
|
||||
"source": [
|
||||
"**Only if your bucket doesn't already exist**: Run the following cell to create your Cloud Storage bucket."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "create_bucket"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil mb -l $REGION $BUCKET_URI"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "validate_bucket"
|
||||
},
|
||||
"source": [
|
||||
"Finally, validate access to your Cloud Storage bucket by examining its contents:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "validate_bucket"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil ls -al $BUCKET_URI"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -383,7 +620,7 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"dataset = aiplatform.TabularDataset.create(\n",
|
||||
" display_name=\"NOAA historical weather data\" + \"_\" + UUID,\n",
|
||||
" display_name=\"NOAA historical weather data\" + \"_\" + TIMESTAMP,\n",
|
||||
" bq_source=[IMPORT_FILE],\n",
|
||||
" labels={\"user_metadata\": BUCKET_URI[5:]},\n",
|
||||
")\n",
|
||||
@@ -458,7 +695,7 @@
|
||||
"gcs_source = IMPORT_FILES\n",
|
||||
"\n",
|
||||
"dataset = aiplatform.TabularDataset.create(\n",
|
||||
" display_name=\"NOAA historical weather data\" + \"_\" + UUID,\n",
|
||||
" display_name=\"NOAA historical weather data\" + \"_\" + TIMESTAMP,\n",
|
||||
" gcs_source=gcs_source,\n",
|
||||
" labels={\"user_metadata\": BUCKET_URI[5:]},\n",
|
||||
")\n",
|
||||
@@ -500,10 +737,10 @@
|
||||
" or BQ_MY_DATASET is None\n",
|
||||
" or BQ_MY_DATASET == \"[your-dataset-name]\"\n",
|
||||
"):\n",
|
||||
" BQ_MY_DATASET = \"mlops_dataset_\" + UUID\n",
|
||||
" BQ_MY_DATASET = \"mlops_dataset_\" + TIMESTAMP\n",
|
||||
"\n",
|
||||
"if BQ_MY_TABLE == \"\" or BQ_MY_TABLE is None or BQ_MY_TABLE == \"[your-view-name]\":\n",
|
||||
" BQ_MY_TABLE = \"mlops_view_\" + UUID"
|
||||
" BQ_MY_TABLE = \"mlops_view_\" + TIMESTAMP"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
@@ -44,7 +44,7 @@
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage1/get_started_dataflow.ipynb\">\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/get_started_dataflow.ipynb\">\n",
|
||||
" <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
|
||||
" Open in Vertex AI Workbench\n",
|
||||
" </a>\n",
|
||||
@@ -186,9 +186,13 @@
|
||||
"if IS_WORKBENCH_NOTEBOOK:\n",
|
||||
" USER_FLAG = \"--user\"\n",
|
||||
"\n",
|
||||
"extra_pkgs = \"tensorflow==2.5 tensorflow-data-validation==1.2 tensorflow-transform==1.2 \\\n",
|
||||
" tensorflow-io==0.18 pyarrow pandas apache-beam[gcp] google-cloud-bigquery\"\n",
|
||||
"! pip3 install --upgrade --quiet {USER_FLAG} google-cloud-aiplatform $extra_pkgs"
|
||||
"! pip3 install -U tensorflow==2.5 $USER_FLAG -q\n",
|
||||
"! pip3 install -U tensorflow-data-validation==1.2 $USER_FLAG -q\n",
|
||||
"! pip3 install -U tensorflow-transform==1.2 $USER_FLAG -q\n",
|
||||
"! pip3 install -U tensorflow-io==0.18 $USER_FLAG -q\n",
|
||||
"! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG -q\n",
|
||||
"! pip3 install --upgrade google-cloud-bigquery $USER_FLAG -q\n",
|
||||
"! pip3 install --upgrade apache-beam[gcp] $USER_FLAG -q"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -210,9 +214,9 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import sys\n",
|
||||
"import os\n",
|
||||
"\n",
|
||||
"if \"google.colab\" in sys.modules:\n",
|
||||
"if not os.getenv(\"IS_TESTING\"):\n",
|
||||
" # Automatically restart kernel after installs\n",
|
||||
" import IPython\n",
|
||||
"\n",
|
||||
@@ -223,42 +227,279 @@
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "fc8fb52b5cca"
|
||||
"id": "84cd83853240"
|
||||
},
|
||||
"source": [
|
||||
"### Common setup\n",
|
||||
"## Before you begin\n",
|
||||
"\n",
|
||||
"Now, execute the common setup for the notebook tutorials."
|
||||
"### Set up your Google Cloud project\n",
|
||||
"\n",
|
||||
"**The following steps are required, regardless of your notebook environment.**\n",
|
||||
"\n",
|
||||
"1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
|
||||
"\n",
|
||||
"1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
|
||||
"\n",
|
||||
"1. [Enable the Vertex AI, BigQuery, Compute Engine and Cloud Storage APIs](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,bigquery,compute_component,storage_component).\n",
|
||||
"\n",
|
||||
"1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
|
||||
"\n",
|
||||
"1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
|
||||
"Cloud SDK uses the right project for all the commands in this notebook.\n",
|
||||
"\n",
|
||||
"**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "project_id"
|
||||
},
|
||||
"source": [
|
||||
"#### Set your project ID\n",
|
||||
"\n",
|
||||
"**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "001a0fcd5d78"
|
||||
"id": "set_project_id"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Common code setup for notebook tutorials\n",
|
||||
"\n",
|
||||
"! wget https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/setup.py -O setup.py\n",
|
||||
"\n",
|
||||
"%run setup.py --bucket"
|
||||
"PROJECT_ID = \"[your-project-id]\" # @param {type:\"string\"}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "d809f07a8935"
|
||||
"id": "autoset_project_id"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Other Common setup instructions for notebook tutorials\n",
|
||||
"if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
|
||||
" # Get your GCP project id from gcloud\n",
|
||||
" shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
|
||||
" PROJECT_ID = shell_output[0]\n",
|
||||
" print(\"Project ID:\", PROJECT_ID)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "set_gcloud_project_id"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gcloud config set project $PROJECT_ID"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "region"
|
||||
},
|
||||
"source": [
|
||||
"#### Region\n",
|
||||
"\n",
|
||||
"! wget https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/setup.md -O setup.md\n",
|
||||
"You can also change the `REGION` variable, which is used for operations\n",
|
||||
"throughout the rest of this notebook. Below are regions supported for Vertex AI. We recommend that you choose the region closest to you.\n",
|
||||
"\n",
|
||||
"%load setup.md "
|
||||
"- Americas: `us-central1`\n",
|
||||
"- Europe: `europe-west4`\n",
|
||||
"- Asia Pacific: `asia-east1`\n",
|
||||
"\n",
|
||||
"You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
|
||||
"\n",
|
||||
"Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "region"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"REGION = \"[your-region]\" # @param {type: \"string\"}\n",
|
||||
"\n",
|
||||
"if REGION == \"[your-region]\":\n",
|
||||
" REGION = \"us-central1\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "timestamp"
|
||||
},
|
||||
"source": [
|
||||
"#### Timestamp\n",
|
||||
"\n",
|
||||
"If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "timestamp"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from datetime import datetime\n",
|
||||
"\n",
|
||||
"TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "77c385f0db59"
|
||||
},
|
||||
"source": [
|
||||
"### Authenticate your Google Cloud account\n",
|
||||
"\n",
|
||||
"**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
|
||||
"\n",
|
||||
"**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
|
||||
"\n",
|
||||
"**Otherwise**, follow these steps:\n",
|
||||
"\n",
|
||||
"In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
|
||||
"\n",
|
||||
"1. **Click Create service account**.\n",
|
||||
"\n",
|
||||
"2. In the **Service account name** field, enter a name, and click **Create**.\n",
|
||||
"\n",
|
||||
"3. In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex AI\" into the filter box, and select **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
|
||||
"\n",
|
||||
"4. Click Create. A JSON file that contains your key downloads to your local environment.\n",
|
||||
"\n",
|
||||
"5. Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "535223fa4b84"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# If you are running this notebook in Colab, run this cell and follow the\n",
|
||||
"# instructions to authenticate your GCP account. This provides access to your\n",
|
||||
"# Cloud Storage bucket and lets you submit training jobs and prediction\n",
|
||||
"# requests.\n",
|
||||
"\n",
|
||||
"import os\n",
|
||||
"import sys\n",
|
||||
"\n",
|
||||
"# If on Vertex AI Workbench, then don't execute this code\n",
|
||||
"IS_COLAB = False\n",
|
||||
"if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
|
||||
" \"DL_ANACONDA_HOME\"\n",
|
||||
"):\n",
|
||||
" if \"google.colab\" in sys.modules:\n",
|
||||
" IS_COLAB = True\n",
|
||||
" from google.colab import auth as google_auth\n",
|
||||
"\n",
|
||||
" google_auth.authenticate_user()\n",
|
||||
"\n",
|
||||
" # If you are running this notebook locally, replace the string below with the\n",
|
||||
" # path to your service account key and run this cell to authenticate your GCP\n",
|
||||
" # account.\n",
|
||||
" elif not os.getenv(\"IS_TESTING\"):\n",
|
||||
" %env GOOGLE_APPLICATION_CREDENTIALS ''"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "bucket:custom"
|
||||
},
|
||||
"source": [
|
||||
"### Create a Cloud Storage bucket\n",
|
||||
"\n",
|
||||
"**The following steps are required, regardless of your notebook environment.**\n",
|
||||
"\n",
|
||||
"When you submit a custom training job using the Vertex SDK, you upload a Python package\n",
|
||||
"containing your training code to a Cloud Storage bucket. Vertex AI runs\n",
|
||||
"the code from this package. In this tutorial, Vertex AI also saves the\n",
|
||||
"trained model that results from your job in the same bucket. You can then\n",
|
||||
"create an `Endpoint` resource based on this output in order to serve\n",
|
||||
"online predictions.\n",
|
||||
"\n",
|
||||
"Set the name of your Cloud Storage bucket below. Bucket names must be globally unique across all Google Cloud projects, including those outside of your organization."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "bucket"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"BUCKET_NAME = \"[your-bucket-name]\" # @param {type:\"string\"}\n",
|
||||
"BUCKET_URI = f\"gs://{BUCKET_NAME}\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "autoset_bucket"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
|
||||
" BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
|
||||
" BUCKET_URI = \"gs://\" + BUCKET_NAME"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "create_bucket"
|
||||
},
|
||||
"source": [
|
||||
"**Only if your bucket doesn't already exist**: Run the following cell to create your Cloud Storage bucket."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "create_bucket"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil mb -l $REGION $BUCKET_URI"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "validate_bucket"
|
||||
},
|
||||
"source": [
|
||||
"Finally, validate access to your Cloud Storage bucket by examining its contents:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "validate_bucket"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil ls -al $BUCKET_URI"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1078,7 +1319,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"delete_storage = False\n",
|
||||
"delete_storage = True\n",
|
||||
"\n",
|
||||
"if delete_storage or os.getenv(\"IS_TESTING\"):\n",
|
||||
" if \"BUCKET_URI\" in globals():\n",
|
||||
|
||||
@@ -33,12 +33,12 @@
|
||||
"\n",
|
||||
"<table align=\"left\">\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/get_started_vertex_datasets.ipynb\">\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/get_started_vertex_datasets.ipynb\">\n",
|
||||
" <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/get_started_vertex_datasets.ipynb\">\n",
|
||||
" <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/get_started_vertex_datasets.ipynb\">\n",
|
||||
"<img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
|
||||
@@ -39,15 +39,18 @@
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/mlops_data_management.ipynb\">\n",
|
||||
" <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\\\" alt=\"Colab logo\"> Run in Colab\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage1/mlops_data_management.ipynb\">\n",
|
||||
" <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
|
||||
" Open in Vertex AI Workbench\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
"</table>\n",
|
||||
"<br/><br/><br/>\n",
|
||||
"\n",
|
||||
"*Note: This notebook is not supported for execution in Colab*"
|
||||
"<br/><br/><br/>"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -166,20 +169,21 @@
|
||||
"if IS_WORKBENCH_NOTEBOOK:\n",
|
||||
" USER_FLAG = \"--user\"\n",
|
||||
"\n",
|
||||
"ONCE_ONLY = False\n",
|
||||
"ONCE_ONLY = True\n",
|
||||
"if ONCE_ONLY:\n",
|
||||
" ! pip3 install -U {USER_FLAG} -q tensorflow==2.5 \\\n",
|
||||
" tensorflow-data-validation==1.2 \\\n",
|
||||
" tensorflow-transform==1.2 \\\n",
|
||||
" tensorflow-io==0.18 \n",
|
||||
" \n",
|
||||
" ! pip3 install --upgrade {USER_FLAG} -q google-cloud-aiplatform[tensorboard] \\\n",
|
||||
" google-cloud-pipeline-components \\\n",
|
||||
" google-cloud-bigquery \\\n",
|
||||
" google-cloud-logging \\\n",
|
||||
" apache-beam[gcp] \\\n",
|
||||
" pyarrow \\\n",
|
||||
" cloudml-hypertune\n"
|
||||
" ! pip3 install -U tensorflow==2.5 $USER_FLAG -q\n",
|
||||
" ! pip3 install -U tensorflow-data-validation==1.2 $USER_FLAG -q\n",
|
||||
" ! pip3 install -U tensorflow-transform==1.2 $USER_FLAG -q\n",
|
||||
" ! pip3 install -U tensorflow-io==0.18 $USER_FLAG -q\n",
|
||||
" ! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG -q\n",
|
||||
" ! pip3 install --upgrade google-cloud-pipeline-components $USER_FLAG -q\n",
|
||||
" ! pip3 install --upgrade google-cloud-bigquery $USER_FLAG -q\n",
|
||||
" ! pip3 install --upgrade google-cloud-logging $USER_FLAG -q\n",
|
||||
" ! pip3 install --upgrade apache-beam[gcp]==2.33.0 $USER_FLAG -q\n",
|
||||
" ! pip3 install --upgrade pyarrow $USER_FLAG -q\n",
|
||||
" ! pip3 install --upgrade cloudml-hypertune $USER_FLAG -q\n",
|
||||
" ! pip3 install --upgrade kfp $USER_FLAG -q\n",
|
||||
" ! pip3 install future $USER_FLAG -q"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -351,7 +355,7 @@
|
||||
"source": [
|
||||
"### Authenticate your Google Cloud account\n",
|
||||
"\n",
|
||||
"**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. \n",
|
||||
"**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
|
||||
"\n",
|
||||
"**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
|
||||
"\n",
|
||||
@@ -413,7 +417,7 @@
|
||||
"\n",
|
||||
"**The following steps are required, regardless of your notebook environment.**\n",
|
||||
"\n",
|
||||
"When you submit a custom training job using the Vertex AI SDK, you upload a Python package\n",
|
||||
"When you submit a custom training job using the Vertex SDK, you upload a Python package\n",
|
||||
"containing your training code to a Cloud Storage bucket. Vertex AI runs\n",
|
||||
"the code from this package. In this tutorial, Vertex AI also saves the\n",
|
||||
"trained model that results from your job in the same bucket. You can then\n",
|
||||
|
||||
@@ -35,59 +35,44 @@ The second stage in MLOps is experimenting in developing one or more baseline mo
|
||||
|
||||
### Get Started
|
||||
|
||||
[Get started with Vertex AI Training for R](community/ml_ops/stage2/get_started_vertex_training_r.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use `Vertex AI Training` for training a R custom model.
|
||||
[Get started with Vertex AI Training for Pytorch](get_started_vertex_training_pytorch.ipynb)
|
||||
|
||||
```
|
||||
The steps performed include:
|
||||
|
||||
- Locally train an R model in a notebook using %%R magic commands
|
||||
- Create a deployment image with trained R model and serving functions.
|
||||
- Test the deployment image locally.
|
||||
- Create a `Vertex AI Model` resource for the deployment image with embedded R model.
|
||||
- Deploy the deployment image with embedded R model to a `Vertex AI Endpoint` resource.
|
||||
- Test the deployment image with embedded R model.
|
||||
- Create a R-to-Python training package.
|
||||
- Create a training image for training the model.
|
||||
- Train a R model using `Vertex AI Trainingh` service with the R-to-Python training package.
|
||||
|
||||
[Get started with Logging](community/ml_ops/stage2/get_started_with_logging.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use Python and Cloud logging awhen training with `Vertex AI`.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Use Python logging to log training configuration/results locally.
|
||||
- Use Google Cloud Logging to log training configuration/results in cloud storage.
|
||||
|
||||
[Get started with Vertex AI Hyperparameter Tuning for XGBoost] (community/ml_ops/stage2/get_started_vertex_hpt_xgboost.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use `Vertex AI Hyperparameter Tuning` for training a XGBoost custom model.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Training using a Python package.
|
||||
- Single node training using a Python package.
|
||||
- Report accuracy when hyperparameter tuning.
|
||||
- Save the model artifacts to Cloud Storage using GCSFuse.
|
||||
- Create a `Vertex AI Model` resource.
|
||||
```
|
||||
|
||||
[Get started with Vertex AI Training for XGBoost](community/ml_ops/stage2/get_started_vertex_training_xgboost.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use `Vertex AI Training` for training a XGBoost custom model.
|
||||
[Get started with prebuilt TFHub models](get_started_with_tfhub_models.ipynb)
|
||||
|
||||
```
|
||||
The steps performed include:
|
||||
- Download a TensorFlow Hub prebuilt model.
|
||||
- Add the task component as a classifier for the CIFAR-10 dataset.
|
||||
- Fine tune locally the model with transfer learning training.
|
||||
- Construct a custom training script:
|
||||
- Get training data from TensorFlow Datasets
|
||||
- Get model architecture from TensorFlow Hub
|
||||
- Train then model
|
||||
- Save model artifacts and upload as Vertex AI Model resource.
|
||||
```
|
||||
|
||||
- Training using a Python package.
|
||||
- Report accuracy when hyperparameter tuning.
|
||||
- Save the model artifacts to Cloud Storage using GCSFuse.
|
||||
- Create a `Vertex AI Model` resource.
|
||||
|
||||
[Get started with TabNet builtin algorithm for training tabular models](community/ml_ops/stage2/get_started_with_tabnet.ipynb)
|
||||
|
||||
In this notebook, you learn how to run `Vertex AI TabNet` built algorithm for training custom tabular models.
|
||||
[Get started with Vertex AI TensorBoard](get_started_vertex_tensorboard.ipynb)
|
||||
|
||||
```
|
||||
The steps performed include:
|
||||
- Create a TensorBoard callback when training a model.
|
||||
- Using Tensorboard with locally trained model.
|
||||
- Using Vertex AI TensorBoard with Vertex AI Training.
|
||||
```
|
||||
|
||||
[Get started with TabNet builtin algorithm for training tabular models](get_started_with_tabnet.ipynb)
|
||||
|
||||
```
|
||||
The steps performed include:
|
||||
- Get the training data.
|
||||
- Configure training parameters for the `Vertex AI TabNet` container.
|
||||
- Train the model using `Vertex AI Training` using CSV data.
|
||||
@@ -97,107 +82,50 @@ The steps performed include:
|
||||
- Hyperparameter tuning the `Vertex AI TabNet` model.
|
||||
- Train the model using `Vertex AI Training` using BigQuery table.
|
||||
|
||||
[Get started with prebuilt TFHub models](community/ml_ops/stage2/get_started_with_tfhub_models.ipynb)
|
||||
```
|
||||
|
||||
In this tutorial, you learn how to use `Vertex AI Training` with prebuilt models from TensorFlow Hub.
|
||||
[Get started with Vertex AI Vizier](get_started_vertex_vizier.ipynb)
|
||||
|
||||
```
|
||||
The steps performed include:
|
||||
|
||||
- Download a TensorFlow Hub prebuilt model.
|
||||
- Add the task component as a classifier for the CIFAR-10 dataset.
|
||||
- Fine tune locally the model with transfer learning training.
|
||||
- Construct a custom training script:
|
||||
- Get training data from TensorFlow Datasets
|
||||
- Get model architecture from TensorFlow Hub
|
||||
- Train then model
|
||||
- Save model artifacts and upload as Vertex AI Model resource.
|
||||
|
||||
[Get started with BigQuery ML Training](community/ml_ops/stage2/get_started_bqml_training.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use `BigQueryML` (BQML) for training with `Vertex AI`.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Create a local BigQuery table in your project
|
||||
- Train a BQML model
|
||||
- Evaluate the BQML model
|
||||
- Export the BQML model as a cloud model
|
||||
- Upload the exported model as a `Vertex AI Model` resource
|
||||
- Hyperparameter tune a BQML model with `Vertex AI Vizier`
|
||||
- Automatically register a BQML model to `Vertex AI Model Registry`
|
||||
|
||||
[Get started with Vertex AI Vizier](community/ml_ops/stage2/get_started_vertex_vizier.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use `Vertex AI Vizier` for when training with `Vertex AI`.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Hyperparameter tuning with Random algorithm.
|
||||
- Hyperparameter tuning with Vizier (Bayesian) algorithm.
|
||||
- Suggesting trials and updating results for Vizier study
|
||||
```
|
||||
|
||||
[Get started with distributed training using DASK](community/ml_ops/stage2/get_started_with_distributed_training_xgboost.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use `Vertex AI Training` for distributed training of XGBoost model using the OSS package DASK. Additionally, you learn to construct and deploy a custom serving container using a Flask web server.
|
||||
[Automl image classfication training with customer managed encryption keys (CMEK)](get_started_with_cmek_training.ipynb)
|
||||
|
||||
```
|
||||
The steps performed include:
|
||||
- Creating a customer managed encryption key.
|
||||
- Creating an image dataset with CMEK encryption.
|
||||
- Train an AutoML model with CMEK encryption.
|
||||
```
|
||||
|
||||
- Construct an XGBoost training script using DASK for distributed training.
|
||||
- Construct a custom training container.
|
||||
- Configure a distributed custom training job.
|
||||
- Execute the custom training job.
|
||||
- Construct a custom serving container using Flask.
|
||||
- Upload the trained XGBoost model as a `Vertex AI Model` resource.
|
||||
- Create a `Vertex AI Endpoint` resource.
|
||||
- Deploy the `Vertex AI Model` resource to `Vertex AI Endpoint` resource.
|
||||
- Make a prediction.
|
||||
|
||||
[Get started with Vertex AI TensorBoard](community/ml_ops/stage2/get_started_vertex_tensorboard.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use `Vertex AI TensorBoard` when training with `Vertex AI`.
|
||||
[Get started with Vertex AI distributed training](get_started_vertex_distributed_training.ipynb)
|
||||
|
||||
```
|
||||
The steps performed include:
|
||||
- `MirroredStrategy`: Train on a single VM with multiple GPUs.
|
||||
- `MultiWorkerMirroredStrategy`: Train on multiple VMs with automatic setup of replicas.
|
||||
- `MultiWorkerMirroredStrategy`: Train on multiple VMs with fine grain control of replicas.
|
||||
- `ReductionServer`: Train on multiple VMS and sync updates across VMS with `Vertex AI Reduction Server`.
|
||||
- `TPUTraining`: Train with multiple Cloud TPUs.
|
||||
```
|
||||
|
||||
- Create a TensorBoard callback when training a model.
|
||||
- Using TensorBoard with locally trained model.
|
||||
- Using Vertex AI TensorBoard with Vertex AI Training.
|
||||
|
||||
[Get started with Vertex AI Training for R using R Kernel](community/ml_ops/stage2/get_started_vertex_training_r_using_r_kernel.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use `Vertex AI`, using an R kernel, for training and deploying an R custom model.
|
||||
[Get started with Vertex AI Training for scikit-learn](get_started_vertex_training_sklearn.ipynb)
|
||||
|
||||
```
|
||||
The steps performed include:
|
||||
- Training using a Python package.
|
||||
- Report accuracy when hyperparameter tuning.
|
||||
- Save the model artifacts to Cloud Storage using GCSFuse.
|
||||
- Create a `Vertex AI Model` resource.
|
||||
```
|
||||
|
||||
- Create a custom R training script
|
||||
- Create a custom R serving script
|
||||
- Create a custom R deployment (serving) container.
|
||||
- Train the model using `Vertex AI` custom training.
|
||||
- Create an `Endpoint` resouce.
|
||||
- Deploy the `Model` resource (trained R model) to the `Endpoint` resource.
|
||||
- Make an online prediction.
|
||||
|
||||
|
||||
[Get started Vision API test preprocessing and AutoML text model generation](community/ml_ops/stage2/get_started_with_visionapi_and_automl.ipynb)
|
||||
|
||||
In this tutorial, you create an `AutoML` text entity extraction model pre-existing extracted data by generating a custom import file. You deploy this mode for online prediction from a Python script using the `BigQuery`, `Vision AI`, Cloud Storage and `Vertex AI SDK` for Python.
|
||||
[Get started with Vertex AI Experiments](get_started_vertex_experiments.ipynb)
|
||||
|
||||
```
|
||||
The steps performed include:
|
||||
|
||||
- Preprocess training files using `Vision AI` APIs to extract the text from PDF files.
|
||||
- Create a custom import file that includes annotation data based on the sample `BigQuery` dataset.
|
||||
- Create a `Vertex AI Dataset` resource.
|
||||
- Train the model.
|
||||
- View the model evaluation.
|
||||
- Deploy the `Vertex AI Model` resource to a serving `Endpoint` resource.
|
||||
- Make a prediction.
|
||||
- Undeploy the `Model`.
|
||||
|
||||
[Get started with Vertex AI Experiments](community/ml_ops/stage2/get_started_vertex_experiments.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use `Vertex AI Experiments` when training with `Vertex AI`.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Local (notebook) Training
|
||||
- Create an experiment
|
||||
- Create a first run in the experiment
|
||||
@@ -214,23 +142,23 @@ The steps performed include:
|
||||
- Create a `Vertex AI Training` custom job
|
||||
- Execute the custom job
|
||||
- Visualize the experiment results
|
||||
```
|
||||
|
||||
[AutoML Image Classfication Training with Customer Managed Encryption Keys (CMEK)](community/ml_ops/stage2/get_started_with_cmek_training.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use a customer managed encryption key (CMEK) for `Vertex AI AutoML` training.
|
||||
[Get started with Vertex AI Hyperparameter Tuning for XGBoost](get_started_vertex_hpt_xgboost.ipynb)
|
||||
|
||||
```
|
||||
The steps performed include:
|
||||
- Training using a Python package.
|
||||
- Report accuracy when hyperparameter tuning.
|
||||
- Save the model artifacts to Cloud Storage using GCSFuse.
|
||||
- Create a `Vertex AI Model` resource.
|
||||
|
||||
- Creating a customer managed encryption key.
|
||||
- Creating an image dataset with CMEK encryption.
|
||||
- Train an AutoML model with CMEK encryption.
|
||||
```
|
||||
|
||||
[Get started with Vertex AI Feature Store](community/ml_ops/stage2/get_started_vertex_feature_store.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use `Vertex AI Feature Store` when training and predicting with `Vertex AI`.
|
||||
[Get started with Vertex AI Feature Store](get_started_vertex_feature_store.ipynb)
|
||||
|
||||
```
|
||||
The steps performed include:
|
||||
|
||||
- Creating a Vertex AI `Featurestore` resource.
|
||||
- Creating `EntityType` resources for the `Featurestore` resource.
|
||||
- Creating `Feature` resources for each `EntityType` resource.
|
||||
@@ -239,26 +167,96 @@ The steps performed include:
|
||||
- From a pandas DataFrame.
|
||||
- Perform online serving from a `Featurestore` resource.
|
||||
- Perform batch serving from a `Featurestore` resource.
|
||||
```
|
||||
|
||||
[Get started with AutoML Training](community/ml_ops/stage2/get_started_automl_training.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use `AutoML` for training with `Vertex AI`.
|
||||
[Get started with Vertex AI Training for R](get_started_vertex_training_r.ipynb)
|
||||
|
||||
```
|
||||
The steps performed include:
|
||||
- Locally train an R model in a notebook using %%R magic commands
|
||||
- Create a deployment image with trained R model and serving functions.
|
||||
- Test the deployment image locally.
|
||||
- Create a `Vertex AI Model` resource for the deployment image with embedded R model.
|
||||
- Deploy the deployment image with embedded R model to a `Vertex AI Endpoint` resource.
|
||||
- Test the deployment image with embedded R model.
|
||||
- Create a R-to-Python training package.
|
||||
- Create a training image for training the model.
|
||||
- Train a R model using `Vertex AI Trainingh` service with the R-to-Python training package.
|
||||
```
|
||||
|
||||
[Get started with logging](get_started_with_logging.ipynb)
|
||||
|
||||
```
|
||||
The steps performed include:
|
||||
- Use Python logging to log training configuration/results locally.
|
||||
- Use Google Cloud Logging to log training configuration/results in cloud storage.
|
||||
```
|
||||
|
||||
[Get started with Vertex AI Training for R using R Kernel](get_started_vertex_training_r_using_r_kernel.ipynb)
|
||||
|
||||
```
|
||||
The steps performed include:
|
||||
- Create a custom R training script
|
||||
- Create a custom R serving script
|
||||
- Create a custom R deployment (serving) container.
|
||||
- Train the model using `Vertex AI` custom training.
|
||||
- Create an `Endpoint` resouce.
|
||||
- Deploy the `Model` resource (trained R model) to the `Endpoint` resource.
|
||||
- Make an online prediction.
|
||||
|
||||
```
|
||||
|
||||
[Get started with BigQuery ML training](get_started_bqml_training.ipynb)
|
||||
|
||||
```
|
||||
The steps performed include:
|
||||
- Create a local BigQuery table in your project
|
||||
- Train a BQML model
|
||||
- Evaluate the BQML model
|
||||
- Export the BQML model as a cloud model
|
||||
- Upload the exported model as a `Vertex AI Model` resource
|
||||
- Hyperparameter tune a BQML model with `Vertex AI Vizier`
|
||||
- Automatically register a BQML model to `Vertex AI Model Registry`
|
||||
|
||||
```
|
||||
|
||||
[Get started with AutoML training](get_started_automl_training.ipynb)
|
||||
|
||||
```
|
||||
The steps performed include:
|
||||
- Train an image model
|
||||
- Export the image model as an edge model
|
||||
- Train a tabular model
|
||||
- Export the tabular model as a cloud model
|
||||
- Train a text model
|
||||
- Train a video model
|
||||
```
|
||||
|
||||
[Get started with Vertex AI Training for LightGBM](community/ml_ops/stage2/get_started_vertex_training_lightgbm.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use `Vertex AI Training` for training a LightGBM custom model.
|
||||
[Get started with Vertex AI Training for XGBoost](get_started_vertex_training_xgboost.ipynb)
|
||||
|
||||
```
|
||||
The steps performed include:
|
||||
- Training using a Python package.
|
||||
- Report accuracy when hyperparameter tuning.
|
||||
- Save the model artifacts to Cloud Storage using GCSFuse.
|
||||
- Create a `Vertex AI Model` resource.
|
||||
|
||||
```
|
||||
|
||||
[Get started with Vertex AI Training](get_started_vertex_training.ipynb)
|
||||
|
||||
```
|
||||
The steps performed include:
|
||||
- Training using a single Python script.
|
||||
- Training using a Python package.
|
||||
- Training using a custom training image.
|
||||
- Laying out a training package.
|
||||
```
|
||||
|
||||
[Get started with Vertex AI Training for LightGBM](get_started_vertex_training_lightgbm.ipynb)
|
||||
|
||||
```
|
||||
The steps performed include:
|
||||
- Training using a Python package.
|
||||
- Save the model artifacts to Cloud Storage using GCSFuse.
|
||||
- Construct a FastAPI prediction server.
|
||||
@@ -266,51 +264,21 @@ The steps performed include:
|
||||
- Test the deployment image locally.
|
||||
- Create a `Vertex AI Model` resource.
|
||||
|
||||
[Get started with Vertex AI Training for Scikit-Learn](community/ml_ops/stage2/get_started_vertex_training_sklearn.ipynb)
|
||||
```
|
||||
|
||||
In this tutorial, you learn how to use `Vertex AI Training` for training a Scikit-Learn custom model.
|
||||
[Get started Vision API test preprocessing and AutoML text model generation](get_started_with_visionapi_and_automl.ipynb)
|
||||
|
||||
```
|
||||
The steps performed include:
|
||||
|
||||
- Training using a Python package.
|
||||
- Report accuracy when hyperparameter tuning.
|
||||
- Save the model artifacts to Cloud Storage using GCSFuse.
|
||||
- Create a `Vertex AI Model` resource.
|
||||
|
||||
[Get started with Vertex AI Training](community/ml_ops/stage2/get_started_vertex_training.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use `Vertex AI Training` for custom models when training with `Vertex AI`.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Training using a single Python script.
|
||||
- Training using a Python package.
|
||||
- Training using a custom training image.
|
||||
- Laying out a training package.
|
||||
|
||||
|
||||
[Get started with Vertex AI Training for Pytorch](community/ml_ops/stage2/get_started_vertex_training_pytorch.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use `Vertex AI Training` for training a Pytorch custom model.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Single node training using a Python package.
|
||||
- Report accuracy when hyperparameter tuning.
|
||||
- Save the model artifacts to Cloud Storage using GCSFuse.
|
||||
- Create a `Vertex AI Model` resource.
|
||||
|
||||
[Get started with Vertex AI Distributed Training](community/ml_ops/stage2/get_started_vertex_distributed_training.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use `Vertex AI Distributed Training` for when training with `Vertex AI`.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- `MirroredStrategy`: Train on a single VM with multiple GPUs.
|
||||
- `MultiWorkerMirroredStrategy`: Train on multiple VMs with automatic setup of replicas.
|
||||
- `MultiWorkerMirroredStrategy`: Train on multiple VMs with fine grain control of replicas.
|
||||
- `ReductionServer`: Train on multiple VMS and sync updates across VMS with `Vertex AI Reduction Server`.
|
||||
- `TPUTraining`: Train with multiple Cloud TPUs.
|
||||
- Preprocess training files using `Vision AI` APIs to extract the text from PDF files.
|
||||
- Create a custom import file that includes annotation data based on the sample `BigQuery` dataset.
|
||||
- Create a `Vertex AI Dataset` resource.
|
||||
- Train the model.
|
||||
- View the model evaluation.
|
||||
- Deploy the `Vertex AI Model` resource to a serving `Endpoint` resource.
|
||||
- Make a prediction.
|
||||
- Undeploy the `Model`.
|
||||
```
|
||||
|
||||
### E2E Stage Example
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -73,7 +73,7 @@
|
||||
"source": [
|
||||
"### Objective\n",
|
||||
"\n",
|
||||
"In this tutorial, you learn how to use `BigQueryML` for training with `Vertex AI`.\n",
|
||||
"In this tutorial, you learn how to use `BigQueryML` (BQML) for training with `Vertex AI`.\n",
|
||||
"\n",
|
||||
"This tutorial uses the following Google Cloud ML services:\n",
|
||||
"\n",
|
||||
@@ -84,12 +84,12 @@
|
||||
"The steps performed include:\n",
|
||||
"\n",
|
||||
"- Create a local BigQuery table in your project\n",
|
||||
"- Train a BigQuery ML model\n",
|
||||
"- Evaluate the BigQuery ML model\n",
|
||||
"- Export the BigQuery ML model as a cloud model\n",
|
||||
"- Train a BQML model\n",
|
||||
"- Evaluate the BQML model\n",
|
||||
"- Export the BQML model as a cloud model\n",
|
||||
"- Upload the exported model as a `Vertex AI Model` resource\n",
|
||||
"- Hyperparameter tune a BigQuery ML model with `Vertex AI Vizier`\n",
|
||||
"- Automatically register a BigQuery ML model to `Vertex AI Model Registry`"
|
||||
"- Hyperparameter tune a BQML model with `Vertex AI Vizier`\n",
|
||||
"- Automatically register a BQML model to `Vertex AI Model Registry`"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -749,9 +749,9 @@
|
||||
"id": "bqml_create_model"
|
||||
},
|
||||
"source": [
|
||||
"### Train BigQuery ML model\n",
|
||||
"### Train BQML model\n",
|
||||
"\n",
|
||||
"Next, you create and train a BigQuery ML tabular classification model from the public dataset penguins and store the model in your project using the `CREATE MODEL` statement. The model configuration is specified in the `OPTIONS` statement as follows:\n",
|
||||
"Next, you create and train a BQML tabular classification model from the public dataset penguins and store the model in your project using the `CREATE MODEL` statement. The model configuration is specified in the `OPTIONS` statement as follows:\n",
|
||||
"\n",
|
||||
"- `model_type`: The type and archictecture of tabular model to train, e.g., DNN classification.\n",
|
||||
"- `labels`: The column which are the labels.\n",
|
||||
@@ -800,9 +800,9 @@
|
||||
"id": "bqml_eval_model"
|
||||
},
|
||||
"source": [
|
||||
"### Evaluate the trained BigQuery ML model\n",
|
||||
"### Evaluate the trained BQML model\n",
|
||||
"\n",
|
||||
"Next, retrieve the model evaluation for the trained BigQuery ML model.\n",
|
||||
"Next, retrieve the model evaluation for the trained BQML model.\n",
|
||||
"\n",
|
||||
"Learn more about [The ML.EVALUATE function](https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-evaluate)."
|
||||
]
|
||||
@@ -833,9 +833,9 @@
|
||||
"id": "bqml_export_model"
|
||||
},
|
||||
"source": [
|
||||
"### Export the model from BigQuery ML\n",
|
||||
"### Export the model from BQML\n",
|
||||
"\n",
|
||||
"The model you trained in BigQuery ML is a TensorFlow model. Next, you export the TensorFlow model artifacts in TF.SavedModel format."
|
||||
"The model you trained in BQML is a TensorFlow model. Next, you export the TensorFlow model artifacts in TF.SavedModel format."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1028,9 +1028,9 @@
|
||||
"id": "bqml_create_model:vizier"
|
||||
},
|
||||
"source": [
|
||||
"### Hyperparameter Tune and train a BigQuery ML model\n",
|
||||
"### Hyperparameter Tune and train a BQML model\n",
|
||||
"\n",
|
||||
"Next, you train a BigQuery ML tabular classification model with hyperparameter tuning using the `Vertex AI Vizier` service. The hyperparameter settings are specified in the `OPTIONS` statement as follows:\n",
|
||||
"Next, you train a BQML tabular classification model with hyperparameter tuning using the `Vertex AI Vizier` service. The hyperparameter settings are specified in the `OPTIONS` statement as follows:\n",
|
||||
"\n",
|
||||
"- `HPARAM_TUNING_ALGORITHM`: The algorithm for selecting the next trial parameters.\n",
|
||||
"- `num_trials`: The number of trials.\n",
|
||||
@@ -1083,9 +1083,9 @@
|
||||
"id": "bqml_eval_model"
|
||||
},
|
||||
"source": [
|
||||
"### Evaluate the BigQuery ML trained model\n",
|
||||
"### Evaluate the BQML trained model\n",
|
||||
"\n",
|
||||
"Next, retrieve the model evaluation results for the trained BigQuery ML model.\n",
|
||||
"Next, retrieve the model evaluation results for the trained BQML model.\n",
|
||||
"\n",
|
||||
"Learn more about [The ML.EVALUATE function](https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-evaluate)."
|
||||
]
|
||||
@@ -1142,9 +1142,9 @@
|
||||
"id": "bqml_create_model:xai"
|
||||
},
|
||||
"source": [
|
||||
"### Train a BigQuery ML model with Explainability\n",
|
||||
"### Train a BQML model with Explainability\n",
|
||||
"\n",
|
||||
"Next, you train the same BigQuery ML model, but this time you enable Vertex AI Explainability on the model predictions by adding the option:\n",
|
||||
"Next, you train the same BQML model, but this time you enable Vertex AI Explainability on the model predictions by adding the option:\n",
|
||||
"\n",
|
||||
"- `ENABLE_GLOBAL_EXPLAIN`"
|
||||
]
|
||||
|
||||
@@ -87,7 +87,7 @@
|
||||
"This tutorial uses the following Google Cloud ML services:\n",
|
||||
"\n",
|
||||
"- `Vertex AI Experiments`\n",
|
||||
"- `Vertex ML Metadata`\n",
|
||||
"- `Vertex AI ML Metadata`\n",
|
||||
"- `Vertex AI Training`\n",
|
||||
"\n",
|
||||
"The steps performed include:\n",
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -44,7 +44,7 @@
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage2/get_started_vertex_tensorboard.ipynb\">\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/notebook_template.ipynb\">\n",
|
||||
" <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
|
||||
" Open in Vertex AI Workbench\n",
|
||||
" </a>\n",
|
||||
|
||||
@@ -48,7 +48,7 @@
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_training_lightgbm.ipynb\">\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/notebooks/community/ml_ops/stage2/get_started_vertex_training_lightgbm.ipynb\">\n",
|
||||
" <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
|
||||
@@ -29,7 +29,7 @@
|
||||
"id": "title:generic,gcp"
|
||||
},
|
||||
"source": [
|
||||
"# E2E ML on GCP: MLOps stage 2 : experimentation: get started with Vertex AI Training for PyTorch\n",
|
||||
"# E2E ML on GCP: MLOps stage 2 : experimentation: get started with Vertex AI Training for Pytorch\n",
|
||||
"\n",
|
||||
"<table align=\"left\">\n",
|
||||
" <td>\n",
|
||||
@@ -44,7 +44,7 @@
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage2/get_started_vertex_training_pytorch.ipynb\">\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/notebook_template.ipynb\">\n",
|
||||
" <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
|
||||
" Open in Vertex AI Workbench\n",
|
||||
" </a>\n",
|
||||
@@ -62,7 +62,7 @@
|
||||
"## Overview\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 2 : experimentation: get started with Vertex AI Training for PyTorch."
|
||||
"This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 2 : experimentation: get started with Vertex AI Training for Pytorch."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -73,7 +73,7 @@
|
||||
"source": [
|
||||
"### Objective\n",
|
||||
"\n",
|
||||
"In this tutorial, you learn how to use `Vertex AI Training` for training a PyTorch custom model.\n",
|
||||
"In this tutorial, you learn how to use `Vertex AI Training` for training a Pytorch custom model.\n",
|
||||
"\n",
|
||||
"This tutorial uses the following Google Cloud ML services:\n",
|
||||
"\n",
|
||||
@@ -97,7 +97,7 @@
|
||||
"source": [
|
||||
"### Dataset\n",
|
||||
"\n",
|
||||
"The dataset used for this tutorial is the [CIFAR10 dataset](https://pytorch.org/vision/stable/datasets.html#cifar) from [PyTorch Datasets](https://pytorch.org/vision/stable/datasets.html). The version of the dataset is built into TensorFlow. The trained model predicts which type of class an image is from ten classes: airplane, automobile, bird, cat, deer, dog, frog, horse, ship, or truck."
|
||||
"The dataset used for this tutorial is the [CIFAR10 dataset](https://pytorch.org/vision/stable/datasets.html#cifar) from [Pytorch Datasets](https://pytorch.org/vision/stable/datasets.html). The version of the dataset is built into TensorFlow. The trained model predicts which type of class an image is from ten classes: airplane, automobile, bird, cat, deer, dog, frog, horse, ship, or truck."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -672,17 +672,17 @@
|
||||
"id": "pytorch_intro"
|
||||
},
|
||||
"source": [
|
||||
"## Introduction to PyTorch training\n",
|
||||
"## Introduction to Pytorch training\n",
|
||||
"\n",
|
||||
"The PyTorch package supports both single node and distributed model training.\n",
|
||||
"The Pytorch package supports both single node and distributed model training.\n",
|
||||
"\n",
|
||||
"Once you have trained a PyTorch model, you will want to save it at a Cloud Storage location, so it can subsequently be uploaded to a `Vertex AI Model` resource.\n",
|
||||
"The PyTorch package does not have support to save the model to a Cloud Storage location. Instead, you will do the following steps to save to a Cloud Storage location.\n",
|
||||
"Once you have trained a Pytorch model, you will want to save it at a Cloud Storage location, so it can subsequently be uploaded to a `Vertex AI Model` resource.\n",
|
||||
"The Pytorch package does not have support to save the model to a Cloud Storage location. Instead, you will do the following steps to save to a Cloud Storage location.\n",
|
||||
"\n",
|
||||
"1. Save the in-memory model to the local filesystem (e.g., model.pth).\n",
|
||||
"2. Use gsutil to copy the local copy to the specified Cloud Storage location.\n",
|
||||
"\n",
|
||||
"*Note*: You can do hyperparameter tuning with a PyTorch model."
|
||||
"*Note*: You can do hyperparameter tuning with a Pytorch model."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1069,9 +1069,9 @@
|
||||
"id": "docker_write,prediction,pytorch"
|
||||
},
|
||||
"source": [
|
||||
"### Make PyTorch container for prediction\n",
|
||||
"### Make Pytorch container for prediction\n",
|
||||
"\n",
|
||||
"Currently, Vertex AI does not have a predefined container for making predictions with a deployed PyTorch model. No problem, you can assemble your own custom container. Typically, one would base the container on the `Torch Server`. For demonstration purpose, you build a placeholder container (not complete) that includes the latest `Torch Server` image, and push it to the `Container Registry`."
|
||||
"Currently, Vertex AI does not have a predefined container for making predictions with a deployed Pytorch model. No problem, you can assemble your own custom container. Typically, one would base the container on the `Torch Server`. For demonstration purpose, you build a placeholder container (not complete) that includes the latest `Torch Server` image, and push it to the `Container Registry`."
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
+2
-2
@@ -43,7 +43,7 @@
|
||||
"<table align=\"left\">\n",
|
||||
"\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_training_r_using_r_kernel.ipynb\">\n",
|
||||
" <a href=\"https://colab.sandbox.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_training_r_using_r_kernel.ipynb\">\n",
|
||||
" <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
@@ -109,7 +109,7 @@
|
||||
"source": [
|
||||
"### Dataset\n",
|
||||
"\n",
|
||||
"The dataset used for this tutorial is [California Housing Dataset](https://www.dcc.fc.up.pt/~ltorgo/Regression/cal_housing.html). The data contains information from the 1990 California census. The data set is publicly available from Cloud Storage at `gs://cloud-samples-data/ai-platform-unified/datasets/tabular/california-housing-tabular-regression.csv`. The dataset is used to train a Random Forest regressor to predict a median housing price, given a longitude and lattitude along with data from the corresponding census block group. A block group is the smallest geographical unit for which the U.S. Census Bureau publishes sample data (a block group typically has a population of 600 to 3,000 people).\n"
|
||||
"The dataset used for this tutorial is [California Housing Dataset](https://www.dcc.fc.up.pt/~ltorgo/Regression/cal_housing.html). The data contains information from the 1990 California census. The data set is publicly available from Google Cloud Storage at `gs://cloud-samples-data/ai-platform-unified/datasets/tabular/california-housing-tabular-regression.csv`. The dataset is used to train a Random Forest regressor to predict a median housing price, given a longitude and lattitude along with data from the corresponding census block group. A block group is the smallest geographical unit for which the U.S. Census Bureau publishes sample data (a block group typically has a population of 600 to 3,000 people).\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
@@ -44,7 +44,7 @@
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage2/get_started_vertex_training_sklearn.ipynb\">\n",
|
||||
" <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage2/get_started_vertex_training_sklearn.ipynb\">\n",
|
||||
" <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
|
||||
"Open in Vertex AI Workbench\n",
|
||||
" </a>\n",
|
||||
|
||||
@@ -297,32 +297,25 @@
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "06571eb4063b"
|
||||
"id": "timestamp"
|
||||
},
|
||||
"source": [
|
||||
"#### UUID\n",
|
||||
"#### Timestamp\n",
|
||||
"\n",
|
||||
"If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
|
||||
"If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "4e166d927e36"
|
||||
"id": "JYtXOocrox9Q"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import random\n",
|
||||
"import string\n",
|
||||
"from datetime import datetime\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Generate a uuid of a specifed length(default=8)\n",
|
||||
"def generate_uuid(length: int = 8) -> str:\n",
|
||||
" return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"UUID = generate_uuid()"
|
||||
"TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -369,11 +362,12 @@
|
||||
"import sys\n",
|
||||
"\n",
|
||||
"# If on Vertex AI Workbench, then don't execute this code\n",
|
||||
"IS_COLAB = \"google.colab\" in sys.modules\n",
|
||||
"IS_COLAB = False\n",
|
||||
"if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
|
||||
" \"DL_ANACONDA_HOME\"\n",
|
||||
"):\n",
|
||||
" if \"google.colab\" in sys.modules:\n",
|
||||
" IS_COLAB = True\n",
|
||||
" from google.colab import auth as google_auth\n",
|
||||
"\n",
|
||||
" google_auth.authenticate_user()\n",
|
||||
@@ -408,8 +402,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"BUCKET_NAME = \"[your-bucket-name]\" # @param {type:\"string\"}\n",
|
||||
"BUCKET_URI = f\"gs://{BUCKET_NAME}\""
|
||||
"BUCKET_URI = \"gs://[your-bucket-name]\" # @param {type:\"string\"}"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -420,9 +413,8 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
|
||||
" BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
|
||||
" BUCKET_URI = \"gs://\" + BUCKET_NAME"
|
||||
"if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
|
||||
" BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -757,7 +749,6 @@
|
||||
"import hypertune\n",
|
||||
"import argparse\n",
|
||||
"import logging\n",
|
||||
"import numpy as np\n",
|
||||
"\n",
|
||||
"from sklearn.model_selection import train_test_split\n",
|
||||
"from sklearn.metrics import accuracy_score\n",
|
||||
@@ -799,23 +790,16 @@
|
||||
"def train_model(dtrain):\n",
|
||||
" logging.info(\"Start training ...\")\n",
|
||||
" # Train XGBoost model\n",
|
||||
" params = {\n",
|
||||
" 'objective': 'multi:softprob',\n",
|
||||
" 'num_class': 3\n",
|
||||
" }\n",
|
||||
" model = xgb.train(params, dtrain, num_boost_round=args.boost_rounds)\n",
|
||||
" model = xgb.train({}, dtrain, num_boost_round=args.boost_rounds)\n",
|
||||
" logging.info(\"Training completed\")\n",
|
||||
" return model\n",
|
||||
"\n",
|
||||
"def evaluate_model(model, test_data, test_labels):\n",
|
||||
" dtest = xgb.DMatrix(test_data)\n",
|
||||
" pred = model.predict(dtest)\n",
|
||||
" predictions = [np.around(value) for value in pred]\n",
|
||||
" predictions = [round(value) for value in pred]\n",
|
||||
" # evaluate predictions\n",
|
||||
" try:\n",
|
||||
" accuracy = accuracy_score(test_labels, predictions)\n",
|
||||
" except:\n",
|
||||
" accuracy = 0.0\n",
|
||||
" accuracy = accuracy_score(test_labels, predictions)\n",
|
||||
" logging.info(f\"Evaluation completed with model accuracy: {accuracy}\")\n",
|
||||
"\n",
|
||||
" # report metric for hyperparameter tuning\n",
|
||||
@@ -909,7 +893,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"DISPLAY_NAME = \"iris_\" + UUID\n",
|
||||
"DISPLAY_NAME = \"iris_\" + TIMESTAMP\n",
|
||||
"\n",
|
||||
"job = aip.CustomPythonPackageTrainingJob(\n",
|
||||
" display_name=DISPLAY_NAME,\n",
|
||||
@@ -948,7 +932,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"MODEL_DIR = \"{}/{}\".format(BUCKET_URI, UUID)\n",
|
||||
"MODEL_DIR = \"{}/{}\".format(BUCKET_URI, TIMESTAMP)\n",
|
||||
"DATASET_DIR = \"gs://cloud-samples-data/ai-platform/iris\"\n",
|
||||
"\n",
|
||||
"ROUNDS = 20\n",
|
||||
@@ -999,7 +983,7 @@
|
||||
"source": [
|
||||
"if TRAIN_GPU:\n",
|
||||
" model = job.run(\n",
|
||||
" model_display_name=\"iris_\" + UUID,\n",
|
||||
" model_display_name=\"iris_\" + TIMESTAMP,\n",
|
||||
" args=CMDARGS,\n",
|
||||
" replica_count=1,\n",
|
||||
" machine_type=TRAIN_COMPUTE,\n",
|
||||
@@ -1010,7 +994,7 @@
|
||||
" )\n",
|
||||
"else:\n",
|
||||
" model = job.run(\n",
|
||||
" model_display_name=\"iris_\" + UUID,\n",
|
||||
" model_display_name=\"iris_\" + TIMESTAMP,\n",
|
||||
" args=CMDARGS,\n",
|
||||
" replica_count=1,\n",
|
||||
" machine_type=TRAIN_COMPUTE,\n",
|
||||
@@ -1111,7 +1095,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"delete_bucket = True\n",
|
||||
"delete_bucket = False\n",
|
||||
"\n",
|
||||
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
|
||||
" ! gsutil rm -r $BUCKET_URI"
|
||||
|
||||
@@ -84,8 +84,7 @@
|
||||
"The steps performed include:\n",
|
||||
"\n",
|
||||
"- Hyperparameter tuning with Random algorithm.\n",
|
||||
"- Hyperparameter tuning with Vizier (Bayesian) algorithm.\n",
|
||||
"- Suggesting trials and updating results for Vizier study"
|
||||
"- Hyperparameter tuning with Vizier (Bayesian) algorithm."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -188,8 +187,7 @@
|
||||
"if IS_WORKBENCH_NOTEBOOK:\n",
|
||||
" USER_FLAG = \"--user\"\n",
|
||||
"\n",
|
||||
"! pip3 install --upgrade $USER_FLAG -q google-cloud-aiplatform \\\n",
|
||||
" google-vizier==0.0.4"
|
||||
"! pip3 install --upgrade google-cloud-aiplatform[tensorboard] $USER_FLAG -q"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -331,32 +329,25 @@
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "06571eb4063b"
|
||||
"id": "timestamp"
|
||||
},
|
||||
"source": [
|
||||
"#### UUID\n",
|
||||
"#### Timestamp\n",
|
||||
"\n",
|
||||
"If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
|
||||
"If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "4e166d927e36"
|
||||
"id": "timestamp"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import random\n",
|
||||
"import string\n",
|
||||
"from datetime import datetime\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Generate a uuid of a specifed length(default=8)\n",
|
||||
"def generate_uuid(length: int = 8) -> str:\n",
|
||||
" return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"UUID = generate_uuid()"
|
||||
"TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -367,7 +358,7 @@
|
||||
"source": [
|
||||
"### Authenticate your Google Cloud account\n",
|
||||
"\n",
|
||||
"**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. \n",
|
||||
"**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
|
||||
"\n",
|
||||
"**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
|
||||
"\n",
|
||||
@@ -455,7 +446,7 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
|
||||
" BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + UUID"
|
||||
" BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -518,8 +509,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import google.cloud.aiplatform as aip\n",
|
||||
"from google.cloud.aiplatform.vizier import Study, pyvizier"
|
||||
"import google.cloud.aiplatform as aip"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -544,6 +534,35 @@
|
||||
"aip.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "aip_constants"
|
||||
},
|
||||
"source": [
|
||||
"#### Vertex AI constants\n",
|
||||
"\n",
|
||||
"Setup up the following constants for Vertex AI:\n",
|
||||
"\n",
|
||||
"- `API_ENDPOINT`: The Vertex AI API service endpoint for `Dataset`, `Model`, `Job`, `Pipeline` and `Endpoint` services.\n",
|
||||
"- `PARENT`: The Vertex AI location root path for `Dataset`, `Model`, `Job`, `Pipeline` and `Endpoint` resources."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "aip_constants"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# API service endpoint\n",
|
||||
"API_ENDPOINT = \"{}-aiplatform.googleapis.com\".format(REGION)\n",
|
||||
"\n",
|
||||
"# Vertex location root path for your dataset, model and endpoint resources\n",
|
||||
"PARENT = \"projects/\" + PROJECT_ID + \"/locations/\" + REGION"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
@@ -607,7 +626,7 @@
|
||||
"if os.getenv(\"IS_TESTING_TF\"):\n",
|
||||
" TF = os.getenv(\"IS_TESTING_TF\")\n",
|
||||
"else:\n",
|
||||
" TF = \"2.5\".replace(\".\", \"-\")\n",
|
||||
" TF = \"2.1\".replace(\".\", \"-\")\n",
|
||||
"\n",
|
||||
"if TF[0] == \"2\":\n",
|
||||
" if TRAIN_GPU:\n",
|
||||
@@ -1012,7 +1031,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"JOB_NAME = \"custom_job_\" + UUID\n",
|
||||
"JOB_NAME = \"custom_job_\" + TIMESTAMP\n",
|
||||
"MODEL_DIR = \"{}/{}\".format(BUCKET_URI, JOB_NAME)\n",
|
||||
"\n",
|
||||
"if not TRAIN_NGPU or TRAIN_NGPU < 2:\n",
|
||||
@@ -1075,7 +1094,9 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"job = aip.CustomJob(display_name=\"boston_\" + UUID, worker_pool_specs=worker_pool_spec)"
|
||||
"job = aip.CustomJob(\n",
|
||||
" display_name=\"boston_\" + TIMESTAMP, worker_pool_specs=worker_pool_spec\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1107,7 +1128,7 @@
|
||||
"from google.cloud.aiplatform import hyperparameter_tuning as hpt\n",
|
||||
"\n",
|
||||
"hpt_job = aip.HyperparameterTuningJob(\n",
|
||||
" display_name=\"boston_\" + UUID,\n",
|
||||
" display_name=\"boston_\" + TIMESTAMP,\n",
|
||||
" custom_job=job,\n",
|
||||
" metric_spec={\n",
|
||||
" \"val_loss\": \"minimize\",\n",
|
||||
@@ -1288,7 +1309,7 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"job = aip.CustomJob(\n",
|
||||
" display_name=\"boston_\" + UUID,\n",
|
||||
" display_name=\"boston_\" + TIMESTAMP,\n",
|
||||
" worker_pool_specs=worker_pool_spec,\n",
|
||||
" base_output_dir=MODEL_DIR,\n",
|
||||
")"
|
||||
@@ -1323,7 +1344,7 @@
|
||||
"from google.cloud.aiplatform import hyperparameter_tuning as hpt\n",
|
||||
"\n",
|
||||
"hpt_job = aip.HyperparameterTuningJob(\n",
|
||||
" display_name=\"boston_\" + UUID,\n",
|
||||
" display_name=\"boston_\" + TIMESTAMP,\n",
|
||||
" custom_job=job,\n",
|
||||
" metric_spec={\n",
|
||||
" \"val_loss\": \"minimize\",\n",
|
||||
@@ -1492,25 +1513,22 @@
|
||||
"id": "vizier_client"
|
||||
},
|
||||
"source": [
|
||||
"### Specify the algorithm used to suggest trial parameters\n",
|
||||
"### Create Vizier client\n",
|
||||
"\n",
|
||||
"First, you create a `StudyConfig`, and specify the algorithm to suggest the next trial.\n",
|
||||
"\n",
|
||||
" GRID_SEARCH: grid search\n",
|
||||
" RANDOM_SEARCH: random search\n",
|
||||
" ALGORIGTHM_UNSPECIFIED: Vizier bayesian algorithm"
|
||||
"Create a client side connection to the Vertex AI Vizier service."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "d7dd26490358"
|
||||
"id": "vizier_client"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"problem = pyvizier.StudyConfig()\n",
|
||||
"problem.algorithm = pyvizier.Algorithm.RANDOM_SEARCH"
|
||||
"vizier_client = aip.gapic.VizierServiceClient(\n",
|
||||
" client_options=dict(api_endpoint=API_ENDPOINT)\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1525,15 +1543,7 @@
|
||||
"\n",
|
||||
"In the following example, the goal is to maximize y = x^2 with x in the range of \\[-10. 10\\]. This example has only one parameter and uses an easily calculated function to help demonstrate how to use Vizier.\n",
|
||||
"\n",
|
||||
"First, you specify the metrics to minimize or maximize in the study as a list to the property `metric_information`. Then you specify the parameters to the study using the `add_XXX_params()` method for the corresponding data type:\n",
|
||||
"\n",
|
||||
" - add_bool_param\n",
|
||||
" - add_categorical_param\n",
|
||||
" - add_discrete_param\n",
|
||||
" - add_float_param\n",
|
||||
" - add_int_param\n",
|
||||
"\n",
|
||||
"You create the study using the `create_or_load()` method."
|
||||
"First, you will create the study using the `create_study()` method."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1544,19 +1554,28 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"STUDY_DISPLAY_NAME = \"xpow2\" + UUID\n",
|
||||
"STUDY_DISPLAY_NAME = \"xpow2\" + TIMESTAMP\n",
|
||||
"\n",
|
||||
"problem.metric_information.append(\n",
|
||||
" pyvizier.MetricInformation(name=\"y\", goal=pyvizier.ObjectiveMetricGoal.MAXIMIZE)\n",
|
||||
")\n",
|
||||
"param_x = {\n",
|
||||
" \"parameter_id\": \"x\",\n",
|
||||
" \"double_value_spec\": {\"min_value\": -10.0, \"max_value\": 10.0},\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"params = problem.search_space.select_root()\n",
|
||||
"params.add_float_param(\"x\", -10.0, 10.0, scale_type=pyvizier.ScaleType.LINEAR)\n",
|
||||
"metric_y = {\"metric_id\": \"y\", \"goal\": \"MAXIMIZE\"}\n",
|
||||
"\n",
|
||||
"study = Study.create_or_load(display_name=STUDY_DISPLAY_NAME, problem=problem)\n",
|
||||
"study = {\n",
|
||||
" \"display_name\": STUDY_DISPLAY_NAME,\n",
|
||||
" \"study_spec\": {\n",
|
||||
" \"algorithm\": \"RANDOM_SEARCH\",\n",
|
||||
" \"parameters\": [param_x],\n",
|
||||
" \"metrics\": [metric_y],\n",
|
||||
" },\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"study = vizier_client.create_study(parent=PARENT, study=study)\n",
|
||||
"STUDY_NAME = study.name\n",
|
||||
"print(\"STUDY_NAME: {}\".format(STUDY_NAME))"
|
||||
"\n",
|
||||
"print(STUDY_NAME)"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1567,7 +1586,9 @@
|
||||
"source": [
|
||||
"### Get Vizier study\n",
|
||||
"\n",
|
||||
"You can get a study using the method `list()`."
|
||||
"You can get a study using the method `get_study()`, with the following key/value pairs:\n",
|
||||
"\n",
|
||||
"- `name`: The name of the study."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1578,8 +1599,9 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"studies = Study.list()\n",
|
||||
"print(studies[0].gca_resource)"
|
||||
"study = vizier_client.get_study({\"name\": STUDY_NAME})\n",
|
||||
"\n",
|
||||
"print(study)"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1590,9 +1612,11 @@
|
||||
"source": [
|
||||
"### Get suggested trial\n",
|
||||
"\n",
|
||||
"Next, query the Vizier service for a suggested trial(s) using the method `suggest()`, with the following key/value pairs:\n",
|
||||
"Next, query the Vizier service for a suggested trial(s) using the method `suggest_trials`, with the following key/value pairs:\n",
|
||||
"\n",
|
||||
"- `count`: The number of trials to suggest.\n",
|
||||
"- `parent`: The name of the study.\n",
|
||||
"- `suggestion_count`: The number of trials to suggest.\n",
|
||||
"- `client_id`: blah\n",
|
||||
"\n",
|
||||
"This call is a long running operation. The method `result()` from the response object will wait until the call has completed."
|
||||
]
|
||||
@@ -1601,13 +1625,18 @@
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "11ff2c4562cb"
|
||||
"id": "vizier_suggest_trial"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"SUGGEST_COUNT = 1\n",
|
||||
"CLIENT_ID = \"1001\"\n",
|
||||
"\n",
|
||||
"trials = study.suggest(count=SUGGEST_COUNT)\n",
|
||||
"response = vizier_client.suggest_trials(\n",
|
||||
" {\"parent\": STUDY_NAME, \"suggestion_count\": SUGGEST_COUNT, \"client_id\": CLIENT_ID}\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"trials = response.result().trials\n",
|
||||
"\n",
|
||||
"print(trials)\n",
|
||||
"\n",
|
||||
@@ -1650,10 +1679,12 @@
|
||||
"source": [
|
||||
"RESULT = 0.01\n",
|
||||
"\n",
|
||||
"measurement = pyvizier.Measurement()\n",
|
||||
"measurement.metrics[\"y\"] = RESULT\n",
|
||||
"\n",
|
||||
"trials[0].add_measurement(measurement)"
|
||||
"vizier_client.add_trial_measurement(\n",
|
||||
" {\n",
|
||||
" \"trial_name\": TRIAL_ID,\n",
|
||||
" \"measurement\": {\"metrics\": [{\"metric_id\": \"y\", \"value\": RESULT}]},\n",
|
||||
" }\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1664,7 +1695,7 @@
|
||||
"source": [
|
||||
"### Delete the Vizier study\n",
|
||||
"\n",
|
||||
"The method 'delete()' will delete the study."
|
||||
"The method 'delete_study()' will delete the study."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1675,7 +1706,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"study.delete()"
|
||||
"vizier_client.delete_study({\"name\": STUDY_NAME})"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
@@ -73,7 +73,7 @@
|
||||
"source": [
|
||||
"### Objective\n",
|
||||
"\n",
|
||||
"In this tutorial, you learn how to use Python and Cloud logging when training with `Vertex AI`.\n",
|
||||
"In this tutorial, you learn how to use Python and Cloud logging awhen training with `Vertex AI`.\n",
|
||||
"\n",
|
||||
"This tutorial uses the following Google Cloud ML services:\n",
|
||||
"\n",
|
||||
|
||||
@@ -116,7 +116,7 @@
|
||||
"\n",
|
||||
"The dataset used for this tutorial is the [Patent PDF Samples with Extracted Structured Data](https://console.cloud.google.com/marketplace/product/global-patents/labeled-patents) from Google Public Data Sets. \n",
|
||||
"\n",
|
||||
"This dataset includes data extracted from over 300 patent documents issued in the US and EU. The dataset includes links to Cloud Storage blobs for the first page of each patent, in addition to a number of extracted entities. \n",
|
||||
"This dataset includes data extracted from over 300 patent documents issued in the US and EU. The dataset includes links to Google Cloud Storage blobs for the first page of each patent, in addition to a number of extracted entities. \n",
|
||||
"\n",
|
||||
"The data is published as a [public dataset](https://cloud.google.com/bigquery/public-data) on `BigQuery`."
|
||||
]
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -33,144 +33,10 @@ The third stage in MLOps is formalization to develop an automated pipeline proce
|
||||
|
||||
### Get Started
|
||||
|
||||
[Get started with AutoML Tabular Pipeline Workflows](get_started_with_automl_tabular_pipeline_workflow.ipynb)
|
||||
|
||||
[Get started with Vertex AI Model Registry](community/ml_ops/stage3/get_started_with_model_registry.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use `Vertex AI Model Registry` to create and register multiple versions of a model.
|
||||
|
||||
```
|
||||
The steps performed include:
|
||||
|
||||
- Create and register a first version of a model to `Vertex AI Model Registry`.
|
||||
- Create and register a second version of a model to `Vertex AI Model Registry`.
|
||||
- Updating the model version which is the default (blessed).
|
||||
- Deleting a model version.
|
||||
- Retraining the next model version.
|
||||
|
||||
[Get started with Dataflow pipeline components](community/ml_ops/stage3/get_started_with_dataflow_pipeline_components.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use prebuilt `Google Cloud Pipeline Components` for `Dataflow`.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Build an Apache Beam data pipeline.
|
||||
- Encapsulate the Apache Beam data pipeline with a Dataflow component in a Vertex AI pipeline.
|
||||
- Execute a Vertex AI pipeline.
|
||||
|
||||
[Get started with Apache Airflow and Vertex AI Pipelines](community/ml_ops/stage3/get_started_with_airflow_and_vertex_pipelines.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use Apache Airflow with `Vertex AI Pipelines`.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Create Cloud Composer environment.
|
||||
- Upload Airflow DAG to Composer environment that performs data processing -- i.e., creates a BigQuery table from a CSV file.
|
||||
- Create a `Vertex AI Pipeline` that triggers the Airflow DAG.
|
||||
- Execute the `Vertex AI Pipeline`.
|
||||
|
||||
[Get started with Kubeflow Pipelines](community/ml_ops/stage3/get_started_with_kubeflow_pipelines.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use `Kubeflow Pipelines`(KFP).
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Building KFP lightweight Python function components.
|
||||
- Assembling and compiling KFP components into a pipeline.
|
||||
- Executing a KFP pipeline using Vertex AI Pipelines.
|
||||
- Loading component and pipeline definitions from a source code repository.
|
||||
- Building sequential, parallel, multiple output components.
|
||||
- Building control flow into pipelines.
|
||||
|
||||
[Get started with Vertex AI custom training pipeline components](community/ml_ops/stage3/get_started_with_custom_training_pipeline_components.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use prebuilt `Google Cloud Pipeline Components` for `Vertex AI Training`.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Construct a pipeline for:
|
||||
- Training a Vertex AI custom trained model.
|
||||
- Test the serving binary with a batch prediction job.
|
||||
- Deploying a Vertex AI custom trained model.
|
||||
- Execute a Vertex AI pipeline.
|
||||
- Construct a pipeline for:
|
||||
- Construct a custom training component.
|
||||
- Convert custom training component to CustomTrainingJobOp.
|
||||
- Training a Vertex AI custom trained model using the converted component.
|
||||
- Deploying a Vertex AI custom trained model.
|
||||
- Execute a Vertex AI pipeline.
|
||||
|
||||
[Get started with Dataproc Serverless pipeline components](community/ml_ops/stage3/get_started_with_dataproc_serverless_pipeline_components.ipynb)
|
||||
|
||||
|
||||
In this tutorial, you learn how to use prebuilt `Google Cloud Pipeline Components` for `Dataproc Serverless` service.
|
||||
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- `DataprocPySparkBatchOp` for running PySpark batch workloads.
|
||||
- `DataprocSparkBatchOp` for running Spark batch workloads.
|
||||
- `DataprocSparkSqlBatchOp` for running Spark SQL batch workloads.
|
||||
- `DataprocSparkRBatchOp` for running SparkR batch workloads.
|
||||
|
||||
[Get started with Vertex AI Hyperparameter Tuning pipeline components](community/ml_ops/stage3/get_started_with_hpt_pipeline_components.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use prebuilt `Google Cloud Pipeline Components` for `Vertex AI Hyperparameter Tuning`.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Construct a pipeline for:
|
||||
- Hyperparameter tune/train a custom model.
|
||||
- Retrieve the tuned hyperparameter values and metrics to optimize.
|
||||
- If the metrics exceed a specified threshold.
|
||||
- Get the location of the model artifacts for the best tuned model.
|
||||
- Upload the model artifacts to a `Vertex AI Model` resource.
|
||||
- Execute a Vertex AI pipeline.
|
||||
|
||||
[Get started with machine management for Vertex AI Pipelines](community/ml_ops/stage3/get_started_with_machine_management.ipynb)
|
||||
|
||||
In this tutorial, you convert a self-contained custom training component into a `Vertex AI CustomJob`, whereby:
|
||||
|
||||
- The training job and artifacts are trackable.
|
||||
- Set machine resources, such as machine-type, cpu/gpu, memory, disk, etc.
|
||||
|
||||
The steps performed in this tutorial include:
|
||||
|
||||
- Create a custom component with a self-contained training job.
|
||||
- Execute pipeline using component-level settings for machine resources
|
||||
- Convert the self-contained training component into a `Vertex AI CustomJob`.
|
||||
- Execute pipeline using customjob-level settings for machine resources
|
||||
|
||||
[Get started with TFX pipelines](community/ml_ops/stage3/get_started_with_tfx_pipeline.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use TensorFlow Extended (TFX) with `Vertex AI Pipelines`.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Create a TFX e2e pipeline.
|
||||
- Execute the pipeline locally.
|
||||
- Execute the pipeline on Google Cloud using `Vertex AI Training`
|
||||
- Execute the pipeline using `Vertex AI Pipelines`.
|
||||
|
||||
[Get started with BigQuery ML pipeline components](community/ml_ops/stage3/get_started_with_bqml_pipeline_components.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use prebuilt `Google Cloud Pipeline Components` for `BigQuery ML`.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Construct a pipeline for:
|
||||
- Training BigQuery ML model.
|
||||
- Evaluating the BigQuery ML model.
|
||||
- Exporting the BigQuery ML model.
|
||||
- Importing the BigQuery ML model to a Vertex AI model.
|
||||
- Deploy the Vertex AI model.
|
||||
- Execute a Vertex AI pipeline.
|
||||
- Make a prediction with the deployed Vertex AI model.
|
||||
|
||||
[Get started with AutoML tabular pipeline workflows](community/ml_ops/stage3/get_started_with_automl_tabular_pipeline_workflow.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use `AutoML Tabular Pipeline Template` for training, exporting and tuning an AutoML tabular model.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Define training specification.
|
||||
- Dataset specification
|
||||
- Hyperparameter overide specification
|
||||
@@ -183,41 +49,162 @@ The steps performed include:
|
||||
- Deploy exported OSS TF model.
|
||||
- Make a prediction.
|
||||
|
||||
[Get started with rapid prototyping with AutoML and BigQuery ML](community/ml_ops/stage3/get_started_with_rapid_prototyping_bqml_automl.ipynb)
|
||||
```
|
||||
|
||||
In this tutorial, you learn how to use `Vertex AI Predictions` for rapid prototyping a model.
|
||||
[Get started with Vertex AI Model Registry](get_started_with_model_registry.ipynb)
|
||||
|
||||
```
|
||||
The steps performed include:
|
||||
- Create and register a first version of a model to `Vertex AI Model Registry`.
|
||||
- Create and register a second version of a model to `Vertex AI Model Registry`.
|
||||
- Updating the model version which is the default (blessed).
|
||||
- Deleting a model version.
|
||||
- Retraining the next model version.
|
||||
```
|
||||
|
||||
- Creating a BigQuery and Vertex AI training dataset.
|
||||
- Training a BigQuery ML and AutoML model.
|
||||
- Extracting evaluation metrics from the BigQueryML and AutoML models.
|
||||
- Selecting the best trained model.
|
||||
- Deploying the best trained model.
|
||||
- Testing the deployed model infrastructure.
|
||||
|
||||
[Get started with AutoML pipeline components](community/ml_ops/stage3/get_started_with_automl_pipeline_components.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use prebuilt `Google Cloud Pipeline Components` for `Vertex AI AutoML`.
|
||||
[Get started with Dataproc serverless pipeline components](get_started_with_dataproc_serverless_pipeline_components.ipynb)
|
||||
|
||||
```
|
||||
The steps performed include:
|
||||
- `DataprocPySparkBatchOp` for running PySpark batch workloads.
|
||||
- `DataprocSparkBatchOp` for running Spark batch workloads.
|
||||
- `DataprocSparkSqlBatchOp` for running Spark SQL batch workloads.
|
||||
- `DataprocSparkRBatchOp` for running SparkR batch workloads.
|
||||
|
||||
```
|
||||
|
||||
[Get started with TFX pipelines](get_started_with_tfx_pipeline.ipynb)
|
||||
|
||||
```
|
||||
The steps performed include:
|
||||
- Create a TFX e2e pipeline.
|
||||
- Execute the pipeline locally.
|
||||
- Execute the pipeline on Google Cloud using `Vertex AI Training`
|
||||
- Execute the pipeline using `Vertex AI Pipelines`.
|
||||
```
|
||||
|
||||
[Get started with Vertex AI Hyperparameter Tuning pipeline components](get_started_with_hpt_pipeline_components.ipynb)
|
||||
|
||||
```
|
||||
The steps performed include:
|
||||
- Construct a pipeline for:
|
||||
- Hyperparameter tune/train a custom model.
|
||||
- Retrieve the tuned hyperparameter values and metrics to optimize.
|
||||
- If the metrics exceed a specified threshold.
|
||||
- Get the location of the model artifacts for the best tuned model.
|
||||
- Upload the model artifacts to a `Vertex AI Model` resource.
|
||||
- Execute a Vertex AI pipeline.
|
||||
|
||||
```
|
||||
|
||||
[Get started with Apache Airflow and Vertex AI Pipelines](get_started_with_airflow_and_vertex_pipelines.ipynb)
|
||||
|
||||
```
|
||||
The steps performed include:
|
||||
- Create Cloud Composer environment.
|
||||
- Upload Airflow DAG to Composer environment that performs data processing -- i.e., creates a BigQuery table from a CSV file.
|
||||
- Create a `Vertex AI Pipeline` that triggers the Airflow DAG.
|
||||
- Execute the `Vertex AI Pipeline`.
|
||||
|
||||
```
|
||||
|
||||
[Get started with Vertex AI custom training pipeline components](get_started_with_custom_training_pipeline_components.ipynb)
|
||||
|
||||
```
|
||||
The steps performed include:
|
||||
- Construct a pipeline for:
|
||||
- Training a Vertex AI custom trained model.
|
||||
- Test the serving binary with a batch prediction job.
|
||||
- Deploying a Vertex AI custom trained model.
|
||||
- Execute a Vertex AI pipeline.
|
||||
- Construct a pipeline for:
|
||||
- Construct a custom training component.
|
||||
- Convert custom training component to CustomTrainingJobOp.
|
||||
- Training a Vertex AI custom trained model using the converted component.
|
||||
- Deploying a Vertex AI custom trained model.
|
||||
- Execute a Vertex AI pipeline.
|
||||
```
|
||||
|
||||
[Get started with AutoML pipeline components](get_started_with_automl_pipeline_components.ipynb)
|
||||
|
||||
```
|
||||
The steps performed include:
|
||||
- Construct a pipeline for:
|
||||
- Training a Vertex AI AutoML trained model.
|
||||
- Test the serving binary with a batch prediction job.
|
||||
- Deploying a Vertex AI AutoML trained model.
|
||||
- Execute a Vertex AI pipeline.
|
||||
|
||||
```
|
||||
|
||||
[Get started with BigQuery and TFDV pipeline components](community/ml_ops/stage3/get_started_with_bq_tfdv_pipeline_components.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use build lightweight Python components for BigQuery and TensorFlow Data Validation.
|
||||
[Get started with Kubeflow pipelines](get_started_with_kubeflow_pipelines.ipynb)
|
||||
|
||||
```
|
||||
The steps performed include:
|
||||
- Building KFP lightweight Python function components.
|
||||
- Assembling and compiling KFP components into a pipeline.
|
||||
- Executing a KFP pipeline using Vertex AI Pipelines.
|
||||
- Loading component and pipeline definitions from a source code repository.
|
||||
- Building sequential, parallel, multiple output components.
|
||||
- Building control flow into pipelines.
|
||||
|
||||
```
|
||||
|
||||
[Get started with machine management for Vertex AI Pipelines](get_started_with_machine_management.ipynb)
|
||||
|
||||
```
|
||||
The steps performed in this tutorial include:
|
||||
- Create a custom component with a self-contained training job.
|
||||
- Execute pipeline using component-level settings for machine resources
|
||||
- Convert the self-contained training component into a `Vertex AI CustomJob`.
|
||||
- Execute pipeline using customjob-level settings for machine resources
|
||||
|
||||
```
|
||||
|
||||
[Get started with BigQuery and TFDV pipeline components](get_started_with_bq_tfdv_pipeline_components.ipynb)
|
||||
|
||||
```
|
||||
The steps performed include:
|
||||
- Build and execute a pipeline component for creating a Vertex AI Tabular Dataset from a BigQuery table.
|
||||
- Build and execute a pipeline component for generating TFDV statistics and schema from a Vertex AI Tabular Dataset.
|
||||
- Execute a Vertex AI pipeline.
|
||||
```
|
||||
|
||||
[Get started with Dataflow pipeline components](get_started_with_dataflow_pipeline_components.ipynb)
|
||||
|
||||
```
|
||||
The steps performed include:
|
||||
- Build an Apache Beam data pipeline.
|
||||
- Encapsulate the Apache Beam data pipeline with a Dataflow component in a Vertex AI pipeline.
|
||||
- Execute a Vertex AI pipeline.
|
||||
```
|
||||
|
||||
[Get started with BigQuery ML pipeline components](get_started_with_bqml_pipeline_components.ipynb)
|
||||
|
||||
```
|
||||
The steps performed include:
|
||||
- Construct a pipeline for:
|
||||
- Training BigQuery ML model.
|
||||
- Evaluating the BigQuery ML model.
|
||||
- Exporting the BigQuery ML model.
|
||||
- Importing the BigQuery ML model to a Vertex AI model.
|
||||
- Deploy the Vertex AI model.
|
||||
- Execute a Vertex AI pipeline.
|
||||
- Make a prediction with the deployed Vertex AI model.
|
||||
```
|
||||
|
||||
[Get started with rapid prototyping with AutoML and BigQuery ML](get_started_with_rapid_prototyping_bqml_automl.ipynb)
|
||||
|
||||
```
|
||||
The steps performed include:
|
||||
- Creating a BigQuery and Vertex AI training dataset.
|
||||
- Training a BigQuery ML and AutoML model.
|
||||
- Extracting evaluation metrics from the BigQueryML and AutoML models.
|
||||
- Selecting the best trained model.
|
||||
- Deploying the best trained model.
|
||||
- Testing the deployed model infrastructure.
|
||||
```
|
||||
|
||||
|
||||
### E2E Stage Example
|
||||
|
||||
|
||||
-1391
File diff suppressed because it is too large
Load Diff
-1328
File diff suppressed because it is too large
Load Diff
@@ -568,6 +568,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import kfp\n",
|
||||
"from google.cloud import aiplatform\n",
|
||||
"from kfp import dsl\n",
|
||||
"from kfp.v2 import compiler\n",
|
||||
@@ -999,7 +1000,11 @@
|
||||
" from google.auth.transport.requests import Request\n",
|
||||
" from google.oauth2 import id_token\n",
|
||||
"\n",
|
||||
" IAM_SCOPE = \"https://www.googleapis.com/auth/iam\"\n",
|
||||
" OAUTH_TOKEN_URI = \"https://www.googleapis.com/oauth2/v4/token\"\n",
|
||||
"\n",
|
||||
" data = '{\"replace_microseconds\":\"false\"}'\n",
|
||||
" context = None\n",
|
||||
"\n",
|
||||
" \"\"\"Makes a POST request to the Composer DAG Trigger API\n",
|
||||
"\n",
|
||||
|
||||
+1
-1
@@ -44,7 +44,7 @@
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage3/get_started_with_automl_tabular_pipeline_workflow.ipynb\">\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_automl_tabular_pipeline_workflow.ipynb\">\n",
|
||||
" <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
|
||||
" Open in Vertex AI Workbench\n",
|
||||
" </a>\n",
|
||||
|
||||
+2
-2
@@ -39,9 +39,9 @@
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_dataflow_pipeline_components.ipynb\">\n",
|
||||
" <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_dataflow_pipeline_components.ipynb\">\n",
|
||||
"<img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> \n",
|
||||
" Run in Colab\n",
|
||||
" Colab logo Run in Colab\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
|
||||
@@ -81,7 +81,7 @@
|
||||
"source": [
|
||||
"### Objective\n",
|
||||
"\n",
|
||||
"In this tutorial, you learn how to convert a self-contained custom training component into a `Vertex AI CustomJob`, whereby:\n",
|
||||
"In this tutorial, you convert a self-contained custom training component into a `Vertex AI CustomJob`, whereby:\n",
|
||||
"\n",
|
||||
" - The training job and artifacts are trackable.\n",
|
||||
" - Set machine resources, such as machine-type, cpu/gpu, memory, disk, etc.\n",
|
||||
@@ -569,6 +569,7 @@
|
||||
"source": [
|
||||
"import json\n",
|
||||
"\n",
|
||||
"import numpy as np\n",
|
||||
"from google.cloud import aiplatform\n",
|
||||
"from google_cloud_pipeline_components.v1.custom_job import \\\n",
|
||||
" create_custom_training_job_from_component\n",
|
||||
@@ -796,6 +797,7 @@
|
||||
" epochs: int,\n",
|
||||
") -> str:\n",
|
||||
" import numpy as np\n",
|
||||
" import tensorflow as tf\n",
|
||||
"\n",
|
||||
" def get_data():\n",
|
||||
" from tensorflow.keras.datasets import mnist\n",
|
||||
@@ -902,7 +904,7 @@
|
||||
" },\n",
|
||||
" ).after(training_job_task)\n",
|
||||
"\n",
|
||||
" _ = ModelUploadOp(\n",
|
||||
" model_upload = ModelUploadOp(\n",
|
||||
" project=project,\n",
|
||||
" display_name=\"mnist_model\",\n",
|
||||
" unmanaged_container_model=import_unmanaged_model_task.outputs[\"artifact\"],\n",
|
||||
@@ -1178,7 +1180,7 @@
|
||||
" },\n",
|
||||
" ).after(training_job_task)\n",
|
||||
"\n",
|
||||
" _ = ModelUploadOp(\n",
|
||||
" model_upload = ModelUploadOp(\n",
|
||||
" project=project,\n",
|
||||
" display_name=\"mnist_model\",\n",
|
||||
" unmanaged_container_model=import_unmanaged_model_task.outputs[\"artifact\"],\n",
|
||||
|
||||
+5
-5
@@ -60,7 +60,7 @@
|
||||
"source": [
|
||||
"## Overview\n",
|
||||
"\n",
|
||||
"This tutorial demonstrates how to use Vertex AI Pipelines to rapid prototype a model using both AutoML and BigQuery ML, do an evaluation comparison, for a baseline, before progressing to a custom model.\n",
|
||||
"This tutorial demonstrates how to use Vertex AI Pipelines to rapid prototype a model using both AutoML and BQML, do an evaluation comparison, for a baseline, before progressing to a custom model.\n",
|
||||
"\n",
|
||||
"<img src=\"https://storage.googleapis.com/rafacarv-public-bucket-do-not-delete/abalone/automl_and_bqml.png\" />"
|
||||
]
|
||||
@@ -834,7 +834,7 @@
|
||||
"source": [
|
||||
"### Create component: Split the dataset into train, test and eval\n",
|
||||
"\n",
|
||||
"For this pipeline, you set aside a portion of the dataset for test evaluation. While both AutoML and BigQuery ML will automatically split then datasets, in this example you will explicitly split the datasets into:\n",
|
||||
"For this pipeline, you set aside a portion of the dataset for test evaluation. While both AutoML and BQML will automatically split then datasets, in this example you will explicitly split the datasets into:\n",
|
||||
"\n",
|
||||
"- TRAIN\n",
|
||||
"- EVALUATE\n",
|
||||
@@ -1000,11 +1000,11 @@
|
||||
"- Construct the CREATE MODEL query using a static Python function `_create_model_query()`, which runs in the context of the pipeline.\n",
|
||||
"- Call the prebuilt component `BigQueryCreateModelOp`, with the constructed query, to train the BigQuery ML model.\n",
|
||||
"\n",
|
||||
"For this tutorial, you use a simple linear regression model on BigQuery ML. \n",
|
||||
"For this tutorial, you use a simple linear regression model on BQML. \n",
|
||||
"\n",
|
||||
"For a full list of models supported by BigQuery ML, look here: [End-to-end user journey for each model](https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-e2e-journey).\n",
|
||||
"For a full list of models supported by BQML, look here: [End-to-end user journey for each model](https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-e2e-journey).\n",
|
||||
"\n",
|
||||
"As pointed out before, BigQuery ML and AutoML use different split terminologies, so we do an adaptation of the <i>split_col</i> column directly on the SELECT portion of the CREATE model query:\n",
|
||||
"As pointed out before, BQML and AutoML use different split terminologies, so we do an adaptation of the <i>split_col</i> column directly on the SELECT portion of the CREATE model query:\n",
|
||||
"\n",
|
||||
"> When the value of DATA_SPLIT_METHOD is 'CUSTOM', the corresponding column should be of type BOOL. The rows with TRUE or NULL values are used as evaluation data. Rows with FALSE values are used as training data."
|
||||
]
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -658,6 +658,8 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from typing import NamedTuple\n",
|
||||
"\n",
|
||||
"from kfp import dsl\n",
|
||||
"from kfp.v2 import compiler\n",
|
||||
"from kfp.v2.dsl import component"
|
||||
@@ -1864,7 +1866,7 @@
|
||||
" exported_tfrec_prefix=exported_tfrec_prefix,\n",
|
||||
" ).after(dataflow_wait_op)\n",
|
||||
"\n",
|
||||
" _ = gcc_aip.TabularDatasetCreateOp(\n",
|
||||
" dataset_op = gcc_aip.TabularDatasetCreateOp(\n",
|
||||
" project=project,\n",
|
||||
" display_name=display_name,\n",
|
||||
" bq_source=bq_table,\n",
|
||||
@@ -2283,7 +2285,7 @@
|
||||
" },\n",
|
||||
" ).after(model_build_op)\n",
|
||||
"\n",
|
||||
" _ = ModelUploadOp(\n",
|
||||
" model_upload = ModelUploadOp(\n",
|
||||
" project=project,\n",
|
||||
" display_name=display_name,\n",
|
||||
" unmanaged_container_model=import_unmanaged_model_task.outputs[\"artifact\"],\n",
|
||||
@@ -3203,7 +3205,7 @@
|
||||
"\n",
|
||||
" with dsl.Condition(warmup == \"True\", name=\"warmup-model\"):\n",
|
||||
"\n",
|
||||
" _ = gcc_aip.CustomPythonPackageTrainingJobRunOp(\n",
|
||||
" warmup_op = gcc_aip.CustomPythonPackageTrainingJobRunOp(\n",
|
||||
" project=project,\n",
|
||||
" display_name=display_name,\n",
|
||||
" # Warmup Training\n",
|
||||
@@ -3247,7 +3249,7 @@
|
||||
" display_name=display_name,\n",
|
||||
" ).after(training_op)\n",
|
||||
"\n",
|
||||
" _ = ModelDeployOp(\n",
|
||||
" deploy_op = ModelDeployOp(\n",
|
||||
" model=training_op.outputs[\"model\"],\n",
|
||||
" endpoint=endpoint_op.outputs[\"endpoint\"],\n",
|
||||
" dedicated_resources_min_replica_count=1,\n",
|
||||
|
||||
@@ -42,192 +42,104 @@ This stage may be done entirely by MLOps. We recommend:
|
||||
|
||||
### Get Started
|
||||
|
||||
[Get started with Vertex Explainable AI](get_started_with_vertex_xai.ipynb)
|
||||
|
||||
[Get started with Vertex AI Model Registry](community/ml_ops/stage3/get_started_with_model_registry.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use `Vertex AI Model Registry` to create and register multiple versions of a model.
|
||||
|
||||
```
|
||||
The steps performed include:
|
||||
- Train an AutoML tabular model.
|
||||
- Do a batch prediction with explanations.
|
||||
- Do an online prediction with explanations.
|
||||
- Train an custom TensorFlow tabular model.
|
||||
- Manually set configuration metadata.
|
||||
- Do a batch prediction with explanations.
|
||||
- Do an online prediction with explanations.
|
||||
- Automatically set configuration metadata.
|
||||
- Train an custom TensorFlow image model.
|
||||
- Manually set configuration metadata.
|
||||
- Do a batch prediction with explanations.
|
||||
- Do an online prediction with explanations.
|
||||
- Train an custom XGBoost tabular model.
|
||||
- Manually set configuration metadata.
|
||||
- Do an online prediction with explanations.
|
||||
- Train an custom scikit-learn tabular model.
|
||||
- Manually set configuration metadata.
|
||||
- Do an online prediction with explanations.
|
||||
|
||||
- Create and register a first version of a model to `Vertex AI Model Registry`.
|
||||
- Create and register a second version of a model to `Vertex AI Model Registry`.
|
||||
- Updating the model version which is the default (blessed).
|
||||
- Deleting a model version.
|
||||
- Retraining the next model version.
|
||||
```
|
||||
|
||||
[Get started with Dataflow pipeline components](community/ml_ops/stage3/get_started_with_dataflow_pipeline_components.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use prebuilt `Google Cloud Pipeline Components` for `Dataflow`.
|
||||
[Get started with Google Artifact Registry](get_started_with_google_artifact_registry.ipynb)
|
||||
|
||||
```
|
||||
The steps performed include:
|
||||
- Creating a private Docker repository.
|
||||
- Tagging a container image, specific to the private Docker repository.
|
||||
- Pushing a container image to the private Docker repository.
|
||||
- Pulling a container image from the private Docker repository.
|
||||
- Deleting a private Docker repository.
|
||||
```
|
||||
|
||||
- Build an Apache Beam data pipeline.
|
||||
- Encapsulate the Apache Beam data pipeline with a Dataflow component in a Vertex AI pipeline.
|
||||
- Execute a Vertex AI pipeline.
|
||||
|
||||
[Get started with Apache Airflow and Vertex AI Pipelines](community/ml_ops/stage3/get_started_with_airflow_and_vertex_pipelines.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use Apache Airflow with `Vertex AI Pipelines`.
|
||||
[Get started with AutoML training and ML Metadata](get_started_with_vertex_ml_metadata_and_automl.ipynb)
|
||||
|
||||
```
|
||||
The steps performed include:
|
||||
- Create a `Dataset` resource.
|
||||
- Create a corresponding `google.VertexDataset` artifact.
|
||||
- Train a model using `AutoML`.
|
||||
- Create a corresponding `google.VertexModel` artifact.
|
||||
- Create an `Endpoint` resource.
|
||||
- Create a corresponding `google.Endpoint` artifact.
|
||||
- Deploy the train model to the `Endpoint`.
|
||||
- Create an execution and context for the `AutoML` training job and deployment.
|
||||
- Add the corresponding artifacts and context to the execution.
|
||||
- Add artifact links (event) to the execution.
|
||||
- Display the execution graph.
|
||||
```
|
||||
|
||||
- Create Cloud Composer environment.
|
||||
- Upload Airflow DAG to Composer environment that performs data processing -- i.e., creates a BigQuery table from a CSV file.
|
||||
- Create a `Vertex AI Pipeline` that triggers the Airflow DAG.
|
||||
- Execute the `Vertex AI Pipeline`.
|
||||
|
||||
[Get started with Kubeflow Pipelines](community/ml_ops/stage3/get_started_with_kubeflow_pipelines.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use `Kubeflow Pipelines`(KFP).
|
||||
[Get started with Vertex AI ML Metadata](get_started_with_vertex_ml_metadata.ipynb)
|
||||
|
||||
```
|
||||
The steps performed include:
|
||||
- Create a `Metadatastore` resource.
|
||||
- Create (record)/List an `Artifact`, with artifacts and metadata.
|
||||
- Create (record)/List an `Execution`.
|
||||
- Create (record)/List a `Context`.
|
||||
- Add `Artifact` to `Execution` as events.
|
||||
- Add `Execution` and `Artifact` into the `Context`
|
||||
- Delete `Artifact`, `Execution` and `Context`.
|
||||
- Create and run a `Vertex AI Pipeline` ML workflow to train and deploy a scikit-learn model.
|
||||
- Create custom pipeline components that generate artifacts and metadata.
|
||||
- Compare Vertex AI Pipelines runs.
|
||||
- Trace the lineage for pipeline-generated artifacts.
|
||||
- Query your pipeline run metadata.
|
||||
```
|
||||
|
||||
- Building KFP lightweight Python function components.
|
||||
- Assembling and compiling KFP components into a pipeline.
|
||||
- Executing a KFP pipeline using Vertex AI Pipelines.
|
||||
- Loading component and pipeline definitions from a source code repository.
|
||||
- Building sequential, parallel, multiple output components.
|
||||
- Building control flow into pipelines.
|
||||
|
||||
[Get started with Vertex AI custom training pipeline components](community/ml_ops/stage3/get_started_with_custom_training_pipeline_components.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use prebuilt `Google Cloud Pipeline Components` for `Vertex AI Training`.
|
||||
[Get started with Vertex AI Model Evaluation](get_started_with_model_evaluation.ipynb)
|
||||
|
||||
```
|
||||
The steps performed include:
|
||||
|
||||
- Construct a pipeline for:
|
||||
- Training a Vertex AI custom trained model.
|
||||
- Test the serving binary with a batch prediction job.
|
||||
- Deploying a Vertex AI custom trained model.
|
||||
- Execute a Vertex AI pipeline.
|
||||
- Construct a pipeline for:
|
||||
- Construct a custom training component.
|
||||
- Convert custom training component to CustomTrainingJobOp.
|
||||
- Training a Vertex AI custom trained model using the converted component.
|
||||
- Deploying a Vertex AI custom trained model.
|
||||
- Execute a Vertex AI pipeline.
|
||||
|
||||
[Get started with Dataproc Serverless pipeline components](community/ml_ops/stage3/get_started_with_dataproc_serverless_pipeline_components.ipynb)
|
||||
|
||||
|
||||
In this tutorial, you learn how to use prebuilt `Google Cloud Pipeline Components` for `Dataproc Serverless` service.
|
||||
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- `DataprocPySparkBatchOp` for running PySpark batch workloads.
|
||||
- `DataprocSparkBatchOp` for running Spark batch workloads.
|
||||
- `DataprocSparkSqlBatchOp` for running Spark SQL batch workloads.
|
||||
- `DataprocSparkRBatchOp` for running SparkR batch workloads.
|
||||
|
||||
[Get started with Vertex AI Hyperparameter Tuning pipeline components](community/ml_ops/stage3/get_started_with_hpt_pipeline_components.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use prebuilt `Google Cloud Pipeline Components` for `Vertex AI Hyperparameter Tuning`.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Construct a pipeline for:
|
||||
- Hyperparameter tune/train a custom model.
|
||||
- Retrieve the tuned hyperparameter values and metrics to optimize.
|
||||
- If the metrics exceed a specified threshold.
|
||||
- Get the location of the model artifacts for the best tuned model.
|
||||
- Upload the model artifacts to a `Vertex AI Model` resource.
|
||||
- Execute a Vertex AI pipeline.
|
||||
|
||||
[Get started with machine management for Vertex AI Pipelines](community/ml_ops/stage3/get_started_with_machine_management.ipynb)
|
||||
|
||||
In this tutorial, you convert a self-contained custom training component into a `Vertex AI CustomJob`, whereby:
|
||||
|
||||
- The training job and artifacts are trackable.
|
||||
- Set machine resources, such as machine-type, cpu/gpu, memory, disk, etc.
|
||||
|
||||
The steps performed in this tutorial include:
|
||||
|
||||
- Create a custom component with a self-contained training job.
|
||||
- Execute pipeline using component-level settings for machine resources
|
||||
- Convert the self-contained training component into a `Vertex AI CustomJob`.
|
||||
- Execute pipeline using customjob-level settings for machine resources
|
||||
|
||||
[Get started with TFX pipelines](community/ml_ops/stage3/get_started_with_tfx_pipeline.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use TensorFlow Extended (TFX) with `Vertex AI Pipelines`.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Create a TFX e2e pipeline.
|
||||
- Execute the pipeline locally.
|
||||
- Execute the pipeline on Google Cloud using `Vertex AI Training`
|
||||
- Execute the pipeline using `Vertex AI Pipelines`.
|
||||
|
||||
[Get started with BigQuery ML pipeline components](community/ml_ops/stage3/get_started_with_bqml_pipeline_components.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use prebuilt `Google Cloud Pipeline Components` for `BigQuery ML`.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Construct a pipeline for:
|
||||
- Training BigQuery ML model.
|
||||
- Evaluating the BigQuery ML model.
|
||||
- Exporting the BigQuery ML model.
|
||||
- Importing the BigQuery ML model to a Vertex AI model.
|
||||
- Deploy the Vertex AI model.
|
||||
- Execute a Vertex AI pipeline.
|
||||
- Make a prediction with the deployed Vertex AI model.
|
||||
|
||||
[Get started with AutoML tabular pipeline workflows](community/ml_ops/stage3/get_started_with_automl_tabular_pipeline_workflow.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use `AutoML Tabular Pipeline Template` for training, exporting and tuning an AutoML tabular model.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Define training specification.
|
||||
- Dataset specification
|
||||
- Hyperparameter overide specification
|
||||
- machine specifications
|
||||
- Construct tabular workflow pipeline.
|
||||
- Compile and execute pipeline.
|
||||
- View evaluation metrics artifact.
|
||||
- Export AutoML model as an OSS TF model.
|
||||
- Create `Endpoint` resource.
|
||||
- Deploy exported OSS TF model.
|
||||
- Make a prediction.
|
||||
|
||||
[Get started with rapid prototyping with AutoML and BigQuery ML](community/ml_ops/stage3/get_started_with_rapid_prototyping_bqml_automl.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use `Vertex AI Predictions` for rapid prototyping a model.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Creating a BigQuery and Vertex AI training dataset.
|
||||
- Training a BigQuery ML and AutoML model.
|
||||
- Extracting evaluation metrics from the BigQueryML and AutoML models.
|
||||
- Selecting the best trained model.
|
||||
- Deploying the best trained model.
|
||||
- Testing the deployed model infrastructure.
|
||||
|
||||
[Get started with AutoML pipeline components](community/ml_ops/stage3/get_started_with_automl_pipeline_components.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use prebuilt `Google Cloud Pipeline Components` for `Vertex AI AutoML`.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Construct a pipeline for:
|
||||
- Training a Vertex AI AutoML trained model.
|
||||
- Test the serving binary with a batch prediction job.
|
||||
- Deploying a Vertex AI AutoML trained model.
|
||||
- Execute a Vertex AI pipeline.
|
||||
|
||||
|
||||
[Get started with BigQuery and TFDV pipeline components](community/ml_ops/stage3/get_started_with_bq_tfdv_pipeline_components.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use build lightweight Python components for BigQuery and TensorFlow Data Validation.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Build and execute a pipeline component for creating a Vertex AI Tabular Dataset from a BigQuery table.
|
||||
- Build and execute a pipeline component for generating TFDV statistics and schema from a Vertex AI Tabular Dataset.
|
||||
- Execute a Vertex AI pipeline.
|
||||
|
||||
- Evaluate an `AutoML` model.
|
||||
- Train an `AutoML` image classification model.
|
||||
- Retrieve the default evaluation metrics from training.
|
||||
- Do a batch evaluation for a custom evaluation slice.
|
||||
- Evaluate a BigQuery ML model.
|
||||
- Train a `BigQuery ML` tabular classification model.
|
||||
- Retrieve the default evaluation metrics from training.
|
||||
- Do a batch evaluation for a custom evaluation slice.
|
||||
- Evaluate a custom model.
|
||||
- Do a batch evaluation for a custom evaluation slice.
|
||||
- Add an evaluation to the `Model Registry` for the `Model` resource.
|
||||
- Evaluate an `AutoML` model.
|
||||
- Train an `AutoML` image classification model.
|
||||
- Retrieve the default evaluation metrics from training.
|
||||
- Do a batch evaluation for a custom evaluation slice.
|
||||
- Evaluate a BigQuery ML model.
|
||||
- Train a `BigQuery ML` tabular classification model.
|
||||
- Retrieve the default evaluation metrics from training.
|
||||
- Do a batch evaluation for a custom evaluation slice.
|
||||
- Evaluate a custom model.
|
||||
- Do a batch evaluation for a custom evaluation slice.
|
||||
- Add an evaluation to the `Model Registry` for the `Model` resource.
|
||||
```
|
||||
### E2E Stage Example
|
||||
|
||||
Stage 4: Evaluation
|
||||
|
||||
@@ -29,7 +29,7 @@
|
||||
"id": "title:generic,gcp"
|
||||
},
|
||||
"source": [
|
||||
"# E2E ML on GCP: MLOps stage 4 : formalization: get started with Vertex ML Metadata\n",
|
||||
"# E2E ML on GCP: MLOps stage 4 : formalization: get started with Vertex AI ML Metadata\n",
|
||||
"\n",
|
||||
"<table align=\"left\">\n",
|
||||
" <td>\n",
|
||||
@@ -62,7 +62,7 @@
|
||||
"## Overview\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 4 : formalization: get started with Vertex ML Metadata."
|
||||
"This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 4 : formalization: get started with Vertex AI ML Metadata."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -73,11 +73,11 @@
|
||||
"source": [
|
||||
"### Objective\n",
|
||||
"\n",
|
||||
"In this tutorial, you learn how to use `Vertex ML Metadata`.\n",
|
||||
"In this tutorial, you learn how to use `Vertex AI ML Metadata`.\n",
|
||||
"\n",
|
||||
"This tutorial uses the following Google Cloud ML services:\n",
|
||||
"\n",
|
||||
"- `Vertex ML Metadata`\n",
|
||||
"- `Vertex AI ML Metadata`\n",
|
||||
"- `Vertex AI Pipelines`\n",
|
||||
"\n",
|
||||
"The steps performed include:\n",
|
||||
@@ -657,7 +657,7 @@
|
||||
"source": [
|
||||
"## Introduction to Vertex AI Metadata\n",
|
||||
"\n",
|
||||
"The `Vertex ML Metadata` service provides you with the ability to record, and subsequently search and analyze, the artifacts and corresponding metadata produced by your ML workflows. For example, during experimentation one might desire to record the location of the model artifacts, as artifacts, and the training hyperparameters and evaluation metrics as the corresponding metadata.\n",
|
||||
"The `Vertex AI ML Metadata` service provides you with the ability to record, and subsequently search and analyze, the artifacts and corresponding metadata produced by your ML workflows. For example, during experimentation one might desire to record the location of the model artifacts, as artifacts, and the training hyperparameters and evaluation metrics as the corresponding metadata.\n",
|
||||
"\n",
|
||||
"The service supports recording ML metadata both manually and automatically, with the later occurring when you use Vertex AI Pipelines.\n",
|
||||
"\n",
|
||||
@@ -675,9 +675,9 @@
|
||||
"\n",
|
||||
"### ML artifact lineage\n",
|
||||
"\n",
|
||||
"Vertex ML Metadata provides the ability to understand changes in the performance of your machine ML system, and analyze the metadata produced by your ML workflow and the lineage of its artifacts. An artifact's lineage includes all the factors that contributed to its creation, as well as artifacts and metadata that descend from this artifact.\n",
|
||||
"Vertex AI ML Metadata provides the ability to understand changes in the performance of your machine ML system, and analyze the metadata produced by your ML workflow and the lineage of its artifacts. An artifact's lineage includes all the factors that contributed to its creation, as well as artifacts and metadata that descend from this artifact.\n",
|
||||
"\n",
|
||||
"Learn more about [Introduction to Vertex ML Metadata ](https://cloud.google.com/vertex-ai/docs/ml-metadata/introduction)"
|
||||
"Learn more about [Introduction to Vertex AI ML Metadata ](https://cloud.google.com/vertex-ai/docs/ml-metadata/introduction)"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
@@ -25,12 +25,10 @@ The fifth stage in MLOps is deployment to production of the blessed model, which
|
||||
### Get Started
|
||||
|
||||
|
||||
[Get started with Vertex AI Endpoints](community/ml_ops/stage5/get_started_with_vertex_endpoints.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use `Vertex AI Endpoint` resources.
|
||||
[Get started with Vertex AI Endpoints](get_started_with_vertex_endpoints.ipynb)
|
||||
|
||||
```
|
||||
The steps performed include:
|
||||
|
||||
- Creating an `Endpoint` resource.
|
||||
- List all `Endpoint` resources.
|
||||
- List `Endpoint` resources by query filter.
|
||||
@@ -45,29 +43,12 @@ The steps performed include:
|
||||
- Delete an `Endpoint` resource.
|
||||
- In pipeline: Create an `Endpoint` resource and deploy an existing `Model` resource to the `Endpoint` resource.
|
||||
- In pipeline: Deploy an existing `Model` resource to an existing `Endpoint` resource.
|
||||
```
|
||||
|
||||
[Get started with Vertex AI Endpoint and shared VM](community/ml_ops/stage5/get_started_with_vertex_endpoint_and_shared_vm.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use deployment resource pools for deploying models. A deployment resouce pool provides one with the ability to co-host more than one model on the same (shared) VM.
|
||||
[Get started with configuring autoscaling for Vertex AI Endpoint deployment](get_started_with_autoscaling.ipynb)
|
||||
|
||||
```
|
||||
The steps performed include:
|
||||
|
||||
- Upload a pre-trained image classification model as a `Model` resource (model A).
|
||||
- Upload a pre-trained text sentence encoder model as a `Model` resource (model B).
|
||||
- Create a shared VM deployment resource pool.
|
||||
- List shared VM deployment resource pools.
|
||||
- Create two `Endpoint` resources.
|
||||
- Deploy first model (model A) to first `Endpoint` resource using deployment resource pool.
|
||||
- Deploy second model (model B) to second `Endpoint` resource using deployment resource pool.
|
||||
- Make a prediction request with first deployed model (model A).
|
||||
- Make a prediction request with second deployed model (model B).
|
||||
|
||||
[Get started with configuring autoscaling for Vertex AI Endpoint deployment](community/ml_ops/stage5/get_started_with_autoscaling.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use fine-tune control auto-scaling configuration when deploying a `Model` resource to an `Endpoint` resource.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Download a pretrained image classification model from TensorFlow Hub.
|
||||
- Upload the pretrained model as a `Model` resource.
|
||||
- Create an `Endpoint` resource.
|
||||
@@ -77,20 +58,31 @@ The steps performed include:
|
||||
- Fine-tune scaling thresholds for CPU utilization.
|
||||
- Fine-tune scaling thresholds for GPU utilization.
|
||||
- Deploy mix of CPU and GPU model instances with auto-scaling to an `Endpoint` resource.
|
||||
```
|
||||
|
||||
[Get started with Vertex AI Private Endpoints](community/ml_ops/stage5/get_started_with_vertex_private_endpoints.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use `Vertex AI Private Endpoint` resources.
|
||||
[Get started with Vertex AI Private Endpoints](get_started_with_vertex_private_endpoints.ipynb)
|
||||
|
||||
```
|
||||
The steps performed include:
|
||||
|
||||
- Creating a `Private Endpoint` resource.
|
||||
- Configure a VPC peering connection.
|
||||
- Configuring the serving binary of a `Model` resource for deployment to a `Private Endpoint` resource.
|
||||
- Deploying a `Model` resource to a `Private Endpoint` resource.
|
||||
- Send a prediction request to a `Private Endpoint`
|
||||
- Enable two additional APIs: Service Networking and Cloud DNS.
|
||||
- Add Compute Admin Network role to your (default) service account.
|
||||
- Issue two gcloud commands to setup the VPC peering for your service account.
|
||||
- There is *currently* no SDK support yet, so private endpoint is created with GAPIC client and has an extra argument for the peering network.
|
||||
- To send a request, you can't use SDK/GAPIC since they do a HTTP internet request. Instead, you use curl to send a peer-to-peer request.
|
||||
|
||||
```
|
||||
|
||||
[Get started with Vertex AI Endpoint and shared VM](get_started_with_vertex_endpoint_and_shared_vm.ipynb)
|
||||
|
||||
```
|
||||
The steps performed include:
|
||||
- Upload a pre-trained image classification model as a `Model` resource (model A).
|
||||
- Upload a pre-trained text sentence encoder model as a `Model` resource (model B).
|
||||
- Create a shared VM deployment resource pool.
|
||||
- List shared VM deployment resource pools.
|
||||
- Create two `Endpoint` resources.
|
||||
- Deploy first model (model A) to first `Endpoint` resource using deployment resource pool.
|
||||
- Deploy second model (model B) to second `Endpoint` resource using deployment resource pool.
|
||||
- Make a prediction request with first deployed model (model A).
|
||||
- Make a prediction request with second deployed model (model B).
|
||||
```
|
||||
|
||||
@@ -30,232 +30,48 @@ This stage may be done entirely by MLOps. We recommend:
|
||||
### Get Started
|
||||
|
||||
|
||||
[Get started with Vertex AI Batch Prediction for AutoML image models](community/ml_ops/stage6/get_started_with_automl_image_model_batch.ipynb)
|
||||
|
||||
In this tutorial, you create an AutoML image classification model from a Python script, and then do a batch prediction using the Vertex AI SDK.
|
||||
[Get started with TensorFlow serving functions with Vertex AI Prediction](get_started_with_tf_serving_function.ipynb)
|
||||
|
||||
```
|
||||
The steps performed include:
|
||||
|
||||
- Create a Vertex `Dataset` resource.
|
||||
- Train an `AutoML` image classification model.
|
||||
- Make a batch prediction with JSONL input.
|
||||
|
||||
[Get started with Vertex AI Matching Engine and Swivel builtin algorithm](community/ml_ops/stage6/get_started_with_matching_engine_swivel.ipynb)
|
||||
|
||||
In this notebook, you learn how to train custom embeddings using Vertex AI Pipelines and subsequently train and deploy a matching engine index using the embeddings.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
1. Train the `Swivel` algorithm to generate embeddings (encoder) for the dataset.
|
||||
2. Make example predictions (embeddings) from then trained encoder.
|
||||
3. Generate embeddings using the trained `Swivel` builtin algorithm.
|
||||
4. Store embeddings to format supported by `Matching Engine`.
|
||||
5. Create a `Matching Engine Index` for the embeddings.
|
||||
6. Deploy the `Matching Engine Index` to a `Index Endpoint`.
|
||||
7. Make a matching engine prediction request.
|
||||
|
||||
[Get started with Vertex AI Matching Engine](community/ml_ops/stage6/get_started_with_matching_engine.ipynb)
|
||||
|
||||
In this notebook, you learn how to create Approximate Nearest Neighbor (ANN) Index, query against indexes.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Create ANN Index.
|
||||
- Create an IndexEndpoint with VPC Network
|
||||
- Deploy ANN Index
|
||||
- Perform online query
|
||||
- Deploy brute force Index.
|
||||
- Perform calibration between ANN and brute force index.
|
||||
|
||||
[Get started with Vertex AI Matching Engine and Two Towers builtin algorithm](community/ml_ops/stage6/get_started_with_matching_engine_twotowers.ipynb)
|
||||
|
||||
|
||||
In this notebook, you learn how to use the `Two-Tower` builtin algorithms for generating embeddings for a dataset, for use with generating an `Matching Engine Index`, with the `Vertex AI Matching Engine` service.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
1. Train the `Two-Tower` algorithm to generate embeddings (encoder) for the dataset.
|
||||
2. Hyperparameter tune the trained `Two-Tower` encoder.
|
||||
3. Make example predictions (embeddings) from then trained encoder.
|
||||
4. Generate embeddings using the trained `Two-Tower` builtin algorithm.
|
||||
5. Store embeddings to format supported by `Matching Engine`.
|
||||
6. Create a `Matching Engine Index` for the embeddings.
|
||||
7. Deploy the `Matching Engine Index` to a `Index Endpoint`.
|
||||
8. Make a matching engine prediction request.
|
||||
|
||||
[Get started with Vertex AI Batch Prediction for custom tabular models](community/ml_ops/stage6/get_started_with_custom_tabular_model_batch.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use `Vertex AI Batch Prediction` with a custom tabular model.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Upload a pretrained tabular model as a `Vertex AI Model` resource.
|
||||
- Make batch prediction to the `Model` resource, in JSONL format.
|
||||
- Make batch prediction to the `Model` resource, in CSV format.
|
||||
- Make batch prediction to the `Model` resource, in BigQuery format.
|
||||
|
||||
[Get started with Optimized TensorFlow Enterprise container with Vertex AI Prediction / text models](community/ml_ops/stage6/get_started_with_optimized_tfe_bert.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use `TensorFlow Enterprise Optimized` container for TensorFlow models deployed to a `Vertex AI Endpoint` resource.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Download a pretrained BERT model from TensorFlow Hub.
|
||||
- Fine-tune (transfer learning) the BERT model as a binary classifier.
|
||||
- Upload the TensorFlow Hub model as a `Vertex AI Model` resource, with standard TensorFlow serving container.
|
||||
- Upload the TensorFlow Hub model as a `Vertex AI Model` resource, with TensorFlow Enterprise Optimized container
|
||||
- Create two `Endpoint` resources.
|
||||
- Deploying both `Model` resources to separate `Endpoint` resources.
|
||||
- Make the same online prediction requests to both `Model` resource instances deployed to the `Endpoint` resources.
|
||||
- Compare the prediction accuracy between the two deployed `Model` resources.
|
||||
- Configuring container settings for fine-tune control of optimizations.
|
||||
- Create a `Private Endpoint` resource.
|
||||
- Deploy the `Model` resoure with then `TensorFlow Enterprise Optimized` to the `Private Endpoint` resource.
|
||||
- Make an online prediction request to the `Private Endpoint` resource.
|
||||
|
||||
[Get started with Vertex AI Batch Prediction and Explainable AI for AutoML tabular models](community/ml_ops/stage6/get_started_with_automl_tabular_model_batch.ipynb)
|
||||
|
||||
In this tutorial, you create an AutoML tabular binary classification model from a Python script, and then do a batch prediction with Explainable AI using the Vertex AI SDK. You can alternatively create and deploy models using the `gcloud` command-line tool or online using the Cloud Console.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Create a Vertex `Dataset` resource.
|
||||
- Train an `AutoML` tabular model.
|
||||
- Make a batch prediction with CSV input.
|
||||
- Make a batch prediction with JSONL objects input.
|
||||
- Make a batch prediction with JSONL list input.
|
||||
- Make a batch prediction with BigQuery table input.
|
||||
- Make a batch prediction with explanations.
|
||||
|
||||
[Get started with re-importing AutoML tabular models](community/ml_ops/stage6/get_started_with_automl_tabular_exported_deploy.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use `AutoML Tabular` for re-importing exported model artifacts as a `Model` resource. This is useful for example, if one wants to move the exported model across projects.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Importing a pretrained AutoML tabular exported model artifacts, as a `Model` resource.
|
||||
- Create an `Endpoint` resource.
|
||||
- Deploy the `Model` resource to the `Endpoint` resource.
|
||||
- Make a prediction.
|
||||
|
||||
[Get started with Vertex AI Batch Prediction for AutoML text models](community/ml_ops/stage6/get_started_with_automl_text_model_batch.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use `Vertex AI Batch Prediction` with a `AutoML` text model.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Create a Vertex `Dataset` resource.
|
||||
- Train an `AutoML` model.
|
||||
- Make a batch prediction with JSONL input
|
||||
|
||||
[Get started with Vertex AI Prediction for AutoML text models](community/ml_ops/stage6/get_started_with_automl_text_model_online.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use `Vertex AI Prediction` with a `AutoML` text model.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Create a Vertex `Dataset` resource.
|
||||
- Train an `AutoML` model.
|
||||
- Deploy the model to an `Endpoint` resource.
|
||||
- Make an online prediction.
|
||||
|
||||
[Get started with TensorFlow serving functions with Vertex AI Raw Prediction](community/ml_ops/stage6/get_started_with_raw_predict.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use `Vertex AI Raw Prediction` on a `Vertex AI Endpoint` resource.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Download a pretrained tabular classification model artifacts for a TensorFlow 1.x estimator.
|
||||
- Upload the TensorFlow estimator model as a `Vertex AI Model` resource.
|
||||
- Creating an `Endpoint` resource.
|
||||
- Deploying the `Model` resource to an `Endpoint` resource.
|
||||
- Make an online raw prediction to the `Model` resource instance deployed to the `Endpoint` resource.
|
||||
|
||||
[Get started with TensorFlow serving functions with Vertex AI Prediction](community/ml_ops/stage6/get_started_with_tf_serving_function.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use `Vertex AI Prediction` on a `Vertex AI Endpoint` resource with a serving function.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Download a pretrained image classification model from TensorFlow Hub.
|
||||
- Create a serving function to receive compressed image data, and output decomopressed preprocessed data for the model input.
|
||||
- Upload the TensorFlow Hub model and serving function as a `Vertex AI Model` resource.
|
||||
- Creating an `Endpoint` resource.
|
||||
- Deploying the `Model` resource to an `Endpoint` resource.
|
||||
- Make an online prediction to the `Model` resource instance deployed to the `Endpoint` resource.
|
||||
```
|
||||
|
||||
[Get started with Vertex Explainable AI using custom deployment container](community/ml_ops/stage6/get_started_with_xai_and_custom_server.ipynb)
|
||||
|
||||
In this tutorial, you learn to build a custom container to serve a PyTorch model on `Vertex AI Endpoint`.
|
||||
[Get started with FastAPI with Vertex AI Prediction](get_started_with_fastapi.ipynb)
|
||||
|
||||
```
|
||||
The steps performed include:
|
||||
|
||||
- Locally train a Pytorch tabular classifier.
|
||||
- Locally test the trained model.
|
||||
- Build a HTTP server using FastAPI.
|
||||
- Create a custom serving container with the trained model and FastAPI server.
|
||||
- Locally test the custom serving container.
|
||||
- Push the custom serving container to the Artifact Registry.
|
||||
- Upload the custom serving container as a `Model` resource.
|
||||
- Deploy the `Model` resource to an `Endpoint` resource.
|
||||
- Make a prediction request to the deployed custom serving container.
|
||||
- Make an explanation request to the deployed custom serving container.
|
||||
|
||||
[Get started with Vertex AI Online Prediction for AutoML image models](community/ml_ops/stage6/get_started_with_automl_image_model_online.ipynb)
|
||||
|
||||
In this tutorial, you create an AutoML image classification model from a Python script, and then do an online prediction using the Vertex AI SDK.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Create a Vertex `Dataset` resource.
|
||||
- Train an `AutoML` image classification model.
|
||||
- Make an online prediction.
|
||||
|
||||
[Get started with FastAPI with Vertex AI Prediction](community/ml_ops/stage6/get_started_with_fastapi.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use `Vertex AI Prediction` on a `Vertex AI Endpoint` with a custom serving binary using `FastAPI`.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Download a pretrained image classification model from TensorFlow Hub.
|
||||
- Create a serving function to receive compressed image data, and output decomopressed preprocessed data for the model input.
|
||||
- Upload the TensorFlow Hub model and serving function as a `Vertex AI Model` resource.
|
||||
- Creating an `Endpoint` resource.
|
||||
- Deploying the `Model` resource to an `Endpoint` resource with `FastAPI` custom serving binary.
|
||||
- Make an online prediction to the `Model` resource instance deployed to the `Endpoint` resource.
|
||||
```
|
||||
|
||||
[Get started with Vertex AI Online Prediction for AutoML tabular models](community/ml_ops/stage6/get_started_with_automl_tabular_model_online.ipynb)
|
||||
[Get started with Nvidia Triton server](get_started_with_nvidia_triton_serving.ipynb)
|
||||
|
||||
In this tutorial, you create an AutoML tabular binary classification model from a Python script, and then do an online prediction using the Vertex AI SDK.
|
||||
```
|
||||
The steps performed in this tutorial include:
|
||||
- Download the model artifacts from TensorFlow Hub.
|
||||
- Create Triton serving configuration file for the model.
|
||||
- Construct a custom container, with Triton serving image, for model deployment.
|
||||
- Upload the model as a `Vertex AI Model` resource.
|
||||
- Deploy the `Vertex AI Model` resource to a `Vertex AI Endpoint` resource.
|
||||
- Make a prediction request
|
||||
- Undeploy the `Model` resource and delete the `Endpoint`
|
||||
|
||||
```
|
||||
|
||||
[Get started with Custom Prediction Routine (CPR)](get_started_with_cpr.ipynb)
|
||||
|
||||
```
|
||||
The steps performed include:
|
||||
|
||||
- Create a Vertex `Dataset` resource.
|
||||
- Train an `AutoML` tabular model.
|
||||
- Deploy the model to an `Endpoint` resource.
|
||||
- Make an online prediction.
|
||||
- Make an online prediction with explanations.
|
||||
|
||||
[Get started with TensorFlow Serving with Vertex AI Prediction](community/ml_ops/stage6/get_started_with_tf_serving.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use `Vertex AI Prediction` on a `Vertex AI Endpoint` resource with `TensorFlow Serving` serving binary.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Download a pretrained image classification model from TensorFlow Hub.
|
||||
- Create a serving function to receive compressed image data, and output decomopressed preprocessed data for the model input.
|
||||
- Upload the TensorFlow Hub model and serving function as a `Vertex AI Model` resource.
|
||||
- Creating an `Endpoint` resource.
|
||||
- Deploying the `Model` resource to an `Endpoint` resource with `TensorFlow Serving` serving binary.
|
||||
- Make an online prediction to the `Model` resource instance deployed to the `Endpoint` resource.
|
||||
- Make a batch prediction to the `Model` resource instance.
|
||||
|
||||
[Get started with Custom Prediction Routine (CPR)](community/ml_ops/stage6/get_started_with_cpr.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use Custom Prediction Routine (CPR) for `Vertex AI Predictions`.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Write a custom data preprocessor.
|
||||
- Train the model.
|
||||
- Build a custom scikit-learn serving container with custom data preprocessing using the Custom Prediction Routine model server.
|
||||
@@ -277,50 +93,115 @@ The steps performed include:
|
||||
- Test the model serving container locally.
|
||||
- Upload and deploy the model serving container to Vertex AI Endpoint.
|
||||
- Make a prediction request.
|
||||
```
|
||||
|
||||
[Get started with Vertex AI Batch Prediction for custom text models](community/ml_ops/stage6/get_started_with_custom_text_model_batch.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use `Vertex AI Batch Prediction` with a custom text model.
|
||||
[Get started with re-importing AutoML tabular models](get_started_automl_tabular_exported_deploy.ipynb)
|
||||
|
||||
```
|
||||
The steps performed include:
|
||||
- Importing a pretrained AutoML tabular exported model artifacts, as a `Model` resource.
|
||||
- Create an `Endpoint` resource.
|
||||
- Deploy the `Model` resource to the `Endpoint` resource.
|
||||
- Make a prediction.
|
||||
```
|
||||
|
||||
- Download a pretrained TensorFlow RNN model.
|
||||
- Upload the pretrained model as a `Vertex AI Model` resource.
|
||||
- Make batch prediction to the `Model` resource, in JSONL format.
|
||||
|
||||
[Get started with NVIDIA Triton server](community/ml_ops/stage6/get_started_with_nvidia_triton_serving.ipynb)
|
||||
|
||||
In this tutorial, you deploy a container running Nvidia Triton Server with a `Vertex AI Model` resource to a `Vertex AI Endpoint` for making online predictions.
|
||||
|
||||
The steps performed in this tutorial include:
|
||||
|
||||
- Download the model artifacts from TensorFlow Hub.
|
||||
- Create Triton serving configuration file for the model.
|
||||
- Construct a custom container, with Triton serving image, for model deployment.
|
||||
- Upload the model as a `Vertex AI Model` resource.
|
||||
- Deploy the `Vertex AI Model` resource to a `Vertex AI Endpoint` resource.
|
||||
- Make a prediction request
|
||||
- Undeploy the `Model` resource and delete the `Endpoint`
|
||||
|
||||
[Get started with Vertex AI Batch Prediction for custom image models](community/ml_ops/stage6/get_started_with_custom_image_model_batch.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use `Vertex AI Batch Prediction` with a custom image model.
|
||||
[Get started with Vertex Explainable AI using custom deployment container](get_started_with_xai_and_custom_server.ipynb)
|
||||
|
||||
```
|
||||
The steps performed include:
|
||||
- Locally train a Pytorch tabular classifier.
|
||||
- Locally test the trained model.
|
||||
- Build a HTTP server using FastAPI.
|
||||
- Create a custom serving container with the trained model and FastAPI server.
|
||||
- Locally test the custom serving container.
|
||||
- Push the custom serving container to the Artifact Registry.
|
||||
- Upload the custom serving container as a `Model` resource.
|
||||
- Deploy the `Model` resource to an `Endpoint` resource.
|
||||
- Make a prediction request to the deployed custom serving container.
|
||||
- Make an explanation request to the deployed custom serving container.
|
||||
|
||||
```
|
||||
|
||||
[Get started with TensorFlow serving functions with Vertex AI Raw Prediction](get_started_with_raw_predict.ipynb)
|
||||
|
||||
```
|
||||
The steps performed include:
|
||||
- Download a pretrained tabular classification model artifacts for a TensorFlow 1.x estimator.
|
||||
- Upload the TensorFlow estimator model as a `Vertex AI Model` resource.
|
||||
- Creating an `Endpoint` resource.
|
||||
- Deploying the `Model` resource to an `Endpoint` resource.
|
||||
- Make an online raw prediction to the `Model` resource instance deployed to the `Endpoint` resource.
|
||||
```
|
||||
|
||||
[Get started with Vertex AI Matching Engine and Two Towers builtin algorithm](get_started_with_matching_engine_twotowers.ipynb)
|
||||
|
||||
```
|
||||
The steps performed include:
|
||||
1. Train the `Two-Tower` algorithm to generate embeddings (encoder) for the dataset.
|
||||
2. Hyperparameter tune the trained `Two-Tower` encoder.
|
||||
3. Make example predictions (embeddings) from then trained encoder.
|
||||
4. Generate embeddings using the trained `Two-Tower` builtin algorithm.
|
||||
5. Store embeddings to format supported by `Matching Engine`.
|
||||
6. Create a `Matching Engine Index` for the embeddings.
|
||||
7. Deploy the `Matching Engine Index` to a `Index Endpoint`.
|
||||
8. Make a matching engine prediction request.
|
||||
|
||||
```
|
||||
|
||||
[Get started with Optimized TensorFlow Enterprise container with Vertex AI Prediction / text models](get_started_with_optimized_tfe_bert.ipynb)
|
||||
|
||||
```
|
||||
The steps performed include:
|
||||
- Download a pretrained BERT model from TensorFlow Hub.
|
||||
- Fine-tune (transfer learning) the BERT model as a binary classifier.
|
||||
- Upload the TensorFlow Hub model as a `Vertex AI Model` resource, with standard TensorFlow serving container.
|
||||
- Upload the TensorFlow Hub model as a `Vertex AI Model` resource, with TensorFlow Enterprise Optimized container
|
||||
- Create two `Endpoint` resources.
|
||||
- Deploying both `Model` resources to separate `Endpoint` resources.
|
||||
- Make the same online prediction requests to both `Model` resource instances deployed to the `Endpoint` resources.
|
||||
- Compare the prediction accuracy between the two deployed `Model` resources.
|
||||
- Configuring container settings for fine-tune control of optimizations.
|
||||
- Create a `Private Endpoint` resource.
|
||||
- Deploy the `Model` resoure with then `TensorFlow Enterprise Optimized` to the `Private Endpoint` resource.
|
||||
- Make an online prediction request to the `Private Endpoint` resource.
|
||||
|
||||
```
|
||||
|
||||
[Get started with Vertex AI Matching Engine](get_started_with_matching_engine.ipynb)
|
||||
|
||||
```
|
||||
The steps performed include:
|
||||
- Create ANN Index.
|
||||
- Create an IndexEndpoint with VPC Network
|
||||
- Deploy ANN Index
|
||||
- Perform online query
|
||||
- Deploy brute force Index.
|
||||
- Perform calibration between ANN and brute force index.
|
||||
|
||||
```
|
||||
|
||||
[Get started with Vertex AI Matching Engine and Swivel builtin algorithm](get_started_with_matching_engine_swivel.ipynb)
|
||||
|
||||
```
|
||||
The steps performed include:
|
||||
1. Train the `Swivel` algorithm to generate embeddings (encoder) for the dataset.
|
||||
2. Make example predictions (embeddings) from then trained encoder.
|
||||
3. Generate embeddings using the trained `Swivel` builtin algorithm.
|
||||
4. Store embeddings to format supported by `Matching Engine`.
|
||||
5. Create a `Matching Engine Index` for the embeddings.
|
||||
6. Deploy the `Matching Engine Index` to a `Index Endpoint`.
|
||||
7. Make a matching engine prediction request.
|
||||
|
||||
```
|
||||
|
||||
[Get started with TensorFlow serving with Vertex AI Prediction](get_started_with_tf_serving.ipynb)
|
||||
|
||||
```
|
||||
The steps performed include:
|
||||
- Download a pretrained image classification model from TensorFlow Hub.
|
||||
- Upload the TensorFlow Hub model as a `Vertex AI Model` resource.
|
||||
- Make batch prediction with raw (uncompressed) image data to the `Model` resource, in JSONL format.
|
||||
- Create a serving function to receive compressed image data, and output decomopressed preprocessed data for the model input.
|
||||
- Upload the TensorFlow Hub model and serving function as a `Vertex AI Model` resource.
|
||||
- Make batch prediction with compressed image data to the `Model` resource, in File-List format.
|
||||
|
||||
[Get started with Vertex AI Batch Prediction for AutoML video models](community/ml_ops/stage6/get_started_with_automl_video_model_batch.ipynb)
|
||||
|
||||
In this tutorial, you learn how to use `Vertex AI Batch Prediction` with a `AutoML` video model.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Create a Vertex `Dataset` resource.
|
||||
- Train an `AutoML` model.
|
||||
- Make a batch prediction with JSONL input.
|
||||
- Creating an `Endpoint` resource.
|
||||
- Deploying the `Model` resource to an `Endpoint` resource with `TensorFlow Serving` serving binary.
|
||||
- Make an online prediction to the `Model` resource instance deployed to the `Endpoint` resource.
|
||||
```
|
||||
+1141
-1122
File diff suppressed because it is too large
Load Diff
+1149
-1138
File diff suppressed because it is too large
Load Diff
+895
-876
File diff suppressed because it is too large
Load Diff
+1859
-1847
File diff suppressed because it is too large
Load Diff
+1381
-1398
File diff suppressed because it is too large
Load Diff
+1121
-1109
File diff suppressed because it is too large
Load Diff
@@ -29,7 +29,7 @@
|
||||
"id": "title"
|
||||
},
|
||||
"source": [
|
||||
"# E2E ML on GCP: MLOps stage 6 : Get started with Vertex AI Batch Prediction for AutoML video models\n",
|
||||
"## E2E ML on GCP: MLOps stage 6 : Get started with Vertex AI Batch Prediction for AutoML video models\n",
|
||||
"\n",
|
||||
"<table align=\"left\">\n",
|
||||
" <td>\n",
|
||||
@@ -103,7 +103,7 @@
|
||||
"source": [
|
||||
"### Dataset\n",
|
||||
"\n",
|
||||
"The dataset used for this tutorial is the [Human Motion dataset](https://TODO) from [MIT](http://cbcl.mit.edu/publications/ps/Kuehne_etal_iccv11.pdf). The version of the dataset you use in this tutorial is stored in a public Cloud Storage bucket."
|
||||
"The dataset used for this tutorial is the [Human Motion dataset](https://TODO) from [MIT](http://cbcl.mit.edu/publications/ps/Kuehne_etal_iccv11.pdf). The version of the dataset you will use in this tutorial is stored in a public Cloud Storage bucket."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -244,7 +244,7 @@
|
||||
"\n",
|
||||
"3. [Enable the following APIs: Vertex AI APIs, Compute Engine APIs, and Cloud Storage.](https://console.cloud.google.com/flows/enableapi?apiid=ml.googleapis.com,compute_component,storage-component.googleapis.com)\n",
|
||||
"\n",
|
||||
"4. If you are running this notebook locally, you need to install the [Cloud SDK]((https://cloud.google.com/sdk)).\n",
|
||||
"4. If you are running this notebook locally, you will need to install the [Cloud SDK]((https://cloud.google.com/sdk)).\n",
|
||||
"\n",
|
||||
"5. Enter your project ID in the cell below. Then run the cell to make sure the\n",
|
||||
"Cloud SDK uses the right project for all the commands in this notebook.\n",
|
||||
@@ -252,17 +252,6 @@
|
||||
"**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$`."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "project_id"
|
||||
},
|
||||
"source": [
|
||||
"#### Set your project ID\n",
|
||||
"\n",
|
||||
"**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
@@ -591,7 +580,7 @@
|
||||
"source": [
|
||||
"#### Quick peek at your data\n",
|
||||
"\n",
|
||||
"You use a version of the MIT Human Motion dataset that is stored in a public Cloud Storage bucket, using a CSV index file.\n",
|
||||
"You will use a version of the MIT Human Motion dataset that is stored in a public Cloud Storage bucket, using a CSV index file.\n",
|
||||
"\n",
|
||||
"Start by doing a quick peek at the data. You count the number of examples by counting the number of rows in the CSV index file (`wc -l`) and then peek at the first few rows."
|
||||
]
|
||||
@@ -839,7 +828,7 @@
|
||||
"source": [
|
||||
"### Get test item(s)\n",
|
||||
"\n",
|
||||
"Now do a batch prediction to your Vertex AI model. You use arbitrary examples out of the dataset as a test items."
|
||||
"Now do a batch prediction to your Vertex AI model. You will use arbitrary examples out of the dataset as a test items."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -872,7 +861,7 @@
|
||||
"source": [
|
||||
"### Make a batch input file\n",
|
||||
"\n",
|
||||
"Now make a batch input file, which you store in your local Cloud Storage bucket. The batch input file can only be in JSONL format. For JSONL file, you make one dictionary entry per line for each data item (instance). The dictionary contains the key/value pairs:\n",
|
||||
"Now make a batch input file, which you will store in your local Cloud Storage bucket. The batch input file can only be in JSONL format. For JSONL file, you make one dictionary entry per line for each data item (instance). The dictionary contains the key/value pairs:\n",
|
||||
"\n",
|
||||
"- `content`: The Cloud Storage path to the video.\n",
|
||||
"- `mimeType`: The content type. In our example, it is an `avi` file.\n",
|
||||
@@ -1062,7 +1051,7 @@
|
||||
],
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"name": "get_started_with_automl_video_model_batch.ipynb",
|
||||
"name": "get_started_with_automl_video_model.ipynb",
|
||||
"toc_visible": true
|
||||
},
|
||||
"kernelspec": {
|
||||
|
||||
@@ -81,7 +81,7 @@
|
||||
"source": [
|
||||
"### Objective\n",
|
||||
"\n",
|
||||
"In this tutorial, you learn how to deploy a container running Nvidia Triton Server with a `Vertex AI Model` resource to a `Vertex AI Endpoint` for making online predictions.\n",
|
||||
"In this tutorial, you deploy a container running Nvidia Triton Server with a `Vertex AI Model` resource to a `Vertex AI Endpoint` for making online predictions.\n",
|
||||
"\n",
|
||||
"This tutorial uses the following Google Cloud ML services:\n",
|
||||
"\n",
|
||||
|
||||
@@ -1054,17 +1054,36 @@
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"collapsed_sections": [],
|
||||
"name": "get_started_with_tf_serving_function.ipynb",
|
||||
"toc_visible": true
|
||||
},
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"name": "python3"
|
||||
}
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"collapsed_sections": [],
|
||||
"name": "get_started_with_tf_serving_function.ipynb",
|
||||
"toc_visible": true
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 0
|
||||
"environment": {
|
||||
"kernel": "python3",
|
||||
"name": "tf2-gpu.2-6.m91",
|
||||
"type": "gcloud",
|
||||
"uri": "gcr.io/deeplearning-platform-release/tf2-gpu.2-6:m91"
|
||||
},
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.7.12"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 4
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -85,7 +85,7 @@
|
||||
"\n",
|
||||
"The steps performed include:\n",
|
||||
"\n",
|
||||
"- Locally train a PyTorch tabular classifier.\n",
|
||||
"- Locally train a Pytorch tabular classifier.\n",
|
||||
"- Locally test the trained model.\n",
|
||||
"- Build a HTTP server using FastAPI.\n",
|
||||
"- Create a custom serving container with the trained model and FastAPI server.\n",
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -34,69 +34,3 @@ This stage may be done entirely by MLOps. We recommend:
|
||||
## Notebooks
|
||||
|
||||
### Get Started
|
||||
|
||||
[Vertex AI Model Monitoring for custom tabular models with TensorFlow Serving container](community/ml_ops/stage7/get_started_with_model_monitoring_custom_tf_serving.ipynb)
|
||||
|
||||
In this notebook, you learn to use the `Vertex AI Model Monitoring` service to detect feature skew and drift in the input predict requests, for custom tabular models, using a custom deployment container.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Download a pre-trained custom tabular model.
|
||||
- Upload the pre-trained model as a `Model` resource.
|
||||
- Deploying the `Model` resource to an `Endpoint` resource with `TensorFlow Serving` serving binary.
|
||||
- Configure the `Endpoint` resource for model monitoring.
|
||||
- Generate synthetic prediction requests for skew.
|
||||
- Wait for email alert notification.
|
||||
- Generate synthetic prediction requests for drift.
|
||||
- Wait for email alert notification.
|
||||
|
||||
|
||||
|
||||
[Vertex AI Model Monitoring for AutoML tabular models](community/ml_ops/stage7/get_started_with_model_monitoring_automl.ipynb)
|
||||
|
||||
In this notebook, you learn to use the `Vertex AI Model Monitoring` service to detect feature skew and drift in the input predict requests, for AutoML tabular models.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Train an `AutoML` model.
|
||||
- Deploy the `Model` resource to the `Endpoint` resource.
|
||||
- Configure the `Endpoint` resource for model monitoring.
|
||||
- Generate synthetic prediction requests for skew.
|
||||
- Wait for email alert notification.
|
||||
- Generate synthetic prediction requests for drift.
|
||||
- Wait for email alert notification.
|
||||
|
||||
|
||||
[Vertex AI Model Monitoring for custom tabular models](community/ml_ops/stage7/get_started_with_model_monitoring_custom.ipynb)
|
||||
|
||||
In this notebook, you learn to use the `Vertex AI Model Monitoring` service to detect feature skew and drift in the input predict requests, for custom tabular models.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Download a pre-trained custom tabular model.
|
||||
- Upload the pre-trained model as a `Model` resource.
|
||||
- Deploy the `Model` resource to the `Endpoint` resource.
|
||||
- Configure the `Endpoint` resource for model monitoring.
|
||||
- Generate synthetic prediction requests for skew.
|
||||
- Wait for email alert notification.
|
||||
- Generate synthetic prediction requests for drift.
|
||||
- Wait for email alert notification.
|
||||
|
||||
|
||||
|
||||
[Vertex AI Model Monitoring for setup for tabular models](community/ml_ops/stage7/get_started_with_model_monitoring_setup.ipynb)
|
||||
|
||||
In this notebook, you learn to setup the `Vertex AI Model Monitoring` service to detect feature skew and drift in the input predict requests.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Download a pre-trained custom tabular model.
|
||||
- Upload the pre-trained model as a `Model` resource.
|
||||
- Deploy the `Model` resource to the `Endpoint` resource.
|
||||
- Configure the `Endpoint` resource for model monitoring.
|
||||
- Skew and drift detection for feature inputs.
|
||||
- Skew and drift detection for feature attributions.
|
||||
- Automatic generation of the `input schema` by sending 1000 prediction request.
|
||||
- List, pause, resume and delete monitoring jobs.
|
||||
- Restart monitoring job with predefined `input schema`.
|
||||
- View logged monitored data.
|
||||
|
||||
+1551
-1532
File diff suppressed because it is too large
Load Diff
+1673
-1654
File diff suppressed because it is too large
Load Diff
+1889
-1870
File diff suppressed because it is too large
Load Diff
+1728
-1709
File diff suppressed because it is too large
Load Diff
@@ -216,10 +216,6 @@
|
||||
"source": [
|
||||
"### Set up your Google Cloud project\n",
|
||||
"\n",
|
||||
"**_NOTE_**: This notebook has been tested in the following environment:\n",
|
||||
"\n",
|
||||
"* Python version = 3.7\n",
|
||||
"\n",
|
||||
"**The following steps are required, regardless of your notebook environment.**\n",
|
||||
"\n",
|
||||
"1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
|
||||
|
||||
-2145
File diff suppressed because it is too large
Load Diff
-2743
File diff suppressed because it is too large
Load Diff
+2
-2
@@ -37,7 +37,7 @@
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/prediction/custom_prediction_routines/SDK_Custom_Predict_SDK_Integration.ipynb\">\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai-samples/main/notebooks/community/prediction/custom_prediction_routines/SDK_Custom_Predict_SDK_Integration.ipynb\">\n",
|
||||
" <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
|
||||
" Open in Vertex AI Workbench\n",
|
||||
" </a>\n",
|
||||
@@ -53,7 +53,7 @@
|
||||
"source": [
|
||||
"## Overview\n",
|
||||
"\n",
|
||||
"This tutorial demonstrates how to use Vertex AI SDK to build a custom container that uses the Custom Prediction Routine model server to serve a scikit-learn model on Vertex AI Predictions.\n",
|
||||
"This tutorial demonstrates how to use Vertex AI SDK to build a custom container that uses the Custom Prediction Routine model server to serve a scikit-learn model on Vertex AI Predictions. This is currently a **preview** feature. Pre-GA products and features might have limited support, and changes to pre-GA products and features might not be compatible with other pre-GA versions. For more information, see the [launch stage descriptions](https://cloud.google.com/products#product-launch-stages).\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
|
||||
+1
-1
@@ -53,7 +53,7 @@
|
||||
"source": [
|
||||
"## Overview\n",
|
||||
"\n",
|
||||
"This tutorial demonstrates how to use Vertex AI SDK to build a custom container that uses the Custom Prediction Routine model server to serve a scikit-learn model on Vertex AI Predictions.\n",
|
||||
"This tutorial demonstrates how to use Vertex AI SDK to build a custom container that uses the Custom Prediction Routine model server to serve a scikit-learn model on Vertex AI Predictions. This is currently a **preview** feature. Pre-GA products and features might have limited support, and changes to pre-GA products and features might not be compatible with other pre-GA versions. For more information, see the [launch stage descriptions](https://cloud.google.com/products#product-launch-stages).\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
|
||||
+1
-1
@@ -53,7 +53,7 @@
|
||||
"source": [
|
||||
"## Overview\n",
|
||||
"\n",
|
||||
"This tutorial demonstrates how to use Vertex AI SDK to build a custom container that uses the Custom Prediction Routine model server to serve a scikit-learn model on Vertex AI Predictions.\n",
|
||||
"This tutorial demonstrates how to use Vertex AI SDK to build a custom container that uses the Custom Prediction Routine model server to serve a scikit-learn model on Vertex AI Predictions. This is currently a **preview** feature. Pre-GA products and features might have limited support, and changes to pre-GA products and features might not be compatible with other pre-GA versions. For more information, see the [launch stage descriptions](https://cloud.google.com/products#product-launch-stages).\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
|
||||
+2
-2
@@ -37,7 +37,7 @@
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/prediction/custom_prediction_routines/SDK_Pytorch_Custom_Predict.ipynb\">\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai-samples/main/notebooks/community/prediction/custom_prediction_routines/SDK_Pytorch_Custom_Predict.ipynb\">\n",
|
||||
" <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
|
||||
" Open in Vertex AI Workbench\n",
|
||||
" </a>\n",
|
||||
@@ -53,7 +53,7 @@
|
||||
"source": [
|
||||
"## Overview\n",
|
||||
"\n",
|
||||
"This tutorial demonstrates how to use Vertex AI SDK to build a custom container that uses the Custom Prediction Routine model server to serve a PyTorch model on Vertex AI Predictions.\n",
|
||||
"This tutorial demonstrates how to use Vertex AI SDK to build a custom container that uses the Custom Prediction Routine model server to serve a PyTorch model on Vertex AI Predictions. This is currently a **preview** feature. Pre-GA products and features might have limited support, and changes to pre-GA products and features might not be compatible with other pre-GA versions. For more information, see the [launch stage descriptions](https://cloud.google.com/products#product-launch-stages).\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
|
||||
+2
-2
@@ -37,7 +37,7 @@
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/prediction/custom_prediction_routines/SDK_Triton_PyTorch_Local_Prediction.ipynb\">\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai-samples/main/notebooks/community/prediction/custom_prediction_routines/SDK_Triton_PyTorch_Local_Prediction.ipynb\">\n",
|
||||
" <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
|
||||
" Open in Vertex AI Workbench\n",
|
||||
" </a>\n",
|
||||
@@ -53,7 +53,7 @@
|
||||
"source": [
|
||||
"## Overview\n",
|
||||
"\n",
|
||||
"This tutorial demonstrates how to use Vertex AI SDK to locally test [NVIDIA Triton inference server](https://developer.nvidia.com/nvidia-triton-inference-server) to serve a PyTorch model and deploy it to Vertex AI Predictions.\n",
|
||||
"This tutorial demonstrates how to use Vertex AI SDK to locally test [NVIDIA Triton inference server](https://developer.nvidia.com/nvidia-triton-inference-server) to serve a PyTorch model and deploy it to Vertex AI Predictions. This is currently a **preview** feature. Pre-GA products and features might have limited support, and changes to pre-GA products and features might not be compatible with other pre-GA versions. For more information, see the [launch stage descriptions](https://cloud.google.com/products#product-launch-stages).\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"### Dataset\n",
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,619 +0,0 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "7fPc-KWUi2Xd"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Copyright 2022 Google LLC\n",
|
||||
"#\n",
|
||||
"# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
|
||||
"# you may not use this file except in compliance with the License.\n",
|
||||
"# You may obtain a copy of the License at\n",
|
||||
"#\n",
|
||||
"# https://www.apache.org/licenses/LICENSE-2.0\n",
|
||||
"#\n",
|
||||
"# Unless required by applicable law or agreed to in writing, software\n",
|
||||
"# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
|
||||
"# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
|
||||
"# See the License for the specific language governing permissions and\n",
|
||||
"# limitations under the License."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "eoXf8TfQoVth"
|
||||
},
|
||||
"source": [
|
||||
"# Convert between Vertex AI Vizier and Open Source Vizier\n",
|
||||
"\n",
|
||||
"<table align=\"left\">\n",
|
||||
"\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/vizier/conversions_vertex_vizier_and_open_source_vizier.ipynb\">\n",
|
||||
" <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/vizier/conversions_vertex_vizier_and_open_source_vizier.ipynb\">\n",
|
||||
" <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/vizier/conversions_vertex_vizier_and_open_source_vizier.ipynb\">\n",
|
||||
" <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
|
||||
" Open in Vertex AI Workbench\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
"</table>"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "b397c59391b1"
|
||||
},
|
||||
"source": [
|
||||
"## Overview\n",
|
||||
"\n",
|
||||
"This tutorial demonstrates how to migrate code between [Vertex AI Vizier](https://cloud.google.com/vertex-ai/docs/vizier/overview) and [Open Source(OSS) Vizier](https://oss-vizier.readthedocs.io/). OSS Vizier is a Python-based service for blackbox optimization and research. It allows you to setup an OSS Vizier Server that can host blackbox optimization algorithms for tuning objective functions and defining abstractions and utilities for implementing new optimization algorithms.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "AksIKBzZ-nre"
|
||||
},
|
||||
"source": [
|
||||
"### Objective\n",
|
||||
"\n",
|
||||
"In this tutorial, you learn how to use `Vertex AI Vizier` to optimize a multi-objective study and convert the code to OSS Vizier.\n",
|
||||
"\n",
|
||||
"The goal is to __`minimize`__ the objective metric:\n",
|
||||
" ```\n",
|
||||
" y1 = r*sin(theta)\n",
|
||||
" ```\n",
|
||||
"\n",
|
||||
"and simultaneously __`maximize`__ the objective metric:\n",
|
||||
" ```\n",
|
||||
" y2 = r*cos(theta)\n",
|
||||
" ```\n",
|
||||
"\n",
|
||||
"so that you will evaluate over the parameter space:\n",
|
||||
"\n",
|
||||
" - __`r`__ in [0,1],\n",
|
||||
"\n",
|
||||
" - __`theta`__ in [0, pi/2]\n",
|
||||
"\n",
|
||||
"### Costs\n",
|
||||
"\n",
|
||||
"This tutorial uses billable components of Google Cloud:\n",
|
||||
"\n",
|
||||
"* Vertex AI\n",
|
||||
"\n",
|
||||
"Learn about [Vertex AI\n",
|
||||
"pricing](https://cloud.google.com/vertex-ai/pricing) and use the [Pricing\n",
|
||||
"Calculator](https://cloud.google.com/products/calculator/)\n",
|
||||
"to generate a cost estimate based on your projected usage.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "iMHz63rPbq6P"
|
||||
},
|
||||
"source": [
|
||||
"## Installation\n",
|
||||
"\n",
|
||||
"Install the packages required for executing this notebook."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "b6f3dc43494b"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"\n",
|
||||
"# Configure the environment for the Vertex AI Workbench notebook.\n",
|
||||
"IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
|
||||
"IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
|
||||
" \"/opt/deeplearning/metadata/env_version\"\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
|
||||
"USER_FLAG = \"\"\n",
|
||||
"if IS_WORKBENCH_NOTEBOOK:\n",
|
||||
" USER_FLAG = \"--user\"\n",
|
||||
"\n",
|
||||
"! pip3 install google-vizier==0.0.4\n",
|
||||
"! pip3 install --upgrade google-cloud-aiplatform {USER_FLAG} -q"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "64d24b4fab2c"
|
||||
},
|
||||
"source": [
|
||||
"### Restart the kernel\n",
|
||||
"\n",
|
||||
"Once you've installed the additional packages, you need to restart the notebook kernel so it can find the packages."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "O8AIwN0abq6U"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"\n",
|
||||
"if not os.getenv(\"IS_TESTING\"):\n",
|
||||
" # Restart the kernel after pip installs\n",
|
||||
" import IPython\n",
|
||||
"\n",
|
||||
" app = IPython.Application.instance()\n",
|
||||
" app.kernel.do_shutdown(True)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "before_you_begin:nogpu"
|
||||
},
|
||||
"source": [
|
||||
"## Before you begin\n",
|
||||
"\n",
|
||||
"### GPU runtime\n",
|
||||
"\n",
|
||||
"This tutorial does not require a GPU runtime.\n",
|
||||
"\n",
|
||||
"### Set up your Google Cloud project\n",
|
||||
"\n",
|
||||
"**The following steps are required, regardless of your notebook environment.**\n",
|
||||
"\n",
|
||||
"1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
|
||||
"\n",
|
||||
"2. [Make sure that billing is enabled for your project.](https://cloud.google.com/billing/docs/how-to/modify-project)\n",
|
||||
"\n",
|
||||
"3. [Enable the Vertex AI APIs, Compute Engine APIs, and Cloud Storage.](https://console.cloud.google.com/flows/enableapi?apiid=ml.googleapis.com,compute_component,storage-component.googleapis.com)\n",
|
||||
"\n",
|
||||
"4. [The Google Cloud SDK](https://cloud.google.com/sdk) is already installed in Google Cloud Notebook.\n",
|
||||
"\n",
|
||||
"5. Enter your project ID in the cell below. Then run the cell to make sure the\n",
|
||||
"Cloud SDK uses the right project for all the commands in this notebook.\n",
|
||||
"\n",
|
||||
"**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$`."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "set_project_id"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"PROJECT_ID = \"[your-project-id]\" # @param {type:\"string\"}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "autoset_project_id"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
|
||||
" # Get your Google Cloud project id from gcloud\n",
|
||||
" shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
|
||||
" PROJECT_ID = shell_output[0]\n",
|
||||
" print(\"Project ID:\", PROJECT_ID)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "set_gcloud_project_id"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gcloud config set project $PROJECT_ID"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "region"
|
||||
},
|
||||
"source": [
|
||||
"#### Region\n",
|
||||
"\n",
|
||||
"You can also change the `REGION` variable, which is used for operations\n",
|
||||
"throughout the rest of this notebook. Below are regions supported for Vertex AI. We recommend that you choose the region closest to you.\n",
|
||||
"\n",
|
||||
"- Americas: `us-central1`\n",
|
||||
"- Europe: `europe-west4`\n",
|
||||
"- Asia Pacific: `asia-east1`\n",
|
||||
"\n",
|
||||
"You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
|
||||
"\n",
|
||||
"Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "04933ed28eef"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"REGION = \"[your-region]\" # @param {type:\"string\"}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "jvNx3KyF2Ou0"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"if REGION == \"[your-region]\":\n",
|
||||
" REGION = \"us-central1\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "h0SMyUsC-mzi"
|
||||
},
|
||||
"source": [
|
||||
"### Authenticate your Google Cloud account\n",
|
||||
"\n",
|
||||
"**If you are using Vertex AI Workbench Notebook**, your environment is already authenticated. Skip this step.\n",
|
||||
"\n",
|
||||
"**If you are using Colab**, run the cell below and follow the instructions\n",
|
||||
"when prompted to authenticate your account via oAuth.\n",
|
||||
"\n",
|
||||
"**Otherwise**, follow these steps:\n",
|
||||
"\n",
|
||||
"1. In the Cloud Console, go to the [**Create service account key**\n",
|
||||
" page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
|
||||
"\n",
|
||||
"2. Click **Create service account**.\n",
|
||||
"\n",
|
||||
"3. In the **Service account name** field, enter a name, and\n",
|
||||
" click **Create**.\n",
|
||||
"\n",
|
||||
"4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
|
||||
"into the filter box, and select\n",
|
||||
" **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
|
||||
"\n",
|
||||
"5. Click *Create*. A JSON file that contains your key downloads to your\n",
|
||||
"local environment.\n",
|
||||
"\n",
|
||||
"6. Enter the path to your service account key as the\n",
|
||||
"`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "iTQY9g4mRo6r"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# If you are running this notebook in Colab, run this cell and follow the\n",
|
||||
"# instructions to authenticate your Google Cloud account. This provides access to your\n",
|
||||
"# Cloud Storage bucket and lets you submit training jobs and prediction\n",
|
||||
"# requests.\n",
|
||||
"\n",
|
||||
"import os\n",
|
||||
"import sys\n",
|
||||
"\n",
|
||||
"# If on Vertex AI Workbench, then don't execute this code\n",
|
||||
"IS_COLAB = \"google.colab\" in sys.modules\n",
|
||||
"if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
|
||||
" \"DL_ANACONDA_HOME\"\n",
|
||||
"):\n",
|
||||
" if \"google.colab\" in sys.modules:\n",
|
||||
" from google.colab import auth as google_auth\n",
|
||||
"\n",
|
||||
" google_auth.authenticate_user()\n",
|
||||
"\n",
|
||||
" # If you are running this notebook locally, replace the string below with the\n",
|
||||
" # path to your service account key and run this cell to authenticate your Google Cloud\n",
|
||||
" # account.\n",
|
||||
" elif not os.getenv(\"IS_TESTING\"):\n",
|
||||
" %env GOOGLE_APPLICATION_CREDENTIALS ''"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "Dax2zrpTi2Xy"
|
||||
},
|
||||
"source": [
|
||||
"### Import libraries and define constants"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "xD60d6Q0i2X0"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import datetime\n",
|
||||
"import math"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "CWuu4wmki2X3"
|
||||
},
|
||||
"source": [
|
||||
"## Tutorial\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "KyEjqIdnad0w"
|
||||
},
|
||||
"source": [
|
||||
"This section defines some parameters to create the study and optimize the objective function.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "8HCgeF8had77"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# These will be automatically filled in.\n",
|
||||
"STUDY_DISPLAY_NAME = \"{}_study_{}\".format(\n",
|
||||
" PROJECT_ID.replace(\"-\", \"\"), datetime.datetime.now().strftime(\"%Y%m%d_%H%M%S\")\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"print(\"REGION: {}\".format(REGION))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "8NBduXsEaRKr"
|
||||
},
|
||||
"source": [
|
||||
"### Define the parameters\n",
|
||||
"\n",
|
||||
"The following is a sample study configuration, built as a hierarchical python dictionary. It is already filled out. Run the cell to configure the study.\n",
|
||||
"\n",
|
||||
"__`USE_VERTEX_VIZIER`__: Uses Vertex Vizier SDK to do the optimization if True. Use OSS Vizier otherwise.\n",
|
||||
"\n",
|
||||
"__`SUGGESTION_COUNT`__: The number of suggestions (trials) requested in a single request.\n",
|
||||
"\n",
|
||||
"__`MAX_NUM_ITERATIONS`__: The number of iterations to explore before stopping. It is set to 4 to shorten the time to run the code, so don't expect convergence. For convergence, it would likely need to be about 20.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "E1VNJ4YBznhR"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"USE_VERTEX_VIZIER = True # @param {type:\"boolean\"}\n",
|
||||
"\n",
|
||||
"MAX_NUM_ITERATIONS = 4 # @param {type:\"integer\"}\n",
|
||||
"\n",
|
||||
"SUGGESTION_COUNT = 2 # @param {type:\"integer\"}\n",
|
||||
"\n",
|
||||
"OWNER = \"owner\" # @param {type:\"string\"}\n",
|
||||
"\n",
|
||||
"SERVICE_ENDPOINT = \"127.0.0.1:8888\" # @param {type:\"string\"}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "4_Yvt-7Z8_re"
|
||||
},
|
||||
"source": [
|
||||
"### Import the package and define `create_study` for different sources\n",
|
||||
"\n",
|
||||
"In Vertex Vizier, `project` and `location` are already specified and `Study.create_or_load` is called to create a study. You need to input the owner of your study and the server address in the format [ip:port]. To bring up the OSS Vizier server, please follow the [instructions](https://oss-vizier.readthedocs.io/) on the OSS Vizier website."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "0sAHZn1406VR"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"if USE_VERTEX_VIZIER:\n",
|
||||
" from google.cloud import aiplatform\n",
|
||||
" from google.cloud.aiplatform.vizier import Study, pyvizier\n",
|
||||
"\n",
|
||||
" def create_study(project, location, display_name, problem):\n",
|
||||
" aiplatform.init(project=project, location=location)\n",
|
||||
" study = Study.create_or_load(display_name=display_name, problem=problem)\n",
|
||||
" return study\n",
|
||||
"\n",
|
||||
"else:\n",
|
||||
" from vizier.service import clients, pyvizier\n",
|
||||
"\n",
|
||||
" def create_study(project, location, display_name, problem):\n",
|
||||
" clients.environment_variables.service_endpoint = SERVICE_ENDPOINT\n",
|
||||
" study = clients.Study.from_study_config(\n",
|
||||
" problem, owner=OWNER, study_id=STUDY_DISPLAY_NAME\n",
|
||||
" )\n",
|
||||
" return study"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "guvkcQe_-zQf"
|
||||
},
|
||||
"source": [
|
||||
"### Metric evaluation functions\n",
|
||||
"\n",
|
||||
"Next, define some functions to evaluate the two objective metrics.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "Fjfk5_c900Oz"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# r * sin(theta)\n",
|
||||
"def Metric1Evaluation(r, theta):\n",
|
||||
" \"\"\"Evaluate the first metric on the trial.\"\"\"\n",
|
||||
" return r * math.sin(theta)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# r * cos(theta)\n",
|
||||
"def Metric2Evaluation(r, theta):\n",
|
||||
" \"\"\"Evaluate the second metric on the trial.\"\"\"\n",
|
||||
" return r * math.cos(theta)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def CreateMetrics(r, theta):\n",
|
||||
" # Evaluate both objective metrics for this trial\n",
|
||||
" y1 = Metric1Evaluation(r, theta)\n",
|
||||
" y2 = Metric2Evaluation(r, theta)\n",
|
||||
" print(\n",
|
||||
" \"[r = {}, theta = {}] => y1 = r*sin(theta) = {}, y2 = r*cos(theta) = {}\".format(\n",
|
||||
" r, theta, y1, y2\n",
|
||||
" )\n",
|
||||
" )\n",
|
||||
" measurement = pyvizier.Measurement()\n",
|
||||
" measurement.metrics[\"y1\"] = y1\n",
|
||||
" measurement.metrics[\"y2\"] = y2\n",
|
||||
"\n",
|
||||
" # Return the results for this trial\n",
|
||||
" return measurement"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "2DgUIEpZ-_fJ"
|
||||
},
|
||||
"source": [
|
||||
"### Optimization\n",
|
||||
"\n",
|
||||
"The following code defines a study with parameters and metrics, evaluates the metric information based on the suggestions from Vizier, and reports the metrics value back. After a few rounds of iteration, you can get optimal trials by calling `optimal_trials()`. The code is adapt to both Vertex Vizier and OSS Vizier."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "s-AHfPOASXXW"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"problem = pyvizier.StudyConfig()\n",
|
||||
"problem.algorithm = pyvizier.Algorithm.RANDOM_SEARCH\n",
|
||||
"\n",
|
||||
"# Objective Metrics\n",
|
||||
"problem.metric_information.append(\n",
|
||||
" pyvizier.MetricInformation(name=\"y1\", goal=pyvizier.ObjectiveMetricGoal.MINIMIZE)\n",
|
||||
")\n",
|
||||
"problem.metric_information.append(\n",
|
||||
" pyvizier.MetricInformation(name=\"y2\", goal=pyvizier.ObjectiveMetricGoal.MAXIMIZE)\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# Defines the parameters configuration.\n",
|
||||
"root = problem.search_space.select_root()\n",
|
||||
"root.add_float_param(\"r\", 0, 1.0, scale_type=pyvizier.ScaleType.LINEAR)\n",
|
||||
"root.add_float_param(\"theta\", 0, 1.57, scale_type=pyvizier.ScaleType.LINEAR)\n",
|
||||
"\n",
|
||||
"study = create_study(\n",
|
||||
" project=PROJECT_ID,\n",
|
||||
" location=REGION,\n",
|
||||
" display_name=STUDY_DISPLAY_NAME,\n",
|
||||
" problem=problem,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"for _ in range(MAX_NUM_ITERATIONS):\n",
|
||||
" trials = study.suggest(count=SUGGESTION_COUNT)\n",
|
||||
" for trial in trials:\n",
|
||||
" materialize_trial = trial.materialize()\n",
|
||||
" measurement = CreateMetrics(\n",
|
||||
" materialize_trial.parameters.get_value(\"r\"),\n",
|
||||
" materialize_trial.parameters.get_value(\"theta\"),\n",
|
||||
" )\n",
|
||||
" trial.add_measurement(measurement=measurement)\n",
|
||||
" trial.complete(measurement=measurement)\n",
|
||||
"\n",
|
||||
"optimal_trials = study.optimal_trials()\n",
|
||||
"print(\"optimal_trials: {}\".format(optimal_trials))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "KAxfq9Fri2YV"
|
||||
},
|
||||
"source": [
|
||||
"## Cleaning up\n",
|
||||
"\n",
|
||||
"To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
|
||||
"project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial. You can also manually delete resources that you created by running the following code."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "zQlLDfvlzYde"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"study.delete()"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"collapsed_sections": [],
|
||||
"name": "conversions_vertex_vizier_and_open_source_vizier.ipynb",
|
||||
"toc_visible": true
|
||||
},
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"name": "python3"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 0
|
||||
}
|
||||
@@ -1,34 +0,0 @@
|
||||
tag,notebook,doc
|
||||
"AutoML, Text data",official/automl/automl-text-classification.ipynb,vertex-ai/docs/text-data/classification/train-model
|
||||
"AutoML, Tabular data",official/automl/sdk_automl_tabular_forecasting_batch.ipynb,vertex-ai/docs/tabular-data/forecasting/tutorials-samples
|
||||
"BigQuery, Vertex AI Workbench",official/workbench/exploratory_data_analysis/explore_data_in_bigquery_with_workbench.ipynb,
|
||||
"BigQuery ML, Vertex AI Model Registry, Batch prediction",official/model-registry/bqml-vertexai-model-registry.ipynb,
|
||||
"BigQuery ML, Vertex AI Model Registry, Online prediction",official/bigquery_ml/bqml-online-prediction.ipynb,
|
||||
Custom Training,official/custom/sdk-custom-image-classification-batch.ipynb,
|
||||
Custom Training,official/custom/sdk-custom-image-classification-online.ipynb,
|
||||
Tabular Data,official/automl/automl_forecasting_bqml_arima_plus_comparison.ipynb,vertex-ai/docs/tabular-data/forecasting-arima/overview
|
||||
"AutoML, Tabular Data",official/automl/automl_tabular_on_vertex_pipelines.ipynb,vertex-ai/docs/tabular-data/tabular-workflows/e2e-automl
|
||||
Vertex AI Experiments,official/experiments/comparing_pipeline_runs.ipynb,
|
||||
Vertex AI Experiments,official/experiments/build_model_experimentation_lineage_with_prebuild_code.ipynb,
|
||||
Vertex AI Experiments,official/experiments/comparing_local_trained_models.ipynb,
|
||||
Vertex AI Feature Store,official/feature_store/sdk-feature-store.ipynb,
|
||||
Matching Engine,official/matching_engine/sdk_matching_engine_for_indexing.ipynb,
|
||||
Model Monitoring,official/model_monitoring/model_monitoring.ipynb,
|
||||
Vertex AI Pipelines,official/pipelines/lightweight_functions_component_io_kfp.ipynb,
|
||||
"Vertex AI Pipelines Image data",official/pipelines/google_cloud_pipeline_components_automl_images.ipynb,
|
||||
"Vertex AI Pipelines, Tabular data",official/pipelines/automl_tabular_classification_beans.ipynb,
|
||||
"Vertex AI Pipelines, Tabular data",official/pipelines/google_cloud_pipeline_components_automl_tabular.ipynb,
|
||||
"Vertex AI Pipelines, Text data",official/pipelines/google_cloud_pipeline_components_automl_text.ipynb,
|
||||
Vertex AI Pipelines,official/pipelines/custom_model_training_and_batch_prediction.ipynb,
|
||||
Vertex AI Pipelines,official/pipelines/google_cloud_pipeline_components_model_train_upload_deploy.ipynb,
|
||||
Vertex AI Pipelines,official/pipelines/control_flow_kfp.ipynb,
|
||||
Vertex AI Pipelines,official/pipelines/metrics_viz_run_compare_kfp.ipynb,
|
||||
Vertex AI Pipelines,official/pipelines/pipelines_intro_kfp.ipynb,
|
||||
Vertex AI Vizier,official/vizier/gapic-vizier-multi-objective-optimization.ipynb,vertex-ai/docs/vizier/using-vizier
|
||||
"Vertex Explainable AI, Tabular data",official/explainable_ai/sdk_automl_tabular_binary_classification_batch_explain.ipynb,vertex-ai/docs/explainable-ai/overview
|
||||
"Vertex Explainable AI, Tabular data",official/explainable_ai/sdk_automl_tabular_classification_online_explain.ipynb,vertex-ai/docs/explainable-ai/overview
|
||||
"Vertex Explainable AI, Image data",official/explainable_ai/sdk_custom_image_classification_batch_explain.ipynb,vertex-ai/docs/explainable-ai/overview
|
||||
"Vertex Explainable AI, Image data",official/explainable_ai/sdk_custom_image_classification_online_explain.ipynb,vertex-ai/docs/explainable-ai/overview
|
||||
"Vertex Explainable AI, Tabular data",official/explainable_ai/sdk_custom_tabular_regression_batch_explain.ipynb,vertex-ai/docs/explainable-ai/overview
|
||||
"Vertex Explainable AI, Tabular data",official/explainable_ai/sdk_custom_tabular_regression_online_explain.ipynb,vertex-ai/docs/explainable-ai/overview
|
||||
Vertex ML Metadata,official/ml_metadata/sdk-metric-parameter-tracking-for-custom-jobs.ipynb,
|
||||
|
@@ -134,30 +134,16 @@
|
||||
"### Set up your local development environment\n",
|
||||
"\n",
|
||||
"**If you are using Colab or Vertex AI Workbench Notebooks**, your environment already meets\n",
|
||||
"all the requirements to run this notebook. You can skip this step."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "24743cf4a1e1"
|
||||
},
|
||||
"source": [
|
||||
"**_NOTE_**: This notebook has been tested in the following environment:\n",
|
||||
"all the requirements to run this notebook. You can skip this step.\n",
|
||||
"\n",
|
||||
"* Python version = 3.9"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "gCuSR8GkAgzl"
|
||||
},
|
||||
"source": [
|
||||
"**Otherwise**, make sure your environment meets this notebook's requirements.\n",
|
||||
"You need the following:\n",
|
||||
"\n",
|
||||
"* The Google Cloud SDK\n",
|
||||
"* Git\n",
|
||||
"* Python 3\n",
|
||||
"* virtualenv\n",
|
||||
"* Jupyter notebook running in a virtual environment with Python 3\n",
|
||||
"\n",
|
||||
"The Google Cloud guide to [Setting up a Python development\n",
|
||||
"environment](https://cloud.google.com/python/setup) and the [Jupyter\n",
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -17,12 +17,11 @@
|
||||
/tensorboard @zbl94
|
||||
|
||||
/bigquery_ml/bqml-online-prediction.ipynb @polong-lin
|
||||
/model_monitoring/model_monitoring.ipynb @andrewferlitsch
|
||||
/model_monitoring/model_monitoring.ipynb @mco-gh
|
||||
/ml_metadata/sdk-metric-parameter-tracking-for-custom-jobs.ipynb @jialuzh
|
||||
/ml_metadata/sdk-metric-parameter-tracking-for-locally-trained-models.ipynb @jialuzh
|
||||
/ml_metadata/vertex-pipelines-ml-metadata.ipynb @sararob
|
||||
/vizier/gapic-vizier-multi-objective-optimization.ipynb @halio-g
|
||||
/training/xgboost_data_parallel_training_on_cpu_using_dask.ipynb @halio-g
|
||||
/feature_store/gapic-feature-store.ipynb @diemtvu
|
||||
/managed_notebooks @GoogleCloudPlatform/notebooks-team
|
||||
/pipelines/google_cloud_pipeline_components_bqml_text.ipynb @inardini
|
||||
@@ -32,9 +31,7 @@
|
||||
/custom/custom_training_tensorboard_profiler.ipynb @itseric
|
||||
/workbench/spark/spark_sample_notebook.ipynb @bradmiro
|
||||
/workbench/spark/spark_ml.ipynb @bradmiro
|
||||
/model_registry/bqml_vertexai_model_registry.ipynb @soheilazangeneh
|
||||
/model-registry/bqml-vertexai-model-registry.ipynb @soheilazangeneh
|
||||
/workbench/exploratory_data_analysis/explore_data_in_bigquery_with_workbench.ipynb @alokpattani
|
||||
/model_evaluation/automl_tabular_classification_model_evaluation.ipynb @soheilazangeneh
|
||||
/model_evaluation/automl_tabular_regression_model_evaluation.ipynb @soheilazangeneh
|
||||
/tabular_workflows/tabnet_on_vertex_pipelines.ipynb @sakagarwal
|
||||
/tabular_workflows/wide_and_deep_on_vertex_pipelines.ipynb @sakagarwal
|
||||
|
||||
+395
-326
@@ -6,29 +6,39 @@ The official notebooks are organized by Google Cloud Vertex AI services.
|
||||
|
||||
## Manifest of Curated Notebooks
|
||||
|
||||
### AutoML Text data
|
||||
### AutoML
|
||||
|
||||
[AutoML text classification model](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/automl-text-classification.ipynb)
|
||||
|
||||
[Create, train, and deploy an AutoML text classification model](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/automl-text-classification.ipynb)
|
||||
<blockquote>
|
||||
In this tutorial, you learn how to use `AutoML` to train a text classification model.
|
||||
|
||||
Learn how to use `AutoML` to train a text classification model.
|
||||
This tutorial uses the following Google Cloud ML services:
|
||||
|
||||
- `AutoML Training`
|
||||
- `Vertex AI Model resource`
|
||||
|
||||
The steps performed include:
|
||||
|
||||
* Create a `Vertex AI Dataset`.
|
||||
* Train an `AutoML` text classification `Model` resource.
|
||||
* Obtain the evaluation metrics for the `Model` resource.
|
||||
* Create an `Endpoint` resource.
|
||||
* Deploy the `Model` resource to the `Endpoint` resource.
|
||||
* Make an online prediction
|
||||
* Make a batch prediction
|
||||
- Create a `Vertex AI Dataset`
|
||||
- Train an `AutoML` text classification `Model` resource.
|
||||
- Obtain the evaluation metrics for the `Model` resource.
|
||||
- Create an `Endpoint` resource.
|
||||
- Deploy the `Model` resource to the `Endpoint` resource.
|
||||
- Make an online prediction.
|
||||
- Make a batch prediction.
|
||||
</blockquote>
|
||||
|
||||
### AutoML Tabular data
|
||||
[AutoML tabular forecasting model](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_tabular_forecasting_batch.ipynb)
|
||||
|
||||
<blockquote>
|
||||
In this tutorial, you create an `AutoML` tabular forecasting model from a Python script, and then do a batch prediction using the Vertex AI SDK.
|
||||
|
||||
[AutoML tabular forecasting model for batch prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_tabular_forecasting_batch.ipynb)
|
||||
This tutorial uses the following Google Cloud ML services:
|
||||
|
||||
Learn how to create an `AutoML` tabular forecasting model from a Python script, and then do a batch prediction using the Vertex AI SDK.
|
||||
- `AutoML Training`
|
||||
- `Vertex AI Batch Prediction`
|
||||
- `Vertex AI Model` resource
|
||||
|
||||
The steps performed include:
|
||||
|
||||
@@ -36,325 +46,63 @@ The steps performed include:
|
||||
- Train an `AutoML` tabular forecasting `Model` resource.
|
||||
- Obtain the evaluation metrics for the `Model` resource.
|
||||
- Make a batch prediction.
|
||||
</blockquote>
|
||||
|
||||
### BigQuery ML Vertex AI Model Registry Batch prediction
|
||||
### Vertex AI Training
|
||||
|
||||
[Custom image classification model training and batch prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/custom/sdk-custom-image-classification-batch.ipynb)
|
||||
|
||||
[Deploy BiqQuery ML Model on Vertex AI Model Registry and make predictions](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model-registry/bqml-vertexai-model-registry.ipynb)
|
||||
<blockquote>
|
||||
In this tutorial, you learn to use `Vertex AI Training` to create a custom trained model and use `Vertex AI Batch Prediction` to do a batch prediction on the trained model.
|
||||
|
||||
Learn how to use `Vertex AI Model Registry` with `BigQuery ML` and make batch predictions:
|
||||
This tutorial uses the following Google Cloud ML services:
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Train a model with `BigQuery ML`
|
||||
- Upload the model to `Vertex AI Model Registry`
|
||||
- Create a `Vertex AI Endpoint` resource
|
||||
- Deploy the `Model` resource to the `Endpoint` resource
|
||||
- Make `prediction` requests to the model endpoint
|
||||
- Run `batch prediction` job on the `Model` resource
|
||||
|
||||
|
||||
### BigQuery ML Vertex AI Model Registry Online prediction
|
||||
|
||||
|
||||
[Online prediction with BigQuery ML](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/bigquery_ml/bqml-online-prediction.ipynb)
|
||||
|
||||
Learn how to train and deploy a churn prediction model for real-time inference, with the data in BigQuery and model trained using BigQuery ML, registered to Vertex AI Model Registry, and deployed to an endpoint on Vertex AI for online predictions.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Using Python & SQL to query the public data in BigQuery
|
||||
- Preparing the data for modeling
|
||||
- Training a classification model using BigQuery ML and registering it to Vertex AI Model Registry
|
||||
- Inspecting the model on Vertex AI Model Registry
|
||||
- Deploying the model to an endpoint on Vertex AI
|
||||
- Making sample online predictions to the model endpoint
|
||||
|
||||
|
||||
### Custom Training
|
||||
|
||||
|
||||
[Custom training and batch prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/custom/sdk-custom-image-classification-batch.ipynb)
|
||||
|
||||
Learn to use `Vertex AI Training` to create a custom trained model and use `Vertex AI Batch Prediction` to do a batch prediction on the trained model.
|
||||
- `Vertex AI Training`
|
||||
- `Vertex AI Batch Prediction`
|
||||
- `Vertex AI Model` resource
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Create a `Vertex AI` custom job for training a TensorFlow model.
|
||||
- Upload the trained model artifacts as a `Model` resource.
|
||||
- Make a batch prediction.
|
||||
</blockquote>
|
||||
|
||||
[Custom training and online prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/custom/sdk-custom-image-classification-online.ipynb)
|
||||
[Custom image classification model training and online prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/custom/sdk-custom-image-classification-online.ipynb)
|
||||
|
||||
Learn to use `Vertex AI Training` to create a custom-trained model from a Python script in a Docker container, and learn to use `Vertex AI Prediction` to do a prediction on the deployed model by sending data.
|
||||
<blockquote>
|
||||
In this tutorial, you learn to use `Vertex AI Training` to create a custom-trained model from a Python script in a Docker container, and learn to use `Vertex AI Prediction` to do a prediction on the deployed model by sending data.
|
||||
|
||||
This tutorial uses the following Google Cloud ML services:
|
||||
|
||||
- `Vertex AI Training`
|
||||
- `Vertex AI Prediction`
|
||||
- `Vertex AI Model` resource
|
||||
- `Vertex AI Endpoint` resource
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Create a `Vertex AI` custom job for training a TensorFlow model.
|
||||
- Upload the trained model artifacts to a `Model` resource.
|
||||
- Create a serving `Endpoint` resource.
|
||||
- Deploy the `Model` resource to a serving `Endpoint` resource.
|
||||
- Deploy the Model resource to a serving `Endpoint` resource.
|
||||
- Make a prediction.
|
||||
- Undeploy the `Model` resource.
|
||||
</blockquote>
|
||||
|
||||
### Tabular Data
|
||||
### Vertex Explainable AI
|
||||
|
||||
[AutoML tabular binary classification model with batch explanations](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/explainable_ai/sdk_automl_tabular_binary_classification_batch_explain.ipynb)
|
||||
|
||||
[Compare Vertex AI Forecasting and BigQuery ML ARIMA_PLUS](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/automl_forecasting_bqml_arima_plus_comparison.ipynb)
|
||||
<blockquote>
|
||||
In this tutorial, you learn to use `AutoML` to create a tabular binary classification model from a Python script, and then learn to use `Vertex AI Batch Prediction` to make predictions with explanations.
|
||||
|
||||
Learn how to create an BQML ARIMA_PLUS model using a training [Vertex AI Pipeline](https://cloud.
|
||||
This tutorial uses the following Google Cloud ML services:
|
||||
|
||||
The steps performed are:
|
||||
|
||||
- Train the BQML ARIMA_PLUS model.
|
||||
- View BQML model evaluation.
|
||||
- Make a batch prediction with the BQML model.
|
||||
- Create a Vertex AI `Dataset` resource.
|
||||
- Train the Vertex AI Forecasting model.
|
||||
- View the Model evaluation.
|
||||
- Make a batch prediction with the Model.
|
||||
|
||||
|
||||
### AutoML Tabular Data
|
||||
|
||||
|
||||
[AutoML Tabular Pipeline](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/automl_tabular_on_vertex_pipelines.ipynb)
|
||||
|
||||
Learn how to create two regression models using [Vertex Pipelines](https://cloud.
|
||||
|
||||
The steps performed are:
|
||||
|
||||
- Create a training pipeline that reduces the search space from the default to save time.
|
||||
- Create a training pipeline that reuses the architecture search results from the previous pipeline to save time.
|
||||
|
||||
### Vertex AI Experiments
|
||||
|
||||
|
||||
[Compare pipeline runs with Vertex AI Experiments](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/experiments/comparing_pipeline_runs.ipynb)
|
||||
|
||||
Learn how to use `Vertex AI Experiments` to log a pipeline job and compare different pipeline jobs.
|
||||
|
||||
|
||||
|
||||
[Build Vertex AI Experiment lineage for custom training](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/experiments/build_model_experimentation_lineage_with_prebuild_code.ipynb)
|
||||
|
||||
Learn how to integrate preprocessing code in a Vertex AI experiments.
|
||||
|
||||
|
||||
|
||||
[Track parameters and metrics for locally trained models](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/experiments/comparing_local_trained_models.ipynb)
|
||||
|
||||
Learn how to use Vertex AI Experiments to compare and evaluate model experiments.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- log the model parameters
|
||||
- log the loss and metrics on every epoch to TensorBoard
|
||||
- log the evaluation metrics
|
||||
|
||||
|
||||
### Vertex AI Feature Store
|
||||
|
||||
|
||||
[Online and Batch predictions using Vertex AI Feature Store](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/feature_store/sdk-feature-store.ipynb)
|
||||
|
||||
Learn how to use `Vertex AI Feature Store` to import feature data, and to access the feature data for both online serving and offline tasks, such as training.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Create featurestore, entity type, and feature resources.
|
||||
- Import feature data into `Vertex AI Feature Store` resource.
|
||||
- Serve online prediction requests using the imported features.
|
||||
- Access imported features in offline jobs, such as training jobs.
|
||||
|
||||
### Matching Engine
|
||||
|
||||
|
||||
[Create Vertex AI Matching Engine index](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/matching_engine/sdk_matching_engine_for_indexing.ipynb)
|
||||
|
||||
Learn how to create Approximate Nearest Neighbor (ANN) Index, query against indexes, and validate the performance of the index.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
* Create ANN Index and Brute Force Index
|
||||
* Create an IndexEndpoint with VPC Network
|
||||
* Deploy ANN Index and Brute Force Index
|
||||
* Perform online query
|
||||
* Compute recall
|
||||
|
||||
|
||||
### Model Monitoring
|
||||
|
||||
|
||||
[Vertex AI Model Monitoring with Explainable AI Feature Attributions](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model_monitoring/model_monitoring.ipynb)
|
||||
|
||||
Learn to use the `Vertex AI Model Monitoring` service to detect drift and anomalies in prediction requests from a deployed `Vertex AI Model` resource.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Upload a pre-trained model as a `Vertex AI Model` resource.
|
||||
- Create an `Vertex AI Endpoint` resource.
|
||||
- Deploy the `Model` resource to the `Endpoint` resource.
|
||||
- Configure the `Endpoint` resource for model monitoring.
|
||||
- Initialize the baseline distribution for model monitoring.
|
||||
- Generate synthetic prediction requests.
|
||||
- Understand how to interpret the statistics, visualizations, other data reported by the model monitoring feature.
|
||||
|
||||
### Vertex AI Pipelines
|
||||
|
||||
|
||||
[Lightweight Python function-based components, and component I/O](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/lightweight_functions_component_io_kfp.ipynb)
|
||||
|
||||
Learn to use the KFP SDK to build lightweight Python function-based components, and then you learn to use `Vertex AI Pipelines` to execute the pipeline.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Build Python function-based KFP components.
|
||||
- Construct a KFP pipeline.
|
||||
- Pass *Artifacts* and *parameters* between components, both by path reference and by value.
|
||||
- Use the `kfp.dsl.importer` method.
|
||||
- Compile the KFP pipeline.
|
||||
- Execute the KFP pipeline using `Vertex AI Pipelines`
|
||||
|
||||
### Vertex AI Pipelines Image data
|
||||
|
||||
|
||||
[AutoML image classification pipelines using google-cloud-pipeline-components](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/google_cloud_pipeline_components_automl_images.ipynb)
|
||||
|
||||
Learn how to use `Vertex AI Pipelines` and `Google Cloud Pipeline Components` to build an `AutoML` image classification model.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Create a KFP pipeline:
|
||||
- Create a `Dataset` resource.
|
||||
- Train an AutoML image classification `Model` resource.
|
||||
- Create an `Endpoint` resource.
|
||||
- Deploys the `Model` resource to the `Endpoint` resource.
|
||||
- Compile the KFP pipeline.
|
||||
- Execute the KFP pipeline using `Vertex AI Pipelines`
|
||||
|
||||
|
||||
|
||||
### Vertex AI Pipelines Tabular data
|
||||
|
||||
|
||||
[AutoML Tabular pipelines using google-cloud-pipeline-components](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/automl_tabular_classification_beans.ipynb)
|
||||
|
||||
Learn to use `Vertex AI Pipelines` and `Google Cloud Pipeline Components` to build an `AutoML` tabular classification model.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Create a KFP pipeline:
|
||||
- Create a `Dataset` resource.
|
||||
- Train an AutoML tabular classification `Model` resource.
|
||||
- Create an `Endpoint` resource.
|
||||
- Deploys the `Model` resource to the `Endpoint` resource.
|
||||
- Compile the KFP pipeline.
|
||||
- Execute the KFP pipeline using `Vertex AI Pipelines`
|
||||
|
||||
|
||||
|
||||
[AutoML tabular regression pipelines using google-cloud-pipeline-components](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/google_cloud_pipeline_components_automl_tabular.ipynb)
|
||||
|
||||
Learn to use `Vertex AI Pipelines` and `Google Cloud Pipeline Components` to build an `AutoML` tabular regression model.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Create a KFP pipeline:
|
||||
- Create a `Dataset` resource.
|
||||
- Train an AutoML tabular regression `Model` resource.
|
||||
- Create an `Endpoint` resource.
|
||||
- Deploys the `Model` resource to the `Endpoint` resource.
|
||||
- Compile the KFP pipeline.
|
||||
- Execute the KFP pipeline using `Vertex AI Pipelines`
|
||||
|
||||
|
||||
|
||||
### Vertex AI Pipelines Text data
|
||||
|
||||
|
||||
[AutoML text classification pipelines using google-cloud-pipeline-components](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/google_cloud_pipeline_components_automl_text.ipynb)
|
||||
|
||||
Learn to use `Vertex AI Pipelines` and `Google Cloud Pipeline Components` to build an `AutoML` text classification model.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Create a KFP pipeline:
|
||||
- Create a `Dataset` resource.
|
||||
- Train an AutoML text classification `Model` resource.
|
||||
- Create an `Endpoint` resource.
|
||||
- Deploys the `Model` resource to the `Endpoint` resource.
|
||||
- Compile the KFP pipeline.
|
||||
- Execute the KFP pipeline using `Vertex AI Pipelines`
|
||||
|
||||
|
||||
|
||||
### Vertex AI Pipelines
|
||||
|
||||
|
||||
[Custom training with pre-built Google Cloud Pipeline Components](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/custom_model_training_and_batch_prediction.ipynb)
|
||||
|
||||
Learn to use `Vertex AI Pipelines` and `Google Cloud Pipeline Components` to build a custom model.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Create a KFP pipeline:
|
||||
- Train a custom model.
|
||||
- Upload the trained model as a `Model` resource.
|
||||
- Create an `Endpoint` resource.
|
||||
- Deploy the `Model` resource to the `Endpoint` resource.
|
||||
- Make a batch prediction request.
|
||||
|
||||
|
||||
|
||||
[Pipeline control structures using the KFP SDK](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/control_flow_kfp.ipynb)
|
||||
|
||||
Learn how to use the KFP SDK to build pipelines that use loops and conditionals, including nested examples.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Create a KFP pipeline:
|
||||
- Use control flow components
|
||||
- Compile the KFP pipeline.
|
||||
- Execute the KFP pipeline using `Vertex AI Pipelines`
|
||||
|
||||
[Metrics visualization and run comparison using the KFP SDK](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/metrics_viz_run_compare_kfp.ipynb)
|
||||
|
||||
Learn how to use the KFP SDK to build pipelines that generate evaluation metrics.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Create KFP components:
|
||||
- Generate ROC curve and confusion matrix visualizations for classification results
|
||||
- Write metrics
|
||||
- Create KFP pipelines.
|
||||
- Execute KFP pipelines
|
||||
- Compare metrics across pipeline runs
|
||||
|
||||
[Pipelines introduction for KFP](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/pipelines_intro_kfp.ipynb)
|
||||
|
||||
Learn how to use the KFP SDK to build pipelines that generate evaluation metrics.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Define and compile a `Vertex AI` pipeline.
|
||||
- Specify which service account to use for a pipeline run.
|
||||
|
||||
### Vertex AI Vizier
|
||||
|
||||
|
||||
[Optimizing multiple objectives with Vertex AI Vizier](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/vizier/gapic-vizier-multi-objective-optimization.ipynb)
|
||||
|
||||
Learn how to use `Vertex AI Vizier` to optimize a multi-objective study.
|
||||
|
||||
|
||||
|
||||
### Vertex Explainable AI Tabular data
|
||||
|
||||
|
||||
[AutoML training tabular binary classification model for batch explanation](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/explainable_ai/sdk_automl_tabular_binary_classification_batch_explain.ipynb)
|
||||
|
||||
Learn to use `AutoML` to create a tabular binary classification model from a Python script, and then learn to use `Vertex AI Batch Prediction` to make predictions with explanations.
|
||||
- `Vertex AI AutoML`
|
||||
- `Vertex AI Batch Prediction`
|
||||
- `Vertex Explainable AI`
|
||||
- `Vertex AI Model` resource
|
||||
|
||||
The steps performed include:
|
||||
|
||||
@@ -362,44 +110,65 @@ The steps performed include:
|
||||
- Train an `AutoML` tabular binary classification model.
|
||||
- View the model evaluation metrics for the trained model.
|
||||
- Make a batch prediction request with explainability.
|
||||
</blockquote>
|
||||
|
||||
[AutoML tabular binary classification model with online explanations](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/explainable_ai/sdk_automl_tabular_classification_online_explain.ipynb)
|
||||
|
||||
* Prediction Service: Does an on-demand prediction for the entire set of instances (i.e., one or more data items) and returns the results in real-time.
|
||||
<blockquote>
|
||||
In this tutorial, you learn to use `AutoML` to create a tabular binary classification model from a Python script, and then learn to use `Vertex AI Online Prediction` to make online predictions with explanations.
|
||||
|
||||
* Batch Prediction Service: Does a queued (batch) prediction for the entire set of instances in the background and stores the results in a Cloud Storage bucket when ready.
|
||||
This tutorial uses the following Google Cloud ML services:
|
||||
|
||||
[AutoML training tabular classification model for online explanation](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/explainable_ai/sdk_automl_tabular_classification_online_explain.ipynb)
|
||||
|
||||
Learn how to use `AutoML` to create a tabular binary classification model from a Python script, and then learn to use `Vertex AI Online Prediction` to make online predictions with explanations.
|
||||
- `Vertex AI AutoML`
|
||||
- `Vertex AI Prediction`
|
||||
- `Vertex Explainable AI`
|
||||
- `Vertex AI Model` resource
|
||||
- `Vertex AI Endpoint` resource
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Create a `Vertex Dataset` resource.
|
||||
- Create a `Vertex AI Dataset` resource.
|
||||
- Train an `AutoML` tabular binary classification model.
|
||||
- View the model evaluation metrics for the trained model.
|
||||
- Create a serving `Endpoint` resource.
|
||||
- Deploy the `Model` resource to a serving `Endpoint` resource.
|
||||
- Make an online prediction request with explainability.
|
||||
- Undeploy the `Model` resource.
|
||||
</blockquote>
|
||||
|
||||
### Vertex Explainable AI Image data
|
||||
[Custom tabular regression model with batch explanations](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/explainable_ai/sdk_custom_tabular_regression_batch_explain.ipynb)
|
||||
|
||||
<blockquote>
|
||||
In this tutorial, you learn to use `Vertex AI Training` and `Explainable AI` to create a custom image classification model with explanations, and then you learn to use `Vertex AI Batch Prediction` to make a batch prediction request with explanations.
|
||||
|
||||
[Custom training image classification model for batch prediction with explainabilty](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/explainable_ai/sdk_custom_image_classification_batch_explain.ipynb)
|
||||
This tutorial uses the following Google Cloud ML services:
|
||||
|
||||
Learn to use `Vertex AI Training and Explainable AI` to create a custom image classification model with explanations, and then you learn to use `Vertex AI Batch Prediction` to make a batch prediction request with explanations.
|
||||
- `Vertex AI Training`
|
||||
- `Vertex AI Batch Prediction`
|
||||
- `Vertex Explainable AI`
|
||||
- `Vertex AI Mode`l resource
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Create a `Vertex AI` custom job for training a TensorFlow model.
|
||||
- View the model evaluation for the trained model.
|
||||
- Set explanation parameters for when the model is deployed.
|
||||
- Upload the trained model artifacts and explanation parameters as a `Model` resource.
|
||||
- Upload the trained model artifacts and explanations as a `Model` resource.
|
||||
- Make a batch prediction with explanations.
|
||||
</blockquote>
|
||||
|
||||
[Custom tabular regression model with online explanations](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/explainable_ai/sdk_custom_tabular_regression_online_explain.ipynb)
|
||||
|
||||
[Custom training image classification model for online prediction with explainabilty](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/explainable_ai/sdk_custom_image_classification_online_explain.ipynb)
|
||||
<blockquote>
|
||||
In this tutorial, you learn to use `Vertex AI Training` and `Explainable AI` to create a custom image classification model with explanations, and then you learn to use `Vertex AI Prediction` to make an online prediction request with explanations.
|
||||
|
||||
Learn how to use `Vertex AI Training and Explainable AI` to create a custom image classification model with explanations, and then you learn to use `Vertex AI Prediction` to make an online prediction request with explanations.
|
||||
This tutorial uses the following Google Cloud ML services:
|
||||
|
||||
- `Vertex AI Training`
|
||||
- `Vertex AI Prediction`
|
||||
- `Vertex Explainable AI`
|
||||
- `Vertex AI Model` resource
|
||||
- `Vertex AI Endpoint` resource
|
||||
|
||||
The steps performed include:
|
||||
|
||||
@@ -411,13 +180,41 @@ The steps performed include:
|
||||
- Deploy the `Model` resource to a serving `Endpoint` resource.
|
||||
- Make a prediction with explanation.
|
||||
- Undeploy the `Model` resource.
|
||||
</blockquote>
|
||||
|
||||
[Custom image classification model with batch explanations](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/explainable_ai/sdk_custom_image_classification_batch_explain.ipynb)
|
||||
|
||||
### Vertex Explainable AI Tabular data
|
||||
<blockquote>
|
||||
In this tutorial, you learn to use `Vertex AI Training and Explainable AI` to create a custom image classification model with explanations, and then you learn to use `Vertex AI Batch Prediction` to make a batch prediction request with explanations.
|
||||
|
||||
This tutorial uses the following Google Cloud ML services:
|
||||
|
||||
[Custom training tabular regression model for batch prediction with explainabilty](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/explainable_ai/sdk_custom_tabular_regression_batch_explain.ipynb)
|
||||
- `Vertex AI Training`
|
||||
- `Vertex AI Batch Prediction`
|
||||
- `Vertex Explainable AI`
|
||||
- `Vertex AI Model` resource
|
||||
|
||||
Learn how to use `Vertex AI Training and Explainable AI` to create a custom image classification model with explanations, and then you learn to use `Vertex AI Batch Prediction` to make a batch prediction request with explanations.
|
||||
The steps performed include:
|
||||
|
||||
- Create a `Vertex AI` custom job for training a TensorFlow model.
|
||||
- View the model evaluation for the trained model.
|
||||
- Set explanation parameters for when the model is deployed.
|
||||
- Upload the trained model artifacts and explanation parameters as a `Model` resource.
|
||||
- Make a batch prediction with explanations.
|
||||
</blockquote>
|
||||
|
||||
[Custom image classification model with online explanations](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/explainable_ai/sdk_custom_image_classification_online_explain.ipynb)
|
||||
|
||||
<blockquote>
|
||||
In this tutorial, you learn to use `Vertex AI Training and Explainable AI` to create a custom image classification model with explanations, and then you learn to use `Vertex AI Prediction` to make an online prediction request with explanations.
|
||||
|
||||
This tutorial uses the following Google Cloud ML services:
|
||||
|
||||
- `Vertex AI Training`
|
||||
- `Vertex AI Online Prediction`
|
||||
- `Vertex Explainable AI`
|
||||
- `Vertex AI Model` resource
|
||||
- `Vertex AI Endpoint` resource
|
||||
|
||||
The steps performed include:
|
||||
|
||||
@@ -425,18 +222,290 @@ The steps performed include:
|
||||
- View the model evaluation for the trained model.
|
||||
- Set explanation parameters for when the model is deployed.
|
||||
- Upload the trained model artifacts and explanations as a `Model` resource.
|
||||
- Make a batch prediction with explanations.
|
||||
- Create a serving `Endpoint` resource.
|
||||
- Deploy the `Model` resource to a serving `Endpoint` resource.
|
||||
- Make a prediction with explanation.
|
||||
- Undeploy the `Model` resource.
|
||||
</blockquote>
|
||||
|
||||
### Vertex ML Metadata
|
||||
### Vertex Feature Store
|
||||
|
||||
[Managing features in a feature store](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/feature_store/gapic-feature-store.ipynb)
|
||||
|
||||
[Track parameters and metrics for custom training jobs](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-custom-jobs.ipynb)
|
||||
<blockquote>
|
||||
In this notebook, you will learn how to use `Vertex AI Feature Store` to import feature data, and to access the feature data for both online serving and offline tasks, such as training.
|
||||
|
||||
Learn how to use Vertex AI SDK for Python to:
|
||||
This tutorial uses the following Google Cloud ML services:
|
||||
|
||||
- `Vertex AI Feature Store`
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Create featurestore, entity type, and feature resources.
|
||||
- Import feature data into `Vertex AI Feature Store` resource.
|
||||
- Serve online prediction requests using the imported features.
|
||||
- Access imported features in offline jobs, such as training jobs.
|
||||
|
||||
</blockquote>
|
||||
|
||||
### Vertex Model Monitoring
|
||||
|
||||
[Monitoring drift detection in online serving](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model_monitoring/model_monitoring.ipynb)
|
||||
|
||||
<blockquote>
|
||||
In this notebook, you learn to use the `Vertex AI Model Monitoring` service to detect drift and anomalies in prediction requests from a deployed `Vertex AI Model` resource.
|
||||
|
||||
This tutorial uses the following Google Cloud ML services:
|
||||
|
||||
- `Vertex AI Model Monitoring`
|
||||
- `Vertex AI Prediction`
|
||||
- `Vertex AI Model` resource
|
||||
- `Vertex AI Endpoint` resource
|
||||
|
||||
The steps performed include:
|
||||
- Track training parameters and prediction metrics for a custom training job.
|
||||
|
||||
- Upload a pre-trained model as a `Vertex AI Model` resource.
|
||||
- Create an `Vertex AI Endpoint` resource.
|
||||
- Deploy the `Model` resource to the `Endpoint` resource.
|
||||
- Configure the `Endpoint` resource for model monitoring.
|
||||
- Generate synthetic prediction requests.
|
||||
- Understand how to interpret the statistics, visualizations, other data reported by the model monitoring feature.
|
||||
</blockquote>
|
||||
|
||||
### Vertex ML Metadata
|
||||
|
||||
[Tracking hyperparameters and metrics in custom training job](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-custom-jobs.ipynb)
|
||||
|
||||
<blockquote>
|
||||
In this notebook, you learn how to use `Vertex ML Metadata` to track training parameters and evaluation metrics.
|
||||
|
||||
This tutorial uses the following Google Cloud ML services:
|
||||
|
||||
- `Vertex ML Metadata`
|
||||
- `Vertex AI Experiments`
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Track parameters and metrics for a `Vertex AI` custom trained model.
|
||||
- Extract and perform analysis for all parameters and metrics within an Experiment.
|
||||
</blockquote>
|
||||
|
||||
[Tracking hyperparameters and metrics in locally trained job](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-locally-trained-models.ipynb)
|
||||
|
||||
<blockquote>
|
||||
In this notebook, you learn how to use `Vertex ML Metadata` to track training parameters and evaluation metrics.
|
||||
|
||||
This tutorial uses the following Google Cloud ML services:
|
||||
|
||||
- `Vertex ML Metadata`
|
||||
- `Vertex AI Experiments`
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Track parameters and metrics for a locally trained model.
|
||||
- Extract and perform analysis for all parameters and metrics within an Experiment.
|
||||
</blockquote>
|
||||
|
||||
### Vertex AI Pipelines
|
||||
|
||||
[Creating Python function KFP components](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/lightweight_functions_component_io_kfp.ipynb)
|
||||
|
||||
<blockquote>
|
||||
In this tutorial, you learn to use the KFP SDK to build lightweight Python function-based components, and then you learn to use `Vertex AI Pipelines` to execute the pipeline.
|
||||
|
||||
This tutorial uses the following Google Cloud ML services:
|
||||
|
||||
- `Vertex AI Pipelines`
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Build Python function-based KFP components.
|
||||
- Construct a KFP pipeline.
|
||||
- Pass Artifacts and parameters between components, both by path reference and by value.
|
||||
- Use the kfp.dsl.importer method.
|
||||
- Compile the KFP pipeline.
|
||||
- Execute the KFP pipeline using `Vertex AI Pipelines`
|
||||
</blockquote>
|
||||
|
||||
[AutoML image classification model pipeline](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/google_cloud_pipeline_components_automl_images.ipynb)
|
||||
|
||||
<blockquote>
|
||||
In this tutorial, you learn to use `Vertex AI Pipelines` and `Google Cloud Pipeline Components` to build an `AutoML` image classification model.
|
||||
|
||||
This tutorial uses the following Google Cloud ML services:
|
||||
|
||||
- `Vertex AI Pipelines`
|
||||
- `Google Cloud Pipeline Components`
|
||||
- `Vertex AutoML`
|
||||
- `Vertex AI Model` resource
|
||||
- `Vertex AI Endpoint` resource
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Create a KFP pipeline:
|
||||
- Create a `Vertex AI Dataset` resource.
|
||||
- Train an `AutoML` image classification `Model` resource.
|
||||
- Create an `Endpoint` resource.
|
||||
- Deploys the `Model` resource to the `Endpoint` resource.
|
||||
- Compile the KFP pipeline.
|
||||
- Execute the KFP pipeline using `Vertex AI Pipelines`
|
||||
</blockquote>
|
||||
|
||||
[AutoML tabular classification model pipeline](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/automl_tabular_classification_beans.ipynb)
|
||||
|
||||
<blockquote>
|
||||
In this tutorial, you learn to use `Vertex AI Pipelines` and `Google Cloud Pipeline Components` to build an AutoML tabular classification model.
|
||||
|
||||
This tutorial uses the following Google Cloud ML services:
|
||||
|
||||
- `Vertex AI Pipelines`
|
||||
- `Google Cloud Pipeline Components`
|
||||
- `Vertex AutoML`
|
||||
- `Vertex AI Model` resource
|
||||
- `Vertex AI Endpoint` resource
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Create a KFP pipeline:
|
||||
- Create a `Dataset` resource.
|
||||
- Train an `AutoML` tabular classification `Model` resource.
|
||||
- Create an `Endpoint` resource.
|
||||
- Deploys the `Model` resource to the `Endpoint` resource.
|
||||
- Compile the KFP pipeline.
|
||||
- Execute the KFP pipeline using `Vertex AI Pipelines`
|
||||
</blockquote>
|
||||
|
||||
[AutoML tabular regression model pipeline](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/google_cloud_pipeline_components_automl_tabular.ipynb)
|
||||
|
||||
<blockquote>
|
||||
In this tutorial, you learn to use `Vertex AI Pipelines` and `Google Cloud Pipeline Components` to build an `AutoML` tabular regression model.
|
||||
|
||||
This tutorial uses the following Google Cloud ML services:
|
||||
|
||||
- `Vertex AI Pipelines`
|
||||
- `Google Cloud Pipeline Components`
|
||||
- `Vertex AutoML`
|
||||
- `Vertex AI Model` resource
|
||||
- `Vertex AI Endpoint` resource
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Create a KFP pipeline:
|
||||
- Create a `Dataset` resource.
|
||||
- Train an `AutoML` tabular regression `Model` resource.
|
||||
- Create an `Endpoint` resource.
|
||||
- Deploys the `Model` resource to the `Endpoint` resource.
|
||||
- Compile the KFP pipeline.
|
||||
- Execute the KFP pipeline using `Vertex AI Pipelines`
|
||||
</blockquote>
|
||||
|
||||
[AutoML text classification model pipeline](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/google_cloud_pipeline_components_automl_text.ipynb)
|
||||
|
||||
<blockquote>
|
||||
In this tutorial, you learn to use `Vertex AI Pipelines` and `Google Cloud Pipeline Components` to build an `AutoML` text classification model.
|
||||
|
||||
This tutorial uses the following Google Cloud ML services:
|
||||
|
||||
- `Vertex AI Pipelines`
|
||||
- `Google Cloud Pipeline Components`
|
||||
- `Vertex AutoML`
|
||||
- `Vertex AI Model` resource
|
||||
"- `Vertex AI Endpoint` resource
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Create a KFP pipeline:
|
||||
- Create a `Dataset` resource.
|
||||
- Train an AutoML text classification `Model` resource.
|
||||
- Create an `Endpoint` resource.
|
||||
- Deploys the `Model` resource to the `Endpoint` resource.
|
||||
- Compile the KFP pipeline.
|
||||
- Execute the KFP pipeline using `Vertex AI Pipelines`
|
||||
</blockquote>
|
||||
|
||||
[Custom training and batch prediction using prebuilt components pipeline](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/custom_model_training_and_batch_prediction.ipynb)
|
||||
|
||||
<blockquote>
|
||||
In this tutorial, you learn to use `Vertex AI Pipelines` and `Google Cloud Pipeline Components` to build a custom model.
|
||||
|
||||
This tutorial uses the following Google Cloud ML services:
|
||||
|
||||
- `Vertex AI Pipelines`
|
||||
- `Google Cloud Pipeline Components`
|
||||
- `Vertex AI Training`
|
||||
- `Vertex AI Model` resource
|
||||
- `Vertex AI Endpoint` resource
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Create a KFP pipeline:
|
||||
- Train a custom model.
|
||||
- Upload the trained model as a `Model` resource.
|
||||
- Create an `Endpoint` resource.
|
||||
- Deploy the `Model` resource to the `Endpoint` resource.
|
||||
- Make a batch prediction request.
|
||||
</blockquote>
|
||||
|
||||
[Custom training using prebuilt and custom components pipeline](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/google_cloud_pipeline_components_model_train_upload_deploy.ipynb)
|
||||
|
||||
<blockquote>
|
||||
In this tutorial, you learn to use `Vertex AI Pipelines` and `Google Cloud Pipeline Components` to build and deploy a custom model.
|
||||
|
||||
This tutorial uses the following Google Cloud ML services:
|
||||
|
||||
- `Vertex AI Pipelines`
|
||||
- `Google Cloud Pipeline Components`
|
||||
- `Vertex AI Training`
|
||||
- `Vertex AI Model` resource
|
||||
- `Vertex AI Endpoint` resource
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Create a KFP pipeline:
|
||||
- Train a custom model.
|
||||
- Uploads the trained model as a `Model` resource.
|
||||
- Creates an `Endpoint` resource.
|
||||
- Deploys the `Model` resource to the `Endpoint` resource.
|
||||
- Compile the KFP pipeline.
|
||||
- Execute the KFP pipeline using `Vertex AI Pipelines`
|
||||
</blockquote>
|
||||
|
||||
[Introduction to control flow in pipelines](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/control_flow_kfp.ipynb)
|
||||
|
||||
<blockquote>
|
||||
In this tutorial, you use the KFP SDK to build pipelines that use loops and conditionals, including nested examples.
|
||||
|
||||
This tutorial uses the following Google Cloud ML services:
|
||||
|
||||
- `Vertex AI Pipelines`
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Create a KFP pipeline:
|
||||
- Use control flow components
|
||||
- Compile the KFP pipeline.
|
||||
- Execute the KFP pipeline using `Vertex AI Pipelines`
|
||||
</blockquote>
|
||||
|
||||
[Introduction to KFP components and pipelines](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/pipelines_intro_kfp.ipynb)
|
||||
|
||||
<blockquote>
|
||||
In this tutorial, you use the KFP SDK to build pipelines.
|
||||
|
||||
This tutorial uses the following Google Cloud ML services:
|
||||
|
||||
- `Vertex AI Pipelines`
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Define and compile a `Vertex AI` pipeline.
|
||||
- Schedule a recurring pipeline run.
|
||||
- Specify which service account to use for a pipeline run.
|
||||
</blockquote>
|
||||
|
||||
### Vertex AI Vizier
|
||||
|
||||
[Using Vizier for multi-objective study](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/vizier/gapic-vizier-multi-objective-optimization.ipynb)
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -1,177 +0,0 @@
|
||||
|
||||
[AutoML Tabular Training and Prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/automl-tabular-classification.ipynb)
|
||||
|
||||
Learn how to train and make predictions on an AutoML model based on a tabular dataset.
|
||||
|
||||
The steps performed include the following:
|
||||
|
||||
- Create a Vertex AI model training job.
|
||||
- Train an AutoML Tabular model.
|
||||
- Deploy the `Model` resource to a serving `Endpoint` resource.
|
||||
- Make a prediction by sending data.
|
||||
- Undeploy the `Model` resource.
|
||||
|
||||
[Create, train, and deploy an AutoML text classification model](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/automl-text-classification.ipynb)
|
||||
|
||||
Learn how to use `AutoML` to train a text classification model.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
* Create a `Vertex AI Dataset`.
|
||||
* Train an `AutoML` text classification `Model` resource.
|
||||
* Obtain the evaluation metrics for the `Model` resource.
|
||||
* Create an `Endpoint` resource.
|
||||
* Deploy the `Model` resource to the `Endpoint` resource.
|
||||
* Make an online prediction
|
||||
* Make a batch prediction
|
||||
|
||||
[AutoML training video classification model for batch prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_video_classification_batch.ipynb)
|
||||
|
||||
Learn how to create an AutoML video classification model from a Python script, and then do a batch prediction using the Vertex AI SDK.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Create a Vertex `Dataset` resource.
|
||||
- Train the model.
|
||||
- View the model evaluation.
|
||||
- Make a batch prediction.
|
||||
|
||||
|
||||
* Prediction Service: Does an on-demand prediction for the entire set of instances (i.e., one or more data items) and returns the results in real-time.
|
||||
|
||||
* Batch Prediction Service: Does a queued (batch) prediction for the entire set of instances in the background and stores the results in a Cloud Storage bucket when ready.
|
||||
|
||||
[AutoML training text entity extraction model for online prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_text_entity_extraction_online.ipynb)
|
||||
|
||||
Learn how to create an AutoML text entity extraction model and deploy for online prediction from a Python script using the Vertex SDK.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Create a Vertex `Dataset` resource.
|
||||
- Train the model.
|
||||
- View the model evaluation.
|
||||
- Deploy the `Model` resource to a serving `Endpoint` resource.
|
||||
- Make a prediction.
|
||||
- Undeploy the `Model`.
|
||||
|
||||
[AutoML tabular forecasting model for batch prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_tabular_forecasting_batch.ipynb)
|
||||
|
||||
Learn how to create an `AutoML` tabular forecasting model from a Python script, and then do a batch prediction using the Vertex AI SDK.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Create a `Vertex AI Dataset` resource.
|
||||
- Train an `AutoML` tabular forecasting `Model` resource.
|
||||
- Obtain the evaluation metrics for the `Model` resource.
|
||||
- Make a batch prediction.
|
||||
|
||||
[AutoML training image object detection model for batch prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_image_object_detection_batch.ipynb)
|
||||
|
||||
Learn how to create an AutoML image object detection model from a Python script, and then do a batch prediction using the Vertex AI SDK.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Create a Vertex `Dataset` resource.
|
||||
- Train the model.
|
||||
- View the model evaluation.
|
||||
- Make a batch prediction.
|
||||
|
||||
|
||||
* Prediction Service: Does an on-demand prediction for the entire set of instances (i.e., one or more data items) and returns the results in real-time.
|
||||
|
||||
* Batch Prediction Service: Does a queued (batch) prediction for the entire set of instances in the background and stores the results in a Cloud Storage bucket when ready.
|
||||
|
||||
[AutoML training video action recognition model for batch prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_video_action_recognition_batch.ipynb)
|
||||
|
||||
Learn how to create an AutoML video action recognition model from a Python script, and then do a batch prediction using the Vertex AI SDK.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Create a Vertex `Dataset` resource.
|
||||
- Train the model.
|
||||
- View the model evaluation.
|
||||
- Make a batch prediction.
|
||||
|
||||
|
||||
* Prediction Service: Does an on-demand prediction for the entire set of instances (i.e., one or more data items) and returns the results in real-time.
|
||||
|
||||
* Batch Prediction Service: Does a queued (batch) prediction for the entire set of instances in the background and stores the results in a Cloud Storage bucket when ready.
|
||||
|
||||
[AutoML Tabular Pipeline](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/automl_tabular_on_vertex_pipelines.ipynb)
|
||||
|
||||
Learn how to create two regression models using [Vertex Pipelines](https://cloud.
|
||||
|
||||
The steps performed are:
|
||||
|
||||
- Create a training pipeline that reduces the search space from the default to save time.
|
||||
- Create a training pipeline that reuses the architecture search results from the previous pipeline to save time.
|
||||
|
||||
[AutoML training text sentiment analysis model for online prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_text_sentiment_analysis_online.ipynb)
|
||||
|
||||
Learn how to create an AutoML text sentiment analysis model and deploy for online prediction from a Python script using the Vertex SDK.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Create a Vertex `Dataset` resource.
|
||||
- Create a training job for the model.
|
||||
- View the model evaluation.
|
||||
- Deploy the `Model` resource to a serving `Endpoint` resource.
|
||||
- Make a prediction.
|
||||
- Undeploy the `Model`.
|
||||
|
||||
[Compare Vertex AI Forecasting and BigQuery ML ARIMA_PLUS](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/automl_forecasting_bqml_arima_plus_comparison.ipynb)
|
||||
|
||||
Learn how to create an BQML ARIMA_PLUS model using a training [Vertex AI Pipeline](https://cloud.
|
||||
|
||||
The steps performed are:
|
||||
|
||||
- Train the BQML ARIMA_PLUS model.
|
||||
- View BQML model evaluation.
|
||||
- Make a batch prediction with the BQML model.
|
||||
- Create a Vertex AI `Dataset` resource.
|
||||
- Train the Vertex AI Forecasting model.
|
||||
- View the Model evaluation.
|
||||
- Make a batch prediction with the Model.
|
||||
|
||||
|
||||
[AutoML training tabular regression model for online prediction using BigQuery](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/master/notebooks/official/automl/sdk_automl_tabular_regression_online_bq.ipynb)
|
||||
|
||||
Learn how to create an AutoML tabular regression model and deploy for online prediction from a Python script using the Vertex AI SDK.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Create a Vertex `Dataset` resource.
|
||||
- Train the model.
|
||||
- View the model evaluation.
|
||||
- Deploy the `Model` resource to a serving `Endpoint` resource.
|
||||
- Make a prediction.
|
||||
- Undeploy the `Model`.
|
||||
|
||||
[AutoML training video object tracking model for batch prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_video_object_tracking_batch.ipynb)
|
||||
|
||||
Learn how to create an AutoML video object tracking model from a Python script, and then do a batch prediction using the Vertex SDK.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Create a Vertex `Dataset` resource.
|
||||
- Train the model.
|
||||
- View the model evaluation.
|
||||
- Make a batch prediction.
|
||||
|
||||
|
||||
* Prediction Service: Does an on-demand prediction for the entire set of instances (i.e., one or more data items) and returns the results in real-time.
|
||||
|
||||
* Batch Prediction Service: Does a queued (batch) prediction for the entire set of instances in the background and stores the results in a Cloud Storage bucket when ready.
|
||||
|
||||
[AutoML training tabular regression model for batch prediction using BigQuery](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_tabular_regression_batch_bq.ipynb)
|
||||
|
||||
Learn how to create an AutoML tabular regression model and deploy it for batch prediction using the Vertex AI SDK for Python.
|
||||
|
||||
The steps performed include:
|
||||
|
||||
- Create a Vertex AI `Dataset` resource.
|
||||
- Train the model.
|
||||
- View the model evaluation.
|
||||
- Deploy the `Model` resource to a serving `Endpoint` resource.
|
||||
- Make a prediction.
|
||||
- Undeploy the `Model`.
|
||||
@@ -29,41 +29,20 @@
|
||||
"id": "JAPoU8Sm5E6e"
|
||||
},
|
||||
"source": [
|
||||
"# Vertex AI SDK for Python: AutoML Tabular Training and Prediction\n",
|
||||
"\n",
|
||||
"<table align=\"left\">\n",
|
||||
"\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/automl-tabular-classification.ipynb\">\n",
|
||||
" <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/official/automl/automl-tabular-classification.ipynb\"\">\n",
|
||||
" <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/automl-tabular-classification.ipynb\">\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/official/automl/automl-tabular-classification.ipynb\">\n",
|
||||
" <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/automl/automl-tabular-classification.ipynb\">\n",
|
||||
" <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
|
||||
" Open in Vertex AI Workbench\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
"</table>\n",
|
||||
"<br/><br/><br/>"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "411c6c769293"
|
||||
},
|
||||
"source": [
|
||||
"## Overview\n",
|
||||
"\n",
|
||||
"This tutorial demonstrates how to use the Vertex AI Python client library to train and deploy a tabular classification model for online prediction.\n",
|
||||
"\n",
|
||||
"**Note**: you may incur charges for training, prediction, storage, or usage of other GCP products in connection with testing this SDK."
|
||||
"</table>"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -72,9 +51,24 @@
|
||||
"id": "tvgnzT1CKxrO"
|
||||
},
|
||||
"source": [
|
||||
"# Vertex AI SDK for Python: AutoML Tabular Training and Prediction\n",
|
||||
"\n",
|
||||
"To use this Colaboratory notebook, you copy the notebook to your own Google Drive and open it with Colaboratory (or Colab). You can run each step, or cell, and see its results. To run a cell, use Shift+Enter. Colab automatically displays the return value of the last line in each cell. For more information about running notebooks in Colab, see the Colab welcome page.\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"## Overview\n",
|
||||
"\n",
|
||||
"This tutorial demonstrates how to use the Vertex AI Python client library to train and deploy a tabular classification model for online prediction.\n",
|
||||
"\n",
|
||||
"**Note**: you may incur charges for training, prediction, storage, or usage of other GCP products in connection with testing this SDK.\n",
|
||||
"\n",
|
||||
"### Dataset\n",
|
||||
"\n",
|
||||
"The dataset we are using is the PetFinder Dataset, available locally in Colab. To learn more about this dataset, visit https://www.kaggle.com/c/petfinder-adoption-prediction.\n",
|
||||
"\n",
|
||||
"### Objective\n",
|
||||
"\n",
|
||||
"In this tutorial, you learn how to train and make predictions on an AutoML model based on a tabular dataset. Alternatively, you can train and make predictions on models by using the `gcloud` command-line tool or by using the online Cloud Console.\n",
|
||||
"This notebook demonstrates, using the Vertex AI Python client library, how to train and make predictions on an AutoML model based on a tabular dataset. Alternatively, you can train and make predictions on models by using the `gcloud` command-line tool or by using the online Cloud Console.\n",
|
||||
"\n",
|
||||
"The steps performed include the following:\n",
|
||||
"\n",
|
||||
@@ -82,26 +76,8 @@
|
||||
"- Train an AutoML Tabular model.\n",
|
||||
"- Deploy the `Model` resource to a serving `Endpoint` resource.\n",
|
||||
"- Make a prediction by sending data.\n",
|
||||
"- Undeploy the `Model` resource."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "d87e05416046"
|
||||
},
|
||||
"source": [
|
||||
"### Dataset\n",
|
||||
"- Undeploy the `Model` resource.\n",
|
||||
"\n",
|
||||
"The dataset we are using is the PetFinder Dataset, available locally in Colab. To learn more about this dataset, visit https://www.kaggle.com/c/petfinder-adoption-prediction."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "5e2eba58ad71"
|
||||
},
|
||||
"source": [
|
||||
"### Costs \n",
|
||||
"\n",
|
||||
"This tutorial uses billable components of Google Cloud:\n",
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user