mirror of
https://github.com/GoogleCloudPlatform/vertex-ai-samples.git
synced 2026-09-27 15:42:05 +00:00
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Andrew FerlitschandGitHub
|
06daeb1575 | ||
|
Andrew Ferlitsch
|
2074fb56a9 |
@@ -38,7 +38,7 @@ from utils import NotebookProcessors, util
|
||||
|
||||
# A buffer so that workers finish before the orchestrating job
|
||||
WORKER_TIMEOUT_BUFFER_IN_SECONDS: int = 60 * 60
|
||||
PYTHON_VERSION = "3.9" # Set default python version
|
||||
PYTHON_VERSION = "3.9" # Set default python version
|
||||
|
||||
|
||||
def format_timedelta(delta: datetime.timedelta) -> str:
|
||||
@@ -102,7 +102,6 @@ def _process_notebook(
|
||||
"VPC_NETWORK": variable_vpc_network,
|
||||
},
|
||||
)
|
||||
unique_strings_preprocessor = NotebookProcessors.UniqueStringsPreprocessor()
|
||||
|
||||
# Use no-execute preprocessor
|
||||
(
|
||||
@@ -128,15 +127,13 @@ def _get_notebook_python_version(notebook_path: str) -> str:
|
||||
src = file.read()
|
||||
nb_json = json.loads(src)
|
||||
|
||||
# Iterate over the cells in the ipynb
|
||||
for cell in nb_json["cells"]:
|
||||
if cell["cell_type"] == "markdown":
|
||||
markdown = str.join("", cell["source"])
|
||||
#Iterate over the cells in the ipynb
|
||||
for cell in nb_json['cells']:
|
||||
if cell['cell_type'] == 'markdown':
|
||||
markdown = str.join('', cell['source'])
|
||||
|
||||
# Look for the python version specification pattern
|
||||
re_match = re.search(
|
||||
"python version = (\d\.\d)", markdown, flags=re.IGNORECASE
|
||||
)
|
||||
re_match = re.search('python version = (\d\.\d)', markdown, flags=re.IGNORECASE)
|
||||
if re_match:
|
||||
# get the version number
|
||||
python_version = re_match.group(1)
|
||||
@@ -204,9 +201,7 @@ def process_and_execute_notebook(
|
||||
operation = None
|
||||
try:
|
||||
# Get the python version for running the notebook if specified
|
||||
notebook_exec_python_version = _get_notebook_python_version(
|
||||
notebook_path=notebook
|
||||
)
|
||||
notebook_exec_python_version = _get_notebook_python_version(notebook_path=notebook)
|
||||
print(f"Running notebook with python {notebook_exec_python_version}")
|
||||
|
||||
# Pre-process notebook by substituting variable names
|
||||
@@ -235,7 +230,7 @@ def process_and_execute_notebook(
|
||||
private_pool_id=private_pool_id,
|
||||
private_pool_region=variable_region,
|
||||
timeout_in_seconds=timeout_in_seconds,
|
||||
python_version=notebook_exec_python_version,
|
||||
python_version=notebook_exec_python_version
|
||||
)
|
||||
|
||||
operation_metadata = BuildOperationMetadata(mapping=operation.metadata)
|
||||
@@ -448,7 +443,7 @@ def process_and_execute_notebooks(
|
||||
result.log_url,
|
||||
result.output_uri,
|
||||
result.output_uri_web,
|
||||
result.logs_bucket,
|
||||
result.logs_bucket
|
||||
]
|
||||
for result in results_sorted
|
||||
],
|
||||
@@ -459,34 +454,34 @@ def process_and_execute_notebooks(
|
||||
"log_url",
|
||||
"output_uri",
|
||||
"output_uri_web",
|
||||
"logs_bucket",
|
||||
"logs_bucket"
|
||||
],
|
||||
)
|
||||
)
|
||||
|
||||
if len(notebooks) == 1:
|
||||
print("=" * 100)
|
||||
print("The notebook execution build log:\n")
|
||||
print("=" * 100)
|
||||
print("="*100)
|
||||
print("The notebook execution build log:\n")
|
||||
print("="*100)
|
||||
|
||||
build_id = results_sorted[0].build_id
|
||||
logs_bucket_name = (results_sorted[0].logs_bucket).removeprefix("gs://")
|
||||
log_file_name = f"log-{build_id}.txt"
|
||||
build_id = results_sorted[0].build_id
|
||||
logs_bucket_name = (results_sorted[0].logs_bucket).removeprefix("gs://")
|
||||
log_file_name = f"log-{build_id}.txt"
|
||||
|
||||
log_contents = util.download_blob_into_memory(
|
||||
bucket_name=logs_bucket_name,
|
||||
blob_name=log_file_name,
|
||||
download_as_text=True,
|
||||
log_contents = util.download_blob_into_memory(
|
||||
bucket_name=logs_bucket_name,
|
||||
blob_name=log_file_name,
|
||||
download_as_text=True
|
||||
)
|
||||
|
||||
# Remove extra steps from the log
|
||||
match = re.search("starting Step #4", log_contents, flags=re.IGNORECASE)
|
||||
# Remove extra steps from the log
|
||||
match = re.search("starting Step #4", log_contents, flags=re.IGNORECASE)
|
||||
|
||||
if match is not None:
|
||||
match_index = match.span()[0]
|
||||
print(log_contents[match_index:])
|
||||
else:
|
||||
print(log_contents)
|
||||
if match is not None:
|
||||
match_index = match.span()[0]
|
||||
print(log_contents[match_index:])
|
||||
else:
|
||||
print(log_contents)
|
||||
|
||||
print("\n=== END RESULTS===\n")
|
||||
|
||||
|
||||
@@ -2,4 +2,5 @@ notebooks/official/vizier/gapic-vizier-multi-objective-optimization.ipynb
|
||||
notebooks/official/pipelines/lightweight_functions_component_io_kfp.ipynb
|
||||
notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-locally-trained-models.ipynb
|
||||
notebooks/official/custom/custom-tabular-bq-managed-dataset.ipynb
|
||||
notebooks/official/matching_engine/sdk_matching_engine_for_indexing.ipynb
|
||||
.cloud-build/tests/python_version_test.ipynb
|
||||
|
||||
@@ -14,8 +14,6 @@
|
||||
# limitations under the License.
|
||||
|
||||
from typing import Dict
|
||||
import random
|
||||
import string
|
||||
|
||||
from nbconvert.preprocessors import Preprocessor
|
||||
|
||||
@@ -65,30 +63,3 @@ class UpdateVariablesPreprocessor(Preprocessor):
|
||||
executable_cells.append(cell)
|
||||
notebook.cells = executable_cells
|
||||
return notebook, resources
|
||||
|
||||
|
||||
# Generate a uuid of a specifed length
|
||||
def generate_uuid(length: int = 8) -> str:
|
||||
return "".join(random.choices(string.ascii_lowercase + string.digits, k=length))
|
||||
|
||||
|
||||
class UniqueStringsPreprocessor(Preprocessor):
|
||||
# A preprocessor that replaces strings that end with "-unique" with a uuid.
|
||||
|
||||
@staticmethod
|
||||
def update_unique_strings(content: str):
|
||||
# Replace strings that end with "-unique" with a uuid.
|
||||
|
||||
return content.replace('-unique"', f'-{generate_uuid()}"')
|
||||
|
||||
def preprocess(self, notebook, resources=None):
|
||||
executable_cells = []
|
||||
for cell in notebook.cells:
|
||||
if cell.cell_type == "code":
|
||||
cell.source = self.update_unique_strings(
|
||||
content=cell.source,
|
||||
)
|
||||
|
||||
executable_cells.append(cell)
|
||||
notebook.cells = executable_cells
|
||||
return notebook, resources
|
||||
|
||||
@@ -40,3 +40,65 @@ def get_updated_value(content: str, variable_name: str, variable_value: str) ->
|
||||
content,
|
||||
flags=re.M,
|
||||
)
|
||||
|
||||
|
||||
def test_update_value():
|
||||
new_content = get_updated_value(
|
||||
content='asdf\nPROJECT_ID = "[your-project-id]" #@param {type:"string"} \nasdf',
|
||||
variable_name="PROJECT_ID",
|
||||
variable_value="sample-project",
|
||||
)
|
||||
assert (
|
||||
new_content
|
||||
== 'asdf\nPROJECT_ID = "sample-project" #@param {type:"string"} \nasdf'
|
||||
)
|
||||
|
||||
|
||||
def test_update_value_single_quotes():
|
||||
new_content = get_updated_value(
|
||||
content="PROJECT_ID = '[your-project-id]'",
|
||||
variable_name="PROJECT_ID",
|
||||
variable_value="sample-project",
|
||||
)
|
||||
assert new_content == "PROJECT_ID = 'sample-project'"
|
||||
|
||||
|
||||
def test_update_value_avoidance():
|
||||
new_content = get_updated_value(
|
||||
content="PROJECT_ID = shell_output[0] ",
|
||||
variable_name="PROJECT_ID",
|
||||
variable_value="sample-project",
|
||||
)
|
||||
assert new_content == "PROJECT_ID = shell_output[0] "
|
||||
|
||||
|
||||
def test_region():
|
||||
new_content = get_updated_value(
|
||||
content='REGION = "[your-region]" # @param {type:"string"}',
|
||||
variable_name="REGION",
|
||||
variable_value="us-central1",
|
||||
)
|
||||
assert new_content == 'REGION = "us-central1" # @param {type:"string"}'
|
||||
|
||||
|
||||
def test_region_equal_equals_ignore():
|
||||
# Tests that == is ignored
|
||||
new_content = get_updated_value(
|
||||
content='REGION == "[your-region]" # @param {type:"string"}',
|
||||
variable_name="REGION",
|
||||
variable_value="us-central1",
|
||||
)
|
||||
assert new_content == 'REGION == "[your-region]" # @param {type:"string"}'
|
||||
|
||||
|
||||
def test_service_account():
|
||||
# Tests that == is ignored
|
||||
new_content = get_updated_value(
|
||||
content='SERVICE_ACCOUNT = "[your-service-account]" # @param {type:"string"}',
|
||||
variable_name="SERVICE_ACCOUNT",
|
||||
variable_value="12345-compute@developer.gserviceaccount.com",
|
||||
)
|
||||
assert (
|
||||
new_content
|
||||
== 'SERVICE_ACCOUNT = "12345-compute@developer.gserviceaccount.com" # @param {type:"string"}'
|
||||
)
|
||||
|
||||
@@ -1,14 +0,0 @@
|
||||
from utils import NotebookProcessors
|
||||
|
||||
|
||||
def test_update_value():
|
||||
# Test that the content was updated
|
||||
preprocessor = NotebookProcessors.UniqueStringsPreprocessor()
|
||||
|
||||
content = 'PROJECT_ID = "your-project-id-unique"'
|
||||
|
||||
new_content = preprocessor.update_unique_strings(content)
|
||||
|
||||
assert new_content != content
|
||||
assert new_content.startswith('PROJECT_ID = "your-project-id-')
|
||||
assert new_content.endswith('"')
|
||||
@@ -1,63 +0,0 @@
|
||||
from utils import UpdateNotebookVariables
|
||||
|
||||
|
||||
def test_update_value():
|
||||
new_content = UpdateNotebookVariables.get_updated_value(
|
||||
content='asdf\nPROJECT_ID = "[your-project-id]" #@param {type:"string"} \nasdf',
|
||||
variable_name="PROJECT_ID",
|
||||
variable_value="sample-project",
|
||||
)
|
||||
assert (
|
||||
new_content
|
||||
== 'asdf\nPROJECT_ID = "sample-project" #@param {type:"string"} \nasdf'
|
||||
)
|
||||
|
||||
|
||||
def test_update_value_single_quotes():
|
||||
new_content = UpdateNotebookVariables.get_updated_value(
|
||||
content="PROJECT_ID = '[your-project-id]'",
|
||||
variable_name="PROJECT_ID",
|
||||
variable_value="sample-project",
|
||||
)
|
||||
assert new_content == "PROJECT_ID = 'sample-project'"
|
||||
|
||||
|
||||
def test_update_value_avoidance():
|
||||
new_content = UpdateNotebookVariables.get_updated_value(
|
||||
content="PROJECT_ID = shell_output[0] ",
|
||||
variable_name="PROJECT_ID",
|
||||
variable_value="sample-project",
|
||||
)
|
||||
assert new_content == "PROJECT_ID = shell_output[0] "
|
||||
|
||||
|
||||
def test_region():
|
||||
new_content = UpdateNotebookVariables.get_updated_value(
|
||||
content='REGION = "[your-region]" # @param {type:"string"}',
|
||||
variable_name="REGION",
|
||||
variable_value="us-central1",
|
||||
)
|
||||
assert new_content == 'REGION = "us-central1" # @param {type:"string"}'
|
||||
|
||||
|
||||
def test_region_equal_equals_ignore():
|
||||
# Tests that == is ignored
|
||||
new_content = UpdateNotebookVariables.get_updated_value(
|
||||
content='REGION == "[your-region]" # @param {type:"string"}',
|
||||
variable_name="REGION",
|
||||
variable_value="us-central1",
|
||||
)
|
||||
assert new_content == 'REGION == "[your-region]" # @param {type:"string"}'
|
||||
|
||||
|
||||
def test_service_account():
|
||||
# Tests that == is ignored
|
||||
new_content = UpdateNotebookVariables.get_updated_value(
|
||||
content='SERVICE_ACCOUNT = "[your-service-account]" # @param {type:"string"}',
|
||||
variable_name="SERVICE_ACCOUNT",
|
||||
variable_value="12345-compute@developer.gserviceaccount.com",
|
||||
)
|
||||
assert (
|
||||
new_content
|
||||
== 'SERVICE_ACCOUNT = "12345-compute@developer.gserviceaccount.com" # @param {type:"string"}'
|
||||
)
|
||||
@@ -6,4 +6,3 @@
|
||||
/sklearn_text_classification_from_script_using_vertex_sdk @maxhardt
|
||||
/pluto_on_workbench @wkharold
|
||||
/cpr-examples @samthrasher
|
||||
/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines @Ark-kun
|
||||
|
||||
-83
@@ -1,83 +0,0 @@
|
||||
name: Train tabular classification logistic regression model using Scikit learn pipeline
|
||||
metadata:
|
||||
annotations:
|
||||
author: Alexey Volkov <alexey.volkov@ark-kun.com>
|
||||
canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_classification_logistic_regression_model_using_Scikit_learn_and_import_to_Vertex_AI/pipeline.component.yaml
|
||||
sdk: https://cloud-pipelines.net/pipeline-editor/
|
||||
implementation:
|
||||
graph:
|
||||
tasks:
|
||||
Download from GCS:
|
||||
componentRef:
|
||||
digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":40,"width":180,"height":40}'
|
||||
Select columns using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: Data
|
||||
taskId: Download from GCS
|
||||
column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":140,"width":180,"height":54}'
|
||||
Fill all missing values using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Select columns using Pandas on CSV data
|
||||
type: CSV
|
||||
replacement_value: '0'
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":250,"width":180,"height":54}'
|
||||
Binarize column using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: d699afd4d7cae862708717cc160f4394ed0c04e536e9515923ef1e8865f01d44
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Fill all missing values using Pandas on CSV data
|
||||
type: CSV
|
||||
column_name: tips
|
||||
predicate: '> 0'
|
||||
new_column_name: class
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":380,"width":180,"height":54}'
|
||||
Train logistic regression model using scikit learn from CSV:
|
||||
componentRef:
|
||||
digest: a864625a822e4b1c8ef6fe4ae1454fd90f15438f70a6712bb4c30e0dda4d35b7
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/cb44b75c9c062fcc40c2b905b2024b4493dbc62b/components/ML_frameworks/Scikit_learn/Train_logistic_regression_model/from_CSV/component.yaml
|
||||
arguments:
|
||||
dataset:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Binarize column using Pandas on CSV data
|
||||
type: CSV
|
||||
label_column_name: class
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":510,"width":180,"height":70}'
|
||||
Upload Scikit learn pickle model to Google Cloud Vertex AI:
|
||||
componentRef:
|
||||
digest: 81c91c8d7d21ec97e0872f669d68bd89edea87279d703685db54aa94743bebcd
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Train logistic regression model using scikit learn from CSV
|
||||
type: ScikitLearnPickleModel
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":660,"width":180,"height":70}'
|
||||
outputValues: {}
|
||||
-73
@@ -1,73 +0,0 @@
|
||||
# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
|
||||
from kfp import components
|
||||
|
||||
# %% Loading components
|
||||
download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml")
|
||||
fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
|
||||
binarize_column_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml")
|
||||
train_logistic_regression_model_using_scikit_learn_from_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/cb44b75c9c062fcc40c2b905b2024b4493dbc62b/components/ML_frameworks/Scikit_learn/Train_logistic_regression_model/from_CSV/component.yaml")
|
||||
upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
|
||||
# %% Pipeline definition
|
||||
def train_tabular_classification_logistic_regression_model_using_Scikit_learn_pipeline():
|
||||
dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
|
||||
feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"] # Excluded "trip_total"
|
||||
label_column = "tips"
|
||||
# Deploying the model might incur additional costs over time
|
||||
deploy_model = False
|
||||
|
||||
classification_label_column = "class"
|
||||
all_columns = [label_column] + feature_columns
|
||||
|
||||
training_data = download_from_gcs_op(
|
||||
gcs_path=dataset_gcs_uri
|
||||
).outputs["Data"]
|
||||
|
||||
training_data = select_columns_using_Pandas_on_CSV_data_op(
|
||||
table=training_data,
|
||||
column_names=all_columns,
|
||||
).outputs["transformed_table"]
|
||||
|
||||
# Cleaning the NaN values.
|
||||
training_data = fill_all_missing_values_using_Pandas_on_CSV_data_op(
|
||||
table=training_data,
|
||||
replacement_value="0",
|
||||
#replacement_type_name="float",
|
||||
).outputs["transformed_table"]
|
||||
|
||||
classification_training_data = binarize_column_using_Pandas_on_CSV_data_op(
|
||||
table=training_data,
|
||||
column_name=label_column,
|
||||
predicate="> 0",
|
||||
new_column_name=classification_label_column,
|
||||
).outputs["transformed_table"]
|
||||
|
||||
model = train_logistic_regression_model_using_scikit_learn_from_CSV_op(
|
||||
dataset=classification_training_data,
|
||||
label_column_name=classification_label_column,
|
||||
# Optional:
|
||||
#penalty="l2",
|
||||
#solver="lbfgs",
|
||||
#max_iterations=100,
|
||||
#multi_class_mode="auto",
|
||||
#random_seed=0,
|
||||
).outputs["model"]
|
||||
|
||||
vertex_model_name = upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op(
|
||||
model=model,
|
||||
).outputs["model_name"]
|
||||
|
||||
# Deploying the model might incur additional costs over time
|
||||
if deploy_model:
|
||||
sklearn_vertex_endpoint_name = deploy_model_to_endpoint_op(
|
||||
model_name=vertex_model_name,
|
||||
).outputs["endpoint_name"]
|
||||
|
||||
pipeline_func = train_tabular_classification_logistic_regression_model_using_Scikit_learn_pipeline
|
||||
|
||||
# %% Pipeline submission
|
||||
if __name__ == '__main__':
|
||||
from google.cloud import aiplatform
|
||||
aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()
|
||||
-114
@@ -1,114 +0,0 @@
|
||||
name: Train tabular classification model using PyTorch pipeline
|
||||
metadata:
|
||||
annotations:
|
||||
author: Alexey Volkov <alexey.volkov@ark-kun.com>
|
||||
canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_classification_model_using_PyTorch_and_import_to_Vertex_AI/pipeline.component.yaml
|
||||
sdk: https://cloud-pipelines.net/pipeline-editor/
|
||||
implementation:
|
||||
graph:
|
||||
tasks:
|
||||
Download from GCS:
|
||||
componentRef:
|
||||
digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
|
||||
annotations:
|
||||
editor.position: '{"x":240,"y":40,"width":180,"height":40}'
|
||||
Select columns using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: Data
|
||||
taskId: Download from GCS
|
||||
column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
|
||||
annotations:
|
||||
editor.position: '{"x":240,"y":140,"width":180,"height":54}'
|
||||
Fill all missing values using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Select columns using Pandas on CSV data
|
||||
type: CSV
|
||||
replacement_value: '0'
|
||||
annotations:
|
||||
editor.position: '{"x":240,"y":250,"width":180,"height":54}'
|
||||
Create fully connected pytorch network:
|
||||
componentRef:
|
||||
digest: d03d8248fd358a0275ec33568ee7dd7dce576cc112b09dfafe2651e4d97e04a9
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1a2ef3eeb77bc278f33cad0dd29008ea2431e191/components/PyTorch/Create_fully_connected_network/component.yaml
|
||||
arguments:
|
||||
input_size: '7'
|
||||
hidden_layer_sizes: '[10]'
|
||||
activation_name: elu
|
||||
output_activation_name: sigmoid
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":360,"width":180,"height":54}'
|
||||
Binarize column using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: d699afd4d7cae862708717cc160f4394ed0c04e536e9515923ef1e8865f01d44
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Fill all missing values using Pandas on CSV data
|
||||
type: CSV
|
||||
column_name: tips
|
||||
predicate: ' > 0'
|
||||
new_column_name: class
|
||||
annotations:
|
||||
editor.position: '{"x":240,"y":360,"width":180,"height":54}'
|
||||
Train pytorch model from csv:
|
||||
componentRef:
|
||||
digest: 40f3185eb61e9727f41a4e0c05dd3d3b44bd802aa0f378cfc31756560033949a
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml
|
||||
arguments:
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Create fully connected pytorch network
|
||||
type: PyTorchScriptModule
|
||||
training_data:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Binarize column using Pandas on CSV data
|
||||
type: CSV
|
||||
label_column_name: class
|
||||
loss_function_name: binary_cross_entropy
|
||||
annotations:
|
||||
editor.position: '{"x":240,"y":490,"width":180,"height":40}'
|
||||
Create PyTorch Model Archive with base handler:
|
||||
componentRef:
|
||||
digest: 8298b5ee1b0f0879f893add4cf352c8dec7cf9e21bb9db134c91a2d046cdb0ec
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/46d51383e6554b7f3ab4fd8cf614d8c2b422fb22/components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml
|
||||
arguments:
|
||||
Model:
|
||||
taskOutput:
|
||||
outputName: trained_model
|
||||
taskId: Train pytorch model from csv
|
||||
type: PyTorchScriptModule
|
||||
Model name: model
|
||||
Model version: '1.0'
|
||||
annotations:
|
||||
editor.position: '{"x":240,"y":590,"width":180,"height":54}'
|
||||
Upload PyTorch model archive to Google Cloud Vertex AI:
|
||||
componentRef:
|
||||
digest: 4450212fae7b9001482aca7eb78b28413c205506eccf08a04e7754a8dfa99004
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
model_archive:
|
||||
taskOutput:
|
||||
outputName: Model archive
|
||||
taskId: Create PyTorch Model Archive with base handler
|
||||
type: PyTorchModelArchive
|
||||
annotations:
|
||||
editor.position: '{"x":240,"y":720,"width":180,"height":70}'
|
||||
outputValues: {}
|
||||
-95
@@ -1,95 +0,0 @@
|
||||
# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
|
||||
from kfp import components
|
||||
|
||||
# %% Loading components
|
||||
download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml")
|
||||
fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
|
||||
binarize_column_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml")
|
||||
create_fully_connected_pytorch_network_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/1a2ef3eeb77bc278f33cad0dd29008ea2431e191/components/PyTorch/Create_fully_connected_network/component.yaml")
|
||||
train_pytorch_model_from_csv_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml")
|
||||
create_pytorch_model_archive_with_base_handler_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/46d51383e6554b7f3ab4fd8cf614d8c2b422fb22/components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml")
|
||||
upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
|
||||
# %% Pipeline definition
|
||||
def train_tabular_classification_model_using_PyTorch_pipeline():
|
||||
dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
|
||||
feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"] # Excluded "trip_total"
|
||||
label_column = "tips"
|
||||
# Deploying the model might incur additional costs over time
|
||||
deploy_model = False
|
||||
|
||||
classification_label_column = "class"
|
||||
all_columns = [label_column] + feature_columns
|
||||
|
||||
training_data = download_from_gcs_op(
|
||||
gcs_path=dataset_gcs_uri
|
||||
).outputs["Data"]
|
||||
|
||||
training_data = select_columns_using_Pandas_on_CSV_data_op(
|
||||
table=training_data,
|
||||
column_names=all_columns,
|
||||
).outputs["transformed_table"]
|
||||
|
||||
# Cleaning the NaN values.
|
||||
training_data = fill_all_missing_values_using_Pandas_on_CSV_data_op(
|
||||
table=training_data,
|
||||
replacement_value="0",
|
||||
#replacement_type_name="float",
|
||||
).outputs["transformed_table"]
|
||||
|
||||
classification_training_data = binarize_column_using_Pandas_on_CSV_data_op(
|
||||
table=training_data,
|
||||
column_name=label_column,
|
||||
predicate=" > 0",
|
||||
new_column_name=classification_label_column,
|
||||
).outputs["transformed_table"]
|
||||
|
||||
network = create_fully_connected_pytorch_network_op(
|
||||
input_size=len(feature_columns),
|
||||
# Optional:
|
||||
hidden_layer_sizes=[10],
|
||||
activation_name="elu",
|
||||
output_activation_name="sigmoid",
|
||||
# output_size=1,
|
||||
).outputs["model"]
|
||||
|
||||
model = train_pytorch_model_from_csv_op(
|
||||
model=network,
|
||||
training_data=classification_training_data,
|
||||
label_column_name=classification_label_column,
|
||||
loss_function_name="binary_cross_entropy",
|
||||
# Optional:
|
||||
#number_of_epochs=1,
|
||||
#learning_rate=0.1,
|
||||
#optimizer_name="Adadelta",
|
||||
#optimizer_parameters={},
|
||||
#batch_size=32,
|
||||
#batch_log_interval=100,
|
||||
#random_seed=0,
|
||||
).outputs["trained_model"]
|
||||
|
||||
model_archive = create_pytorch_model_archive_with_base_handler_op(
|
||||
model=model,
|
||||
# Optional:
|
||||
# model_name="model",
|
||||
# model_version="1.0",
|
||||
).outputs["Model archive"]
|
||||
|
||||
vertex_model_name = upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op(
|
||||
model_archive=model_archive,
|
||||
).outputs["model_name"]
|
||||
|
||||
# Deploying the model might incur additional costs over time
|
||||
if deploy_model:
|
||||
vertex_endpoint_name = deploy_model_to_endpoint_op(
|
||||
model_name=vertex_model_name,
|
||||
).outputs["endpoint_name"]
|
||||
|
||||
pipeline_func=train_tabular_classification_model_using_PyTorch_pipeline
|
||||
|
||||
# %% Pipeline submission
|
||||
if __name__ == '__main__':
|
||||
from google.cloud import aiplatform
|
||||
aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()
|
||||
-132
@@ -1,132 +0,0 @@
|
||||
name: Train tabular classification model using TensorFlow pipeline
|
||||
metadata:
|
||||
annotations:
|
||||
author: Alexey Volkov <alexey.volkov@ark-kun.com>
|
||||
canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_classification_model_using_TensorFlow_and_import_to_Vertex_AI/pipeline.component.yaml
|
||||
sdk: https://cloud-pipelines.net/pipeline-editor/
|
||||
implementation:
|
||||
graph:
|
||||
tasks:
|
||||
Download from GCS:
|
||||
componentRef:
|
||||
digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":40,"width":180,"height":40}'
|
||||
Select columns using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: Data
|
||||
taskId: Download from GCS
|
||||
column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":140,"width":180,"height":54}'
|
||||
Fill all missing values using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Select columns using Pandas on CSV data
|
||||
type: CSV
|
||||
replacement_value: '0'
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":250,"width":180,"height":54}'
|
||||
Binarize column using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: d699afd4d7cae862708717cc160f4394ed0c04e536e9515923ef1e8865f01d44
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Fill all missing values using Pandas on CSV data
|
||||
type: CSV
|
||||
column_name: tips
|
||||
predicate: ' > 0'
|
||||
new_column_name: class
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":370,"width":180,"height":54}'
|
||||
Split rows into subsets:
|
||||
componentRef:
|
||||
digest: a609c3c9196484290f24a1174955f95b27f07a7b458aa5cb8cde28866cb2cb46
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Binarize column using Pandas on CSV data
|
||||
type: CSV
|
||||
fraction_1: '0.8'
|
||||
annotations:
|
||||
editor.position: '{"x":170,"y":500,"width":180,"height":40}'
|
||||
Create fully connected tensorflow network:
|
||||
componentRef:
|
||||
digest: bfcafbc5ce711b1f69cabf1338212d10d50136a73db9f9f7c984de7b80b4bfb0
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/9ca0f9eecf5f896f65b8538bbd809747052617d1/components/tensorflow/Create_fully_connected_network/component.yaml
|
||||
arguments:
|
||||
input_size: '7'
|
||||
hidden_layer_sizes: '[10]'
|
||||
activation_name: elu
|
||||
output_activation_name: sigmoid
|
||||
annotations:
|
||||
editor.position: '{"x":370,"y":500,"width":180,"height":54}'
|
||||
Train model using Keras on CSV:
|
||||
componentRef:
|
||||
digest: 42ae60c889034dbad74815653e95b4f7d576b5f47f803173e8679c7b54984609
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c504a4010348c50eaaf6d4337586ccc008f4dcef/components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml
|
||||
arguments:
|
||||
training_data:
|
||||
taskOutput:
|
||||
outputName: split_1
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Create fully connected tensorflow network
|
||||
type: TensorflowSavedModel
|
||||
label_column_name: class
|
||||
loss_function_name: binary_crossentropy
|
||||
number_of_epochs: '10'
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":620,"width":180,"height":54}'
|
||||
Upload Tensorflow model to Google Cloud Vertex AI:
|
||||
componentRef:
|
||||
digest: 2e45263ff640b1a688e359b6936e27a81b2407749a84f340af2aa5547e0cb92c
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: trained_model
|
||||
taskId: Train model using Keras on CSV
|
||||
type: TensorflowSavedModel
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":750,"width":180,"height":54}'
|
||||
Predict with TensorFlow model on CSV data:
|
||||
componentRef:
|
||||
digest: 921bb1563e93a78233b8acceab87055b9154ccf5595d056028cf0396ca224cd4
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/59c759ce6f543184e30db6817d2a703879bc0f39/components/tensorflow/Predict/on_CSV/component.yaml
|
||||
arguments:
|
||||
dataset:
|
||||
taskOutput:
|
||||
outputName: split_2
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: trained_model
|
||||
taskId: Train model using Keras on CSV
|
||||
type: TensorflowSavedModel
|
||||
label_column_name: class
|
||||
annotations:
|
||||
editor.position: '{"x":240,"y":750,"width":180,"height":54}'
|
||||
outputValues: {}
|
||||
-106
@@ -1,106 +0,0 @@
|
||||
# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
|
||||
from kfp import components
|
||||
|
||||
# %% Loading components
|
||||
download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml")
|
||||
fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
|
||||
binarize_column_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml")
|
||||
split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
|
||||
create_fully_connected_tensorflow_network_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/9ca0f9eecf5f896f65b8538bbd809747052617d1/components/tensorflow/Create_fully_connected_network/component.yaml")
|
||||
train_model_using_Keras_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c504a4010348c50eaaf6d4337586ccc008f4dcef/components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml")
|
||||
predict_with_TensorFlow_model_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/59c759ce6f543184e30db6817d2a703879bc0f39/components/tensorflow/Predict/on_CSV/component.yaml")
|
||||
upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
|
||||
# %% Pipeline definition
|
||||
def train_tabular_classification_model_using_TensorFlow_pipeline():
|
||||
dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
|
||||
feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"] # Excluded "trip_total"
|
||||
label_column = "tips"
|
||||
training_set_fraction = 0.8
|
||||
# Deploying the model might incur additional costs over time
|
||||
deploy_model = False
|
||||
|
||||
classification_label_column = "class"
|
||||
all_columns = [label_column] + feature_columns
|
||||
|
||||
dataset = download_from_gcs_op(
|
||||
gcs_path=dataset_gcs_uri
|
||||
).outputs["Data"]
|
||||
|
||||
dataset = select_columns_using_Pandas_on_CSV_data_op(
|
||||
table=dataset,
|
||||
column_names=all_columns,
|
||||
).outputs["transformed_table"]
|
||||
|
||||
dataset = fill_all_missing_values_using_Pandas_on_CSV_data_op(
|
||||
table=dataset,
|
||||
replacement_value="0",
|
||||
# # Optional:
|
||||
# column_names=None, # =[...]
|
||||
).outputs["transformed_table"]
|
||||
|
||||
classification_dataset = binarize_column_using_Pandas_on_CSV_data_op(
|
||||
table=dataset,
|
||||
column_name=label_column,
|
||||
predicate=" > 0",
|
||||
new_column_name=classification_label_column,
|
||||
).outputs["transformed_table"]
|
||||
|
||||
split_task = split_rows_into_subsets_op(
|
||||
table=classification_dataset,
|
||||
fraction_1=training_set_fraction,
|
||||
)
|
||||
classification_training_data = split_task.outputs["split_1"]
|
||||
classification_testing_data = split_task.outputs["split_2"]
|
||||
|
||||
network = create_fully_connected_tensorflow_network_op(
|
||||
input_size=len(feature_columns),
|
||||
# Optional:
|
||||
hidden_layer_sizes=[10],
|
||||
activation_name="elu",
|
||||
output_activation_name="sigmoid",
|
||||
# output_size=1,
|
||||
).outputs["model"]
|
||||
|
||||
model = train_model_using_Keras_on_CSV_op(
|
||||
training_data=classification_training_data,
|
||||
model=network,
|
||||
label_column_name=classification_label_column,
|
||||
# Optional:
|
||||
loss_function_name="binary_crossentropy",
|
||||
number_of_epochs=10,
|
||||
#learning_rate=0.1,
|
||||
#optimizer_name="Adadelta",
|
||||
#optimizer_parameters={},
|
||||
#batch_size=32,
|
||||
#metric_names=["mean_absolute_error"],
|
||||
#random_seed=0,
|
||||
).outputs["trained_model"]
|
||||
|
||||
predictions = predict_with_TensorFlow_model_on_CSV_data_op(
|
||||
dataset=classification_testing_data,
|
||||
model=model,
|
||||
# label_column_name needs to be set when doing prediction on a dataset that has labels
|
||||
label_column_name=classification_label_column,
|
||||
# Optional:
|
||||
# batch_size=1000,
|
||||
).outputs["predictions"]
|
||||
|
||||
vertex_model_name = upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op(
|
||||
model=model,
|
||||
).outputs["model_name"]
|
||||
|
||||
# Deploying the model might incur additional costs over time
|
||||
if deploy_model:
|
||||
vertex_endpoint_name = deploy_model_to_endpoint_op(
|
||||
model_name=vertex_model_name,
|
||||
).outputs["endpoint_name"]
|
||||
|
||||
pipeline_func = train_tabular_classification_model_using_TensorFlow_pipeline
|
||||
|
||||
# %% Pipeline submission
|
||||
if __name__ == '__main__':
|
||||
from google.cloud import aiplatform
|
||||
aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()
|
||||
-115
@@ -1,115 +0,0 @@
|
||||
name: Train tabular classification model using XGBoost pipeline
|
||||
metadata:
|
||||
annotations:
|
||||
author: Alexey Volkov <alexey.volkov@ark-kun.com>
|
||||
canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_classification_model_using_XGBoost_and_import_to_Vertex_AI/pipeline.component.yaml
|
||||
sdk: https://cloud-pipelines.net/pipeline-editor/
|
||||
implementation:
|
||||
graph:
|
||||
tasks:
|
||||
Download from GCS:
|
||||
componentRef:
|
||||
digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":40,"width":180,"height":40}'
|
||||
Select columns using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: Data
|
||||
taskId: Download from GCS
|
||||
column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":140,"width":180,"height":54}'
|
||||
Fill all missing values using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Select columns using Pandas on CSV data
|
||||
type: CSV
|
||||
replacement_value: '0'
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":250,"width":180,"height":54}'
|
||||
Binarize column using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: d699afd4d7cae862708717cc160f4394ed0c04e536e9515923ef1e8865f01d44
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Fill all missing values using Pandas on CSV data
|
||||
type: CSV
|
||||
column_name: tips
|
||||
predicate: '> 0'
|
||||
new_column_name: class
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":380,"width":180,"height":54}'
|
||||
Split rows into subsets:
|
||||
componentRef:
|
||||
digest: a609c3c9196484290f24a1174955f95b27f07a7b458aa5cb8cde28866cb2cb46
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Binarize column using Pandas on CSV data
|
||||
type: CSV
|
||||
fraction_1: '0.8'
|
||||
annotations:
|
||||
editor.position: '{"x":170,"y":510,"width":180,"height":40}'
|
||||
Train XGBoost model on CSV:
|
||||
componentRef:
|
||||
digest: 538c5a01eb38deaf532d619f0bbeaff4efc550fe1f0f776fc06791097b68ceac
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/58d3a47f904f32a64af8403330ba7e2134cae46d/components/XGBoost/Train/component.yaml
|
||||
arguments:
|
||||
training_data:
|
||||
taskOutput:
|
||||
outputName: split_1
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
label_column_name: class
|
||||
objective: binary:logistic
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":630,"width":180,"height":40}'
|
||||
Upload XGBoost model to Google Cloud Vertex AI:
|
||||
componentRef:
|
||||
digest: 5a5a273c403670743820986c03a4175b7cb4595a556524fefcce403656286977
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Train XGBoost model on CSV
|
||||
type: XGBoostModel
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":750,"width":180,"height":54}'
|
||||
Xgboost predict on CSV:
|
||||
componentRef:
|
||||
digest: 0876233a0c7306fefec188bd70f059b46d1fb5aa57be231799570e3bbbdd0d95
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/4694ec97baccf59284c2a1db4aa2250c22291eab/components/XGBoost/Predict/component.yaml
|
||||
arguments:
|
||||
data:
|
||||
taskOutput:
|
||||
outputName: split_2
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Train XGBoost model on CSV
|
||||
type: XGBoostModel
|
||||
label_column_name: class
|
||||
annotations:
|
||||
editor.position: '{"x":240,"y":750,"width":180,"height":40}'
|
||||
outputValues: {}
|
||||
-94
@@ -1,94 +0,0 @@
|
||||
# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
|
||||
from kfp import components
|
||||
|
||||
# %% Loading components
|
||||
download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml")
|
||||
fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
|
||||
binarize_column_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml")
|
||||
split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
|
||||
train_XGBoost_model_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/58d3a47f904f32a64af8403330ba7e2134cae46d/components/XGBoost/Train/component.yaml")
|
||||
xgboost_predict_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/4694ec97baccf59284c2a1db4aa2250c22291eab/components/XGBoost/Predict/component.yaml")
|
||||
upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
|
||||
# %% Pipeline definition
|
||||
def train_tabular_classification_model_using_XGBoost_pipeline():
|
||||
dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
|
||||
feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"] # Excluded "trip_total"
|
||||
label_column = "tips"
|
||||
training_set_fraction = 0.8
|
||||
# Deploying the model might incur additional costs over time
|
||||
deploy_model = False
|
||||
|
||||
classification_label_column = "class"
|
||||
all_columns = [label_column] + feature_columns
|
||||
|
||||
dataset = download_from_gcs_op(
|
||||
gcs_path=dataset_gcs_uri
|
||||
).outputs["Data"]
|
||||
|
||||
dataset = select_columns_using_Pandas_on_CSV_data_op(
|
||||
table=dataset,
|
||||
column_names=all_columns,
|
||||
).outputs["transformed_table"]
|
||||
|
||||
dataset = fill_all_missing_values_using_Pandas_on_CSV_data_op(
|
||||
table=dataset,
|
||||
replacement_value="0",
|
||||
# # Optional:
|
||||
# column_names=None, # =[...]
|
||||
).outputs["transformed_table"]
|
||||
|
||||
classification_dataset = binarize_column_using_Pandas_on_CSV_data_op(
|
||||
table=dataset,
|
||||
column_name=label_column,
|
||||
predicate="> 0",
|
||||
new_column_name=classification_label_column,
|
||||
).outputs["transformed_table"]
|
||||
|
||||
split_task = split_rows_into_subsets_op(
|
||||
table=classification_dataset,
|
||||
fraction_1=training_set_fraction,
|
||||
)
|
||||
classification_training_data = split_task.outputs["split_1"]
|
||||
classification_testing_data = split_task.outputs["split_2"]
|
||||
|
||||
model = train_XGBoost_model_on_CSV_op(
|
||||
training_data=classification_training_data,
|
||||
label_column_name=classification_label_column,
|
||||
objective="binary:logistic",
|
||||
# Optional:
|
||||
#starting_model=None,
|
||||
#num_iterations=10,
|
||||
#booster_params={},
|
||||
#booster="gbtree",
|
||||
#learning_rate=0.3,
|
||||
#min_split_loss=0,
|
||||
#max_depth=6,
|
||||
).outputs["model"]
|
||||
|
||||
# Predicting on the testing data
|
||||
predictions = xgboost_predict_on_CSV_op(
|
||||
data=classification_testing_data,
|
||||
model=model,
|
||||
# label_column needs to be set when doing prediction on a dataset that has labels
|
||||
label_column_name=classification_label_column,
|
||||
).outputs["predictions"]
|
||||
|
||||
vertex_model_name = upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op(
|
||||
model=model,
|
||||
).outputs["model_name"]
|
||||
|
||||
# Deploying the model might incur additional costs over time
|
||||
if deploy_model:
|
||||
vertex_endpoint_name = deploy_model_to_endpoint_op(
|
||||
model_name=vertex_model_name,
|
||||
).outputs["endpoint_name"]
|
||||
|
||||
pipeline_func = train_tabular_classification_model_using_XGBoost_pipeline
|
||||
|
||||
# %% Pipeline submission
|
||||
if __name__ == '__main__':
|
||||
from google.cloud import aiplatform
|
||||
aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()
|
||||
-257
@@ -1,257 +0,0 @@
|
||||
name: Train tabular classification model using all frameworks pipeline
|
||||
metadata:
|
||||
annotations:
|
||||
author: Alexey Volkov <alexey.volkov@ark-kun.com>
|
||||
canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_classification_model_using_all_frameworks_and_import_to_Vertex_AI/pipeline.component.yaml
|
||||
sdk: https://cloud-pipelines.net/pipeline-editor/
|
||||
implementation:
|
||||
graph:
|
||||
tasks:
|
||||
Download from GCS:
|
||||
componentRef:
|
||||
digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
|
||||
annotations:
|
||||
editor.position: '{"x":550,"y":40,"width":180,"height":40}'
|
||||
Select columns using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: Data
|
||||
taskId: Download from GCS
|
||||
column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
|
||||
annotations:
|
||||
editor.position: '{"x":550,"y":140,"width":180,"height":54}'
|
||||
Fill all missing values using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Select columns using Pandas on CSV data
|
||||
type: CSV
|
||||
replacement_value: '0'
|
||||
annotations:
|
||||
editor.position: '{"x":550,"y":250,"width":180,"height":54}'
|
||||
Binarize column using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: d699afd4d7cae862708717cc160f4394ed0c04e536e9515923ef1e8865f01d44
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Fill all missing values using Pandas on CSV data
|
||||
type: CSV
|
||||
column_name: tips
|
||||
predicate: ' > 0'
|
||||
new_column_name: class
|
||||
annotations:
|
||||
editor.position: '{"x":550,"y":380,"width":180,"height":54}'
|
||||
Split rows into subsets:
|
||||
componentRef:
|
||||
digest: a609c3c9196484290f24a1174955f95b27f07a7b458aa5cb8cde28866cb2cb46
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Binarize column using Pandas on CSV data
|
||||
type: CSV
|
||||
fraction_1: '0.8'
|
||||
annotations:
|
||||
editor.position: '{"x":550,"y":490,"width":180,"height":40}'
|
||||
Create fully connected pytorch network:
|
||||
componentRef:
|
||||
digest: d03d8248fd358a0275ec33568ee7dd7dce576cc112b09dfafe2651e4d97e04a9
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1a2ef3eeb77bc278f33cad0dd29008ea2431e191/components/PyTorch/Create_fully_connected_network/component.yaml
|
||||
arguments:
|
||||
input_size: '7'
|
||||
hidden_layer_sizes: '[10]'
|
||||
activation_name: elu
|
||||
output_activation_name: sigmoid
|
||||
annotations:
|
||||
editor.position: '{"x":380,"y":620,"width":180,"height":54}'
|
||||
Create fully connected tensorflow network:
|
||||
componentRef:
|
||||
digest: bfcafbc5ce711b1f69cabf1338212d10d50136a73db9f9f7c984de7b80b4bfb0
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/9ca0f9eecf5f896f65b8538bbd809747052617d1/components/tensorflow/Create_fully_connected_network/component.yaml
|
||||
arguments:
|
||||
input_size: '7'
|
||||
hidden_layer_sizes: '[10]'
|
||||
activation_name: elu
|
||||
output_activation_name: sigmoid
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":630,"width":180,"height":54}'
|
||||
Train model using Keras on CSV:
|
||||
componentRef:
|
||||
digest: 42ae60c889034dbad74815653e95b4f7d576b5f47f803173e8679c7b54984609
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c504a4010348c50eaaf6d4337586ccc008f4dcef/components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml
|
||||
arguments:
|
||||
training_data:
|
||||
taskOutput:
|
||||
outputName: split_1
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Create fully connected tensorflow network
|
||||
type: TensorflowSavedModel
|
||||
label_column_name: class
|
||||
loss_function_name: binary_crossentropy
|
||||
number_of_epochs: '10'
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":750,"width":180,"height":54}'
|
||||
Train pytorch model from csv:
|
||||
componentRef:
|
||||
digest: 40f3185eb61e9727f41a4e0c05dd3d3b44bd802aa0f378cfc31756560033949a
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml
|
||||
arguments:
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Create fully connected pytorch network
|
||||
type: PyTorchScriptModule
|
||||
training_data:
|
||||
taskOutput:
|
||||
outputName: split_1
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
label_column_name: class
|
||||
loss_function_name: binary_cross_entropy
|
||||
annotations:
|
||||
editor.position: '{"x":380,"y":750,"width":180,"height":40}'
|
||||
Train XGBoost model on CSV:
|
||||
componentRef:
|
||||
digest: 538c5a01eb38deaf532d619f0bbeaff4efc550fe1f0f776fc06791097b68ceac
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/58d3a47f904f32a64af8403330ba7e2134cae46d/components/XGBoost/Train/component.yaml
|
||||
arguments:
|
||||
training_data:
|
||||
taskOutput:
|
||||
outputName: split_1
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
label_column_name: class
|
||||
objective: binary:logistic
|
||||
annotations:
|
||||
editor.position: '{"x":720,"y":750,"width":180,"height":40}'
|
||||
Train logistic regression model using scikit learn from CSV:
|
||||
componentRef:
|
||||
digest: a864625a822e4b1c8ef6fe4ae1454fd90f15438f70a6712bb4c30e0dda4d35b7
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/cb44b75c9c062fcc40c2b905b2024b4493dbc62b/components/ML_frameworks/Scikit_learn/Train_logistic_regression_model/from_CSV/component.yaml
|
||||
arguments:
|
||||
dataset:
|
||||
taskOutput:
|
||||
outputName: split_1
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
label_column_name: class
|
||||
annotations:
|
||||
editor.position: '{"x":1030,"y":750,"width":180,"height":70}'
|
||||
Predict with TensorFlow model on CSV data:
|
||||
componentRef:
|
||||
digest: 921bb1563e93a78233b8acceab87055b9154ccf5595d056028cf0396ca224cd4
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/59c759ce6f543184e30db6817d2a703879bc0f39/components/tensorflow/Predict/on_CSV/component.yaml
|
||||
arguments:
|
||||
dataset:
|
||||
taskOutput:
|
||||
outputName: split_2
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: trained_model
|
||||
taskId: Train model using Keras on CSV
|
||||
type: TensorflowSavedModel
|
||||
label_column_name: class
|
||||
annotations:
|
||||
editor.position: '{"x":160,"y":880,"width":180,"height":54}'
|
||||
Create PyTorch Model Archive with base handler:
|
||||
componentRef:
|
||||
digest: 8298b5ee1b0f0879f893add4cf352c8dec7cf9e21bb9db134c91a2d046cdb0ec
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/46d51383e6554b7f3ab4fd8cf614d8c2b422fb22/components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml
|
||||
arguments:
|
||||
Model:
|
||||
taskOutput:
|
||||
outputName: trained_model
|
||||
taskId: Train pytorch model from csv
|
||||
type: PyTorchScriptModule
|
||||
Model name: model
|
||||
Model version: '1.0'
|
||||
annotations:
|
||||
editor.position: '{"x":380,"y":880,"width":180,"height":54}'
|
||||
Xgboost predict on CSV:
|
||||
componentRef:
|
||||
digest: 0876233a0c7306fefec188bd70f059b46d1fb5aa57be231799570e3bbbdd0d95
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/4694ec97baccf59284c2a1db4aa2250c22291eab/components/XGBoost/Predict/component.yaml
|
||||
arguments:
|
||||
data:
|
||||
taskOutput:
|
||||
outputName: split_2
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Train XGBoost model on CSV
|
||||
type: XGBoostModel
|
||||
label_column_name: class
|
||||
annotations:
|
||||
editor.position: '{"x":810,"y":880,"width":180,"height":40}'
|
||||
Upload Scikit learn pickle model to Google Cloud Vertex AI:
|
||||
componentRef:
|
||||
digest: 81c91c8d7d21ec97e0872f669d68bd89edea87279d703685db54aa94743bebcd
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Train logistic regression model using scikit learn from CSV
|
||||
type: ScikitLearnPickleModel
|
||||
annotations:
|
||||
editor.position: '{"x":1030,"y":880,"width":180,"height":70}'
|
||||
Upload Tensorflow model to Google Cloud Vertex AI:
|
||||
componentRef:
|
||||
digest: 2e45263ff640b1a688e359b6936e27a81b2407749a84f340af2aa5547e0cb92c
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: trained_model
|
||||
taskId: Train model using Keras on CSV
|
||||
type: TensorflowSavedModel
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":1010,"width":180,"height":54}'
|
||||
Upload PyTorch model archive to Google Cloud Vertex AI:
|
||||
componentRef:
|
||||
digest: 4450212fae7b9001482aca7eb78b28413c205506eccf08a04e7754a8dfa99004
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
model_archive:
|
||||
taskOutput:
|
||||
outputName: Model archive
|
||||
taskId: Create PyTorch Model Archive with base handler
|
||||
type: PyTorchModelArchive
|
||||
annotations:
|
||||
editor.position: '{"x":380,"y":1010,"width":180,"height":70}'
|
||||
Upload XGBoost model to Google Cloud Vertex AI:
|
||||
componentRef:
|
||||
digest: 5a5a273c403670743820986c03a4175b7cb4595a556524fefcce403656286977
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Train XGBoost model on CSV
|
||||
type: XGBoostModel
|
||||
annotations:
|
||||
editor.position: '{"x":720,"y":1010,"width":180,"height":54}'
|
||||
outputValues: {}
|
||||
-224
@@ -1,224 +0,0 @@
|
||||
# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
|
||||
from kfp import components
|
||||
|
||||
# %% Loading components
|
||||
download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml")
|
||||
fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
|
||||
binarize_column_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml")
|
||||
split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
|
||||
|
||||
# TensorFlow
|
||||
create_fully_connected_tensorflow_network_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/9ca0f9eecf5f896f65b8538bbd809747052617d1/components/tensorflow/Create_fully_connected_network/component.yaml")
|
||||
train_model_using_Keras_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c504a4010348c50eaaf6d4337586ccc008f4dcef/components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml")
|
||||
predict_with_TensorFlow_model_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/59c759ce6f543184e30db6817d2a703879bc0f39/components/tensorflow/Predict/on_CSV/component.yaml")
|
||||
upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
|
||||
# PyTorch
|
||||
create_fully_connected_pytorch_network_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/1a2ef3eeb77bc278f33cad0dd29008ea2431e191/components/PyTorch/Create_fully_connected_network/component.yaml")
|
||||
train_pytorch_model_from_csv_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml")
|
||||
create_pytorch_model_archive_with_base_handler_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/46d51383e6554b7f3ab4fd8cf614d8c2b422fb22/components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml")
|
||||
upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
|
||||
# XGBoost
|
||||
train_XGBoost_model_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/58d3a47f904f32a64af8403330ba7e2134cae46d/components/XGBoost/Train/component.yaml")
|
||||
xgboost_predict_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/4694ec97baccf59284c2a1db4aa2250c22291eab/components/XGBoost/Predict/component.yaml")
|
||||
upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
|
||||
# Scikit-learn
|
||||
#train_linear_regression_model_using_scikit_learn_from_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/f807e02b54d4886c65a05f40848fd51c72407f40/components/ML_frameworks/Scikit_learn/Train_linear_regression_model/from_CSV/component.yaml")
|
||||
train_logistic_regression_model_using_scikit_learn_from_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/cb44b75c9c062fcc40c2b905b2024b4493dbc62b/components/ML_frameworks/Scikit_learn/Train_logistic_regression_model/from_CSV/component.yaml")
|
||||
upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
|
||||
# Vertex AI
|
||||
deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
|
||||
# %% Pipeline definition
|
||||
def train_tabular_classification_model_using_all_frameworks_pipeline():
|
||||
dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
|
||||
feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"] # Excluded "trip_total"
|
||||
label_column = "tips"
|
||||
training_set_fraction = 0.8
|
||||
# Deploying the model might incur additional costs over time
|
||||
deploy_model = False
|
||||
|
||||
classification_label_column = "class"
|
||||
all_columns = [label_column] + feature_columns
|
||||
|
||||
dataset = download_from_gcs_op(
|
||||
gcs_path=dataset_gcs_uri
|
||||
).outputs["Data"]
|
||||
|
||||
dataset = select_columns_using_Pandas_on_CSV_data_op(
|
||||
table=dataset,
|
||||
column_names=all_columns,
|
||||
).outputs["transformed_table"]
|
||||
|
||||
dataset = fill_all_missing_values_using_Pandas_on_CSV_data_op(
|
||||
table=dataset,
|
||||
replacement_value="0",
|
||||
# # Optional:
|
||||
# column_names=None, # =[...]
|
||||
).outputs["transformed_table"]
|
||||
|
||||
classification_dataset = binarize_column_using_Pandas_on_CSV_data_op(
|
||||
table=dataset,
|
||||
column_name=label_column,
|
||||
predicate=" > 0",
|
||||
new_column_name=classification_label_column,
|
||||
).outputs["transformed_table"]
|
||||
|
||||
split_task = split_rows_into_subsets_op(
|
||||
table=classification_dataset,
|
||||
fraction_1=training_set_fraction,
|
||||
)
|
||||
classification_training_data = split_task.outputs["split_1"]
|
||||
classification_testing_data = split_task.outputs["split_2"]
|
||||
|
||||
# TensorFlow
|
||||
tensorflow_network = create_fully_connected_tensorflow_network_op(
|
||||
input_size=len(feature_columns),
|
||||
# Optional:
|
||||
hidden_layer_sizes=[10],
|
||||
activation_name="elu",
|
||||
output_activation_name="sigmoid",
|
||||
# output_size=1,
|
||||
).outputs["model"]
|
||||
|
||||
tensorflow_model = train_model_using_Keras_on_CSV_op(
|
||||
training_data=classification_training_data,
|
||||
model=tensorflow_network,
|
||||
label_column_name=classification_label_column,
|
||||
# Optional:
|
||||
loss_function_name="binary_crossentropy",
|
||||
number_of_epochs=10,
|
||||
#learning_rate=0.1,
|
||||
#optimizer_name="Adadelta",
|
||||
#optimizer_parameters={},
|
||||
#batch_size=32,
|
||||
#metric_names=["mean_absolute_error"],
|
||||
#random_seed=0,
|
||||
).outputs["trained_model"]
|
||||
|
||||
tensorflow_predictions = predict_with_TensorFlow_model_on_CSV_data_op(
|
||||
dataset=classification_testing_data,
|
||||
model=tensorflow_model,
|
||||
# label_column_name needs to be set when doing prediction on a dataset that has labels
|
||||
label_column_name=classification_label_column,
|
||||
# Optional:
|
||||
# batch_size=1000,
|
||||
).outputs["predictions"]
|
||||
|
||||
tensorflow_vertex_model_name = upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op(
|
||||
model=tensorflow_model,
|
||||
).outputs["model_name"]
|
||||
|
||||
# Deploying the model might incur additional costs over time
|
||||
if deploy_model:
|
||||
tensorflow_vertex_endpoint_name = deploy_model_to_endpoint_op(
|
||||
model_name=tensorflow_vertex_model_name,
|
||||
).outputs["endpoint_name"]
|
||||
|
||||
# PyTorch
|
||||
pytorch_network = create_fully_connected_pytorch_network_op(
|
||||
input_size=len(feature_columns),
|
||||
# Optional:
|
||||
hidden_layer_sizes=[10],
|
||||
activation_name="elu",
|
||||
output_activation_name="sigmoid",
|
||||
# output_size=1,
|
||||
).outputs["model"]
|
||||
|
||||
pytorch_model = train_pytorch_model_from_csv_op(
|
||||
model=pytorch_network,
|
||||
training_data=classification_training_data,
|
||||
label_column_name=classification_label_column,
|
||||
loss_function_name="binary_cross_entropy",
|
||||
# Optional:
|
||||
#number_of_epochs=1,
|
||||
#learning_rate=0.1,
|
||||
#optimizer_name="Adadelta",
|
||||
#optimizer_parameters={},
|
||||
#batch_size=32,
|
||||
#batch_log_interval=100,
|
||||
#random_seed=0,
|
||||
).outputs["trained_model"]
|
||||
|
||||
pytorch_model_archive = create_pytorch_model_archive_with_base_handler_op(
|
||||
model=pytorch_model,
|
||||
# Optional:
|
||||
# model_name="model",
|
||||
# model_version="1.0",
|
||||
).outputs["Model archive"]
|
||||
|
||||
pytorch_vertex_model_name = upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op(
|
||||
model_archive=pytorch_model_archive,
|
||||
).outputs["model_name"]
|
||||
|
||||
# Deploying the model might incur additional costs over time
|
||||
if deploy_model:
|
||||
pytorch_vertex_endpoint_name = deploy_model_to_endpoint_op(
|
||||
model_name=pytorch_vertex_model_name,
|
||||
).outputs["endpoint_name"]
|
||||
|
||||
# XGBoost
|
||||
xgboost_model = train_XGBoost_model_on_CSV_op(
|
||||
training_data=classification_training_data,
|
||||
label_column_name=classification_label_column,
|
||||
objective="binary:logistic",
|
||||
# Optional:
|
||||
#starting_model=None,
|
||||
#num_iterations=10,
|
||||
#booster_params={},
|
||||
#booster="gbtree",
|
||||
#learning_rate=0.3,
|
||||
#min_split_loss=0,
|
||||
#max_depth=6,
|
||||
).outputs["model"]
|
||||
|
||||
# Predicting on the testing data
|
||||
xgboost_predictions = xgboost_predict_on_CSV_op(
|
||||
data=classification_testing_data,
|
||||
model=xgboost_model,
|
||||
# label_column needs to be set when doing prediction on a dataset that has labels
|
||||
label_column_name=classification_label_column,
|
||||
).outputs["predictions"]
|
||||
|
||||
xgboost_vertex_model_name = upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op(
|
||||
model=xgboost_model,
|
||||
).outputs["model_name"]
|
||||
|
||||
# Deploying the model might incur additional costs over time
|
||||
if deploy_model:
|
||||
xgboost_vertex_endpoint_name = deploy_model_to_endpoint_op(
|
||||
model_name=xgboost_vertex_model_name,
|
||||
).outputs["endpoint_name"]
|
||||
|
||||
# Scikit-learn
|
||||
sklearn_model = train_logistic_regression_model_using_scikit_learn_from_CSV_op(
|
||||
dataset=classification_training_data,
|
||||
label_column_name=classification_label_column,
|
||||
# Optional:
|
||||
#penalty="l2",
|
||||
#solver="lbfgs",
|
||||
#max_iterations=100,
|
||||
#multi_class_mode="auto",
|
||||
#random_seed=0,
|
||||
).outputs["model"]
|
||||
|
||||
sklearn_vertex_model_name = upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op(
|
||||
model=sklearn_model,
|
||||
).outputs["model_name"]
|
||||
|
||||
# Deploying the model might incur additional costs over time
|
||||
if deploy_model:
|
||||
sklearn_vertex_endpoint_name = deploy_model_to_endpoint_op(
|
||||
model_name=sklearn_vertex_model_name,
|
||||
).outputs["endpoint_name"]
|
||||
|
||||
pipeline_func=train_tabular_classification_model_using_all_frameworks_pipeline
|
||||
|
||||
# %% Pipeline submission
|
||||
if __name__ == '__main__':
|
||||
from google.cloud import aiplatform
|
||||
aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()
|
||||
-68
@@ -1,68 +0,0 @@
|
||||
name: Train tabular regression linear model using Scikit learn pipeline
|
||||
metadata:
|
||||
annotations:
|
||||
author: Alexey Volkov <alexey.volkov@ark-kun.com>
|
||||
canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_regression_linear_model_using_Scikit_learn_and_import_to_Vertex_AI/pipeline.component.yaml
|
||||
sdk: https://cloud-pipelines.net/pipeline-editor/
|
||||
implementation:
|
||||
graph:
|
||||
tasks:
|
||||
Download from GCS:
|
||||
componentRef:
|
||||
digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":40,"width":180,"height":40}'
|
||||
Select columns using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: Data
|
||||
taskId: Download from GCS
|
||||
column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":140,"width":180,"height":54}'
|
||||
Fill all missing values using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Select columns using Pandas on CSV data
|
||||
type: CSV
|
||||
replacement_value: '0'
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":250,"width":180,"height":54}'
|
||||
Train linear regression model using scikit learn from CSV:
|
||||
componentRef:
|
||||
digest: c7fe7912ab0d1fb45d201d452e9ce6be5544e7d8c6d229db7a4b931ff58560f3
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/f807e02b54d4886c65a05f40848fd51c72407f40/components/ML_frameworks/Scikit_learn/Train_linear_regression_model/from_CSV/component.yaml
|
||||
arguments:
|
||||
dataset:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Fill all missing values using Pandas on CSV data
|
||||
type: CSV
|
||||
label_column_name: tips
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":360,"width":180,"height":54}'
|
||||
Upload Scikit learn pickle model to Google Cloud Vertex AI:
|
||||
componentRef:
|
||||
digest: 81c91c8d7d21ec97e0872f669d68bd89edea87279d703685db54aa94743bebcd
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Train linear regression model using scikit learn from CSV
|
||||
type: ScikitLearnPickleModel
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":490,"width":180,"height":70}'
|
||||
outputValues: {}
|
||||
-57
@@ -1,57 +0,0 @@
|
||||
# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
|
||||
from kfp import components
|
||||
|
||||
# %% Loading components
|
||||
download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml")
|
||||
fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
|
||||
train_linear_regression_model_using_scikit_learn_from_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/f807e02b54d4886c65a05f40848fd51c72407f40/components/ML_frameworks/Scikit_learn/Train_linear_regression_model/from_CSV/component.yaml")
|
||||
upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
|
||||
# %% Pipeline definition
|
||||
def train_tabular_regression_linear_model_using_Scikit_learn_pipeline():
|
||||
dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
|
||||
feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"] # Excluded "trip_total"
|
||||
label_column = "tips"
|
||||
all_columns = [label_column] + feature_columns
|
||||
# Deploying the model might incur additional costs over time
|
||||
deploy_model = False
|
||||
|
||||
training_data = download_from_gcs_op(
|
||||
gcs_path=dataset_gcs_uri
|
||||
).outputs["Data"]
|
||||
|
||||
training_data = select_columns_using_Pandas_on_CSV_data_op(
|
||||
table=training_data,
|
||||
column_names=all_columns,
|
||||
).outputs["transformed_table"]
|
||||
|
||||
# Cleaning the NaN values.
|
||||
training_data = fill_all_missing_values_using_Pandas_on_CSV_data_op(
|
||||
table=training_data,
|
||||
replacement_value="0",
|
||||
#replacement_type_name="float",
|
||||
).outputs["transformed_table"]
|
||||
|
||||
model = train_linear_regression_model_using_scikit_learn_from_CSV_op(
|
||||
dataset=training_data,
|
||||
label_column_name=label_column,
|
||||
).outputs["model"]
|
||||
|
||||
vertex_model_name = upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op(
|
||||
model=model,
|
||||
).outputs["model_name"]
|
||||
|
||||
# Deploying the model might incur additional costs over time
|
||||
if deploy_model:
|
||||
sklearn_vertex_endpoint_name = deploy_model_to_endpoint_op(
|
||||
model_name=vertex_model_name,
|
||||
).outputs["endpoint_name"]
|
||||
|
||||
pipeline_func = train_tabular_regression_linear_model_using_Scikit_learn_pipeline
|
||||
|
||||
# %% Pipeline submission
|
||||
if __name__ == '__main__':
|
||||
from google.cloud import aiplatform
|
||||
aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()
|
||||
-97
@@ -1,97 +0,0 @@
|
||||
name: Train tabular regression model using PyTorch pipeline
|
||||
metadata:
|
||||
annotations:
|
||||
author: Alexey Volkov <alexey.volkov@ark-kun.com>
|
||||
canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_regression_model_using_PyTorch_and_import_to_Vertex_AI/pipeline.component.yaml
|
||||
sdk: https://cloud-pipelines.net/pipeline-editor/
|
||||
implementation:
|
||||
graph:
|
||||
tasks:
|
||||
Download from GCS:
|
||||
componentRef:
|
||||
digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
|
||||
annotations:
|
||||
editor.position: '{"x":240,"y":40,"width":180,"height":40}'
|
||||
Select columns using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: Data
|
||||
taskId: Download from GCS
|
||||
column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
|
||||
annotations:
|
||||
editor.position: '{"x":240,"y":130,"width":180,"height":54}'
|
||||
Create fully connected pytorch network:
|
||||
componentRef:
|
||||
digest: d03d8248fd358a0275ec33568ee7dd7dce576cc112b09dfafe2651e4d97e04a9
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1a2ef3eeb77bc278f33cad0dd29008ea2431e191/components/PyTorch/Create_fully_connected_network/component.yaml
|
||||
arguments:
|
||||
input_size: '7'
|
||||
hidden_layer_sizes: '[10]'
|
||||
activation_name: elu
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":240,"width":180,"height":54}'
|
||||
Fill all missing values using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Select columns using Pandas on CSV data
|
||||
type: CSV
|
||||
replacement_value: '0'
|
||||
annotations:
|
||||
editor.position: '{"x":240,"y":240,"width":180,"height":54}'
|
||||
Train pytorch model from csv:
|
||||
componentRef:
|
||||
digest: 40f3185eb61e9727f41a4e0c05dd3d3b44bd802aa0f378cfc31756560033949a
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml
|
||||
arguments:
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Create fully connected pytorch network
|
||||
type: PyTorchScriptModule
|
||||
training_data:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Fill all missing values using Pandas on CSV data
|
||||
type: CSV
|
||||
label_column_name: tips
|
||||
annotations:
|
||||
editor.position: '{"x":240,"y":380,"width":180,"height":40}'
|
||||
Create PyTorch Model Archive with base handler:
|
||||
componentRef:
|
||||
digest: 8298b5ee1b0f0879f893add4cf352c8dec7cf9e21bb9db134c91a2d046cdb0ec
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/46d51383e6554b7f3ab4fd8cf614d8c2b422fb22/components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml
|
||||
arguments:
|
||||
Model:
|
||||
taskOutput:
|
||||
outputName: trained_model
|
||||
taskId: Train pytorch model from csv
|
||||
type: PyTorchScriptModule
|
||||
Model name: model
|
||||
Model version: '1.0'
|
||||
annotations:
|
||||
editor.position: '{"x":240,"y":500,"width":180,"height":54}'
|
||||
Upload PyTorch model archive to Google Cloud Vertex AI:
|
||||
componentRef:
|
||||
digest: 4450212fae7b9001482aca7eb78b28413c205506eccf08a04e7754a8dfa99004
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
model_archive:
|
||||
taskOutput:
|
||||
outputName: Model archive
|
||||
taskId: Create PyTorch Model Archive with base handler
|
||||
type: PyTorchModelArchive
|
||||
annotations:
|
||||
editor.position: '{"x":240,"y":630,"width":180,"height":70}'
|
||||
outputValues: {}
|
||||
-85
@@ -1,85 +0,0 @@
|
||||
# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
|
||||
from kfp import components
|
||||
|
||||
# %% Loading components
|
||||
download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml")
|
||||
fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
|
||||
create_fully_connected_pytorch_network_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/1a2ef3eeb77bc278f33cad0dd29008ea2431e191/components/PyTorch/Create_fully_connected_network/component.yaml")
|
||||
train_pytorch_model_from_csv_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml")
|
||||
create_pytorch_model_archive_with_base_handler_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/46d51383e6554b7f3ab4fd8cf614d8c2b422fb22/components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml")
|
||||
upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
|
||||
# %% Pipeline definition
|
||||
def train_tabular_regression_model_using_PyTorch_pipeline():
|
||||
dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
|
||||
feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"] # Excluded "trip_total"
|
||||
label_column = "tips"
|
||||
all_columns = [label_column] + feature_columns
|
||||
# Deploying the model might incur additional costs over time
|
||||
deploy_model = False
|
||||
|
||||
training_data = download_from_gcs_op(
|
||||
gcs_path=dataset_gcs_uri
|
||||
).outputs["Data"]
|
||||
|
||||
training_data = select_columns_using_Pandas_on_CSV_data_op(
|
||||
table=training_data,
|
||||
column_names=all_columns,
|
||||
).outputs["transformed_table"]
|
||||
|
||||
# Cleaning the NaN values.
|
||||
training_data = fill_all_missing_values_using_Pandas_on_CSV_data_op(
|
||||
table=training_data,
|
||||
replacement_value="0",
|
||||
#replacement_type_name="float",
|
||||
).outputs["transformed_table"]
|
||||
|
||||
network = create_fully_connected_pytorch_network_op(
|
||||
input_size=len(feature_columns),
|
||||
# Optional:
|
||||
hidden_layer_sizes=[10],
|
||||
activation_name="elu",
|
||||
# output_activation_name=None,
|
||||
# output_size=1,
|
||||
).outputs["model"]
|
||||
|
||||
model = train_pytorch_model_from_csv_op(
|
||||
model=network,
|
||||
training_data=training_data,
|
||||
label_column_name=label_column,
|
||||
# Optional:
|
||||
#loss_function_name="mse_loss",
|
||||
#number_of_epochs=1,
|
||||
#learning_rate=0.1,
|
||||
#optimizer_name="Adadelta",
|
||||
#optimizer_parameters={},
|
||||
#batch_size=32,
|
||||
#batch_log_interval=100,
|
||||
#random_seed=0,
|
||||
).outputs["trained_model"]
|
||||
|
||||
model_archive = create_pytorch_model_archive_with_base_handler_op(
|
||||
model=model,
|
||||
# Optional:
|
||||
# model_name="model",
|
||||
# model_version="1.0",
|
||||
).outputs["Model archive"]
|
||||
|
||||
vertex_model_name = upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op(
|
||||
model_archive=model_archive,
|
||||
).outputs["model_name"]
|
||||
|
||||
# Deploying the model might incur additional costs over time
|
||||
if deploy_model:
|
||||
vertex_endpoint_name = deploy_model_to_endpoint_op(
|
||||
model_name=vertex_model_name,
|
||||
).outputs["endpoint_name"]
|
||||
|
||||
pipeline_func=train_tabular_regression_model_using_PyTorch_pipeline
|
||||
|
||||
# %% Pipeline submission
|
||||
if __name__ == '__main__':
|
||||
from google.cloud import aiplatform
|
||||
aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()
|
||||
-116
@@ -1,116 +0,0 @@
|
||||
name: Train tabular regression model using Tensorflow pipeline
|
||||
metadata:
|
||||
annotations:
|
||||
author: Alexey Volkov <alexey.volkov@ark-kun.com>
|
||||
canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_regression_model_using_TensorFlow_and_import_to_Vertex_AI/pipeline.component.yaml
|
||||
sdk: https://cloud-pipelines.net/pipeline-editor/
|
||||
implementation:
|
||||
graph:
|
||||
tasks:
|
||||
Download from GCS:
|
||||
componentRef:
|
||||
digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":40,"width":180,"height":40}'
|
||||
Select columns using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: Data
|
||||
taskId: Download from GCS
|
||||
column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":140,"width":180,"height":54}'
|
||||
Fill all missing values using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Select columns using Pandas on CSV data
|
||||
type: CSV
|
||||
replacement_value: '0'
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":250,"width":180,"height":54}'
|
||||
Split rows into subsets:
|
||||
componentRef:
|
||||
digest: a609c3c9196484290f24a1174955f95b27f07a7b458aa5cb8cde28866cb2cb46
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Fill all missing values using Pandas on CSV data
|
||||
type: CSV
|
||||
fraction_1: '0.8'
|
||||
annotations:
|
||||
editor.position: '{"x":170,"y":380,"width":180,"height":40}'
|
||||
Create fully connected tensorflow network:
|
||||
componentRef:
|
||||
digest: bfcafbc5ce711b1f69cabf1338212d10d50136a73db9f9f7c984de7b80b4bfb0
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/9ca0f9eecf5f896f65b8538bbd809747052617d1/components/tensorflow/Create_fully_connected_network/component.yaml
|
||||
arguments:
|
||||
input_size: '7'
|
||||
hidden_layer_sizes: '[10]'
|
||||
activation_name: elu
|
||||
annotations:
|
||||
editor.position: '{"x":370,"y":380,"width":180,"height":54}'
|
||||
Train model using Keras on CSV:
|
||||
componentRef:
|
||||
digest: 42ae60c889034dbad74815653e95b4f7d576b5f47f803173e8679c7b54984609
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c504a4010348c50eaaf6d4337586ccc008f4dcef/components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml
|
||||
arguments:
|
||||
training_data:
|
||||
taskOutput:
|
||||
outputName: split_1
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Create fully connected tensorflow network
|
||||
type: TensorflowSavedModel
|
||||
label_column_name: tips
|
||||
number_of_epochs: '10'
|
||||
metric_names: '["mean_absolute_error"]'
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":500,"width":180,"height":54}'
|
||||
Upload Tensorflow model to Google Cloud Vertex AI:
|
||||
componentRef:
|
||||
digest: 2e45263ff640b1a688e359b6936e27a81b2407749a84f340af2aa5547e0cb92c
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: trained_model
|
||||
taskId: Train model using Keras on CSV
|
||||
type: TensorflowSavedModel
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":630,"width":180,"height":54}'
|
||||
Predict with TensorFlow model on CSV data:
|
||||
componentRef:
|
||||
digest: 921bb1563e93a78233b8acceab87055b9154ccf5595d056028cf0396ca224cd4
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/59c759ce6f543184e30db6817d2a703879bc0f39/components/tensorflow/Predict/on_CSV/component.yaml
|
||||
arguments:
|
||||
dataset:
|
||||
taskOutput:
|
||||
outputName: split_2
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: trained_model
|
||||
taskId: Train model using Keras on CSV
|
||||
type: TensorflowSavedModel
|
||||
label_column_name: tips
|
||||
annotations:
|
||||
editor.position: '{"x":240,"y":630,"width":180,"height":54}'
|
||||
outputValues: {}
|
||||
-97
@@ -1,97 +0,0 @@
|
||||
# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
|
||||
from kfp import components
|
||||
|
||||
# %% Loading components
|
||||
download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml")
|
||||
fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
|
||||
split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
|
||||
create_fully_connected_tensorflow_network_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/9ca0f9eecf5f896f65b8538bbd809747052617d1/components/tensorflow/Create_fully_connected_network/component.yaml")
|
||||
train_model_using_Keras_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c504a4010348c50eaaf6d4337586ccc008f4dcef/components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml")
|
||||
predict_with_TensorFlow_model_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/59c759ce6f543184e30db6817d2a703879bc0f39/components/tensorflow/Predict/on_CSV/component.yaml")
|
||||
upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
|
||||
# %% Pipeline definition
|
||||
def train_tabular_regression_model_using_Tensorflow_pipeline():
|
||||
dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
|
||||
feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"] # Excluded "trip_total"
|
||||
label_column = "tips"
|
||||
training_set_fraction = 0.8
|
||||
# Deploying the model might incur additional costs over time
|
||||
deploy_model = False
|
||||
|
||||
all_columns = [label_column] + feature_columns
|
||||
|
||||
dataset = download_from_gcs_op(
|
||||
gcs_path=dataset_gcs_uri
|
||||
).outputs["Data"]
|
||||
|
||||
dataset = select_columns_using_Pandas_on_CSV_data_op(
|
||||
table=dataset,
|
||||
column_names=all_columns,
|
||||
).outputs["transformed_table"]
|
||||
|
||||
dataset = fill_all_missing_values_using_Pandas_on_CSV_data_op(
|
||||
table=dataset,
|
||||
replacement_value="0",
|
||||
# # Optional:
|
||||
# column_names=None, # =[...]
|
||||
).outputs["transformed_table"]
|
||||
|
||||
split_task = split_rows_into_subsets_op(
|
||||
table=dataset,
|
||||
fraction_1=training_set_fraction,
|
||||
)
|
||||
training_data = split_task.outputs["split_1"]
|
||||
testing_data = split_task.outputs["split_2"]
|
||||
|
||||
network = create_fully_connected_tensorflow_network_op(
|
||||
input_size=len(feature_columns),
|
||||
# Optional:
|
||||
hidden_layer_sizes=[10],
|
||||
activation_name="elu",
|
||||
# output_activation_name=None,
|
||||
# output_size=1,
|
||||
).outputs["model"]
|
||||
|
||||
model = train_model_using_Keras_on_CSV_op(
|
||||
training_data=training_data,
|
||||
model=network,
|
||||
label_column_name=label_column,
|
||||
# Optional:
|
||||
#loss_function_name="mean_squared_error",
|
||||
number_of_epochs=10,
|
||||
#learning_rate=0.1,
|
||||
#optimizer_name="Adadelta",
|
||||
#optimizer_parameters={},
|
||||
#batch_size=32,
|
||||
metric_names=["mean_absolute_error"],
|
||||
#random_seed=0,
|
||||
).outputs["trained_model"]
|
||||
|
||||
predictions = predict_with_TensorFlow_model_on_CSV_data_op(
|
||||
dataset=testing_data,
|
||||
model=model,
|
||||
# label_column_name needs to be set when doing prediction on a dataset that has labels
|
||||
label_column_name=label_column,
|
||||
# Optional:
|
||||
# batch_size=1000,
|
||||
).outputs["predictions"]
|
||||
|
||||
vertex_model_name = upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op(
|
||||
model=model,
|
||||
).outputs["model_name"]
|
||||
|
||||
# Deploying the model might incur additional costs over time
|
||||
if deploy_model:
|
||||
vertex_endpoint_name = deploy_model_to_endpoint_op(
|
||||
model_name=vertex_model_name,
|
||||
).outputs["endpoint_name"]
|
||||
|
||||
pipeline_func=train_tabular_regression_model_using_Tensorflow_pipeline
|
||||
|
||||
# %% Pipeline submission
|
||||
if __name__ == '__main__':
|
||||
from google.cloud import aiplatform
|
||||
aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()
|
||||
-99
@@ -1,99 +0,0 @@
|
||||
name: Train tabular regression model using XGBoost pipeline
|
||||
metadata:
|
||||
annotations:
|
||||
author: Alexey Volkov <alexey.volkov@ark-kun.com>
|
||||
canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_regression_model_using_XGBoost_and_import_to_Vertex_AI/pipeline.component.yaml
|
||||
sdk: https://cloud-pipelines.net/pipeline-editor/
|
||||
implementation:
|
||||
graph:
|
||||
tasks:
|
||||
Download from GCS:
|
||||
componentRef:
|
||||
digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":40,"width":180,"height":40}'
|
||||
Select columns using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: Data
|
||||
taskId: Download from GCS
|
||||
column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":140,"width":180,"height":54}'
|
||||
Fill all missing values using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Select columns using Pandas on CSV data
|
||||
type: CSV
|
||||
replacement_value: '0'
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":250,"width":180,"height":54}'
|
||||
Split rows into subsets:
|
||||
componentRef:
|
||||
digest: a609c3c9196484290f24a1174955f95b27f07a7b458aa5cb8cde28866cb2cb46
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Fill all missing values using Pandas on CSV data
|
||||
type: CSV
|
||||
fraction_1: '0.8'
|
||||
annotations:
|
||||
editor.position: '{"x":170,"y":360,"width":180,"height":40}'
|
||||
Train XGBoost model on CSV:
|
||||
componentRef:
|
||||
digest: 538c5a01eb38deaf532d619f0bbeaff4efc550fe1f0f776fc06791097b68ceac
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/58d3a47f904f32a64af8403330ba7e2134cae46d/components/XGBoost/Train/component.yaml
|
||||
arguments:
|
||||
training_data:
|
||||
taskOutput:
|
||||
outputName: split_1
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
label_column_name: tips
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":480,"width":180,"height":40}'
|
||||
Upload XGBoost model to Google Cloud Vertex AI:
|
||||
componentRef:
|
||||
digest: 5a5a273c403670743820986c03a4175b7cb4595a556524fefcce403656286977
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Train XGBoost model on CSV
|
||||
type: XGBoostModel
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":600,"width":180,"height":54}'
|
||||
Xgboost predict on CSV:
|
||||
componentRef:
|
||||
digest: 0876233a0c7306fefec188bd70f059b46d1fb5aa57be231799570e3bbbdd0d95
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/4694ec97baccf59284c2a1db4aa2250c22291eab/components/XGBoost/Predict/component.yaml
|
||||
arguments:
|
||||
data:
|
||||
taskOutput:
|
||||
outputName: split_2
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Train XGBoost model on CSV
|
||||
type: XGBoostModel
|
||||
label_column_name: tips
|
||||
annotations:
|
||||
editor.position: '{"x":240,"y":600,"width":180,"height":40}'
|
||||
outputValues: {}
|
||||
-85
@@ -1,85 +0,0 @@
|
||||
# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
|
||||
from kfp import components
|
||||
|
||||
# %% Loading components
|
||||
download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml")
|
||||
fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
|
||||
split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
|
||||
train_XGBoost_model_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/58d3a47f904f32a64af8403330ba7e2134cae46d/components/XGBoost/Train/component.yaml")
|
||||
xgboost_predict_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/4694ec97baccf59284c2a1db4aa2250c22291eab/components/XGBoost/Predict/component.yaml")
|
||||
upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
|
||||
# %% Pipeline definition
|
||||
def train_tabular_regression_model_using_XGBoost_pipeline():
|
||||
dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
|
||||
feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"] # Excluded "trip_total"
|
||||
label_column = "tips"
|
||||
training_set_fraction = 0.8
|
||||
# Deploying the model might incur additional costs over time
|
||||
deploy_model = False
|
||||
|
||||
all_columns = [label_column] + feature_columns
|
||||
|
||||
dataset = download_from_gcs_op(
|
||||
gcs_path=dataset_gcs_uri
|
||||
).outputs["Data"]
|
||||
|
||||
dataset = select_columns_using_Pandas_on_CSV_data_op(
|
||||
table=dataset,
|
||||
column_names=all_columns,
|
||||
).outputs["transformed_table"]
|
||||
|
||||
dataset = fill_all_missing_values_using_Pandas_on_CSV_data_op(
|
||||
table=dataset,
|
||||
replacement_value="0",
|
||||
# # Optional:
|
||||
# column_names=None, # =[...]
|
||||
).outputs["transformed_table"]
|
||||
|
||||
split_task = split_rows_into_subsets_op(
|
||||
table=dataset,
|
||||
fraction_1=training_set_fraction,
|
||||
)
|
||||
training_data = split_task.outputs["split_1"]
|
||||
testing_data = split_task.outputs["split_2"]
|
||||
|
||||
model = train_XGBoost_model_on_CSV_op(
|
||||
training_data=training_data,
|
||||
label_column_name=label_column,
|
||||
# Optional:
|
||||
#starting_model=None,
|
||||
#num_iterations=10,
|
||||
#booster_params={},
|
||||
#objective="reg:squarederror",
|
||||
#booster="gbtree",
|
||||
#learning_rate=0.3,
|
||||
#min_split_loss=0,
|
||||
#max_depth=6,
|
||||
).outputs["model"]
|
||||
|
||||
# Predicting on the testing data
|
||||
predictions = xgboost_predict_on_CSV_op(
|
||||
data=testing_data,
|
||||
model=model,
|
||||
# label_column needs to be set when doing prediction on a dataset that has labels
|
||||
label_column_name=label_column,
|
||||
).outputs["predictions"]
|
||||
|
||||
vertex_model_name = upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op(
|
||||
model=model,
|
||||
).outputs["model_name"]
|
||||
|
||||
# Deploying the model might incur additional costs over time
|
||||
if deploy_model:
|
||||
vertex_endpoint_name = deploy_model_to_endpoint_op(
|
||||
model_name=vertex_model_name,
|
||||
).outputs["endpoint_name"]
|
||||
|
||||
pipeline_func = train_tabular_regression_model_using_XGBoost_pipeline
|
||||
|
||||
# %% Pipeline submission
|
||||
if __name__ == '__main__':
|
||||
from google.cloud import aiplatform
|
||||
aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()
|
||||
-238
@@ -1,238 +0,0 @@
|
||||
name: Train tabular regression model using all frameworks pipeline
|
||||
metadata:
|
||||
annotations:
|
||||
author: Alexey Volkov <alexey.volkov@ark-kun.com>
|
||||
canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_regression_model_using_all_frameworks_and_import_to_Vertex_AI/pipeline.component.yaml
|
||||
sdk: https://cloud-pipelines.net/pipeline-editor/
|
||||
implementation:
|
||||
graph:
|
||||
tasks:
|
||||
Download from GCS:
|
||||
componentRef:
|
||||
digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
|
||||
annotations:
|
||||
editor.position: '{"x":550,"y":40,"width":180,"height":40}'
|
||||
Select columns using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: Data
|
||||
taskId: Download from GCS
|
||||
column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
|
||||
annotations:
|
||||
editor.position: '{"x":550,"y":140,"width":180,"height":54}'
|
||||
Fill all missing values using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Select columns using Pandas on CSV data
|
||||
type: CSV
|
||||
replacement_value: '0'
|
||||
annotations:
|
||||
editor.position: '{"x":550,"y":250,"width":180,"height":54}'
|
||||
Split rows into subsets:
|
||||
componentRef:
|
||||
digest: a609c3c9196484290f24a1174955f95b27f07a7b458aa5cb8cde28866cb2cb46
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Fill all missing values using Pandas on CSV data
|
||||
type: CSV
|
||||
fraction_1: '0.8'
|
||||
annotations:
|
||||
editor.position: '{"x":550,"y":360,"width":180,"height":40}'
|
||||
Create fully connected pytorch network:
|
||||
componentRef:
|
||||
digest: d03d8248fd358a0275ec33568ee7dd7dce576cc112b09dfafe2651e4d97e04a9
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1a2ef3eeb77bc278f33cad0dd29008ea2431e191/components/PyTorch/Create_fully_connected_network/component.yaml
|
||||
arguments:
|
||||
input_size: '7'
|
||||
hidden_layer_sizes: '[10]'
|
||||
activation_name: elu
|
||||
annotations:
|
||||
editor.position: '{"x":380,"y":490,"width":180,"height":54}'
|
||||
Create fully connected tensorflow network:
|
||||
componentRef:
|
||||
digest: bfcafbc5ce711b1f69cabf1338212d10d50136a73db9f9f7c984de7b80b4bfb0
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/9ca0f9eecf5f896f65b8538bbd809747052617d1/components/tensorflow/Create_fully_connected_network/component.yaml
|
||||
arguments:
|
||||
input_size: '7'
|
||||
hidden_layer_sizes: '[10]'
|
||||
activation_name: elu
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":500,"width":180,"height":54}'
|
||||
Train model using Keras on CSV:
|
||||
componentRef:
|
||||
digest: 42ae60c889034dbad74815653e95b4f7d576b5f47f803173e8679c7b54984609
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c504a4010348c50eaaf6d4337586ccc008f4dcef/components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml
|
||||
arguments:
|
||||
training_data:
|
||||
taskOutput:
|
||||
outputName: split_1
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Create fully connected tensorflow network
|
||||
type: TensorflowSavedModel
|
||||
label_column_name: tips
|
||||
number_of_epochs: '10'
|
||||
metric_names: '["mean_absolute_error"]'
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":620,"width":180,"height":54}'
|
||||
Train pytorch model from csv:
|
||||
componentRef:
|
||||
digest: 40f3185eb61e9727f41a4e0c05dd3d3b44bd802aa0f378cfc31756560033949a
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml
|
||||
arguments:
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Create fully connected pytorch network
|
||||
type: PyTorchScriptModule
|
||||
training_data:
|
||||
taskOutput:
|
||||
outputName: split_1
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
label_column_name: tips
|
||||
annotations:
|
||||
editor.position: '{"x":380,"y":620,"width":180,"height":40}'
|
||||
Train XGBoost model on CSV:
|
||||
componentRef:
|
||||
digest: 538c5a01eb38deaf532d619f0bbeaff4efc550fe1f0f776fc06791097b68ceac
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/58d3a47f904f32a64af8403330ba7e2134cae46d/components/XGBoost/Train/component.yaml
|
||||
arguments:
|
||||
training_data:
|
||||
taskOutput:
|
||||
outputName: split_1
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
label_column_name: tips
|
||||
annotations:
|
||||
editor.position: '{"x":720,"y":620,"width":180,"height":40}'
|
||||
Train linear regression model using scikit learn from CSV:
|
||||
componentRef:
|
||||
digest: c7fe7912ab0d1fb45d201d452e9ce6be5544e7d8c6d229db7a4b931ff58560f3
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/f807e02b54d4886c65a05f40848fd51c72407f40/components/ML_frameworks/Scikit_learn/Train_linear_regression_model/from_CSV/component.yaml
|
||||
arguments:
|
||||
dataset:
|
||||
taskOutput:
|
||||
outputName: split_1
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
label_column_name: tips
|
||||
annotations:
|
||||
editor.position: '{"x":1030,"y":620,"width":180,"height":54}'
|
||||
Predict with TensorFlow model on CSV data:
|
||||
componentRef:
|
||||
digest: 921bb1563e93a78233b8acceab87055b9154ccf5595d056028cf0396ca224cd4
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/59c759ce6f543184e30db6817d2a703879bc0f39/components/tensorflow/Predict/on_CSV/component.yaml
|
||||
arguments:
|
||||
dataset:
|
||||
taskOutput:
|
||||
outputName: split_2
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: trained_model
|
||||
taskId: Train model using Keras on CSV
|
||||
type: TensorflowSavedModel
|
||||
label_column_name: tips
|
||||
annotations:
|
||||
editor.position: '{"x":160,"y":750,"width":180,"height":54}'
|
||||
Create PyTorch Model Archive with base handler:
|
||||
componentRef:
|
||||
digest: 8298b5ee1b0f0879f893add4cf352c8dec7cf9e21bb9db134c91a2d046cdb0ec
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/46d51383e6554b7f3ab4fd8cf614d8c2b422fb22/components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml
|
||||
arguments:
|
||||
Model:
|
||||
taskOutput:
|
||||
outputName: trained_model
|
||||
taskId: Train pytorch model from csv
|
||||
type: PyTorchScriptModule
|
||||
Model name: model
|
||||
Model version: '1.0'
|
||||
annotations:
|
||||
editor.position: '{"x":380,"y":750,"width":180,"height":54}'
|
||||
Xgboost predict on CSV:
|
||||
componentRef:
|
||||
digest: 0876233a0c7306fefec188bd70f059b46d1fb5aa57be231799570e3bbbdd0d95
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/4694ec97baccf59284c2a1db4aa2250c22291eab/components/XGBoost/Predict/component.yaml
|
||||
arguments:
|
||||
data:
|
||||
taskOutput:
|
||||
outputName: split_2
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Train XGBoost model on CSV
|
||||
type: XGBoostModel
|
||||
label_column_name: tips
|
||||
annotations:
|
||||
editor.position: '{"x":810,"y":750,"width":180,"height":40}'
|
||||
Upload Scikit learn pickle model to Google Cloud Vertex AI:
|
||||
componentRef:
|
||||
digest: 81c91c8d7d21ec97e0872f669d68bd89edea87279d703685db54aa94743bebcd
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Train linear regression model using scikit learn from CSV
|
||||
type: ScikitLearnPickleModel
|
||||
annotations:
|
||||
editor.position: '{"x":1030,"y":750,"width":180,"height":70}'
|
||||
Upload Tensorflow model to Google Cloud Vertex AI:
|
||||
componentRef:
|
||||
digest: 2e45263ff640b1a688e359b6936e27a81b2407749a84f340af2aa5547e0cb92c
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: trained_model
|
||||
taskId: Train model using Keras on CSV
|
||||
type: TensorflowSavedModel
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":880,"width":180,"height":54}'
|
||||
Upload PyTorch model archive to Google Cloud Vertex AI:
|
||||
componentRef:
|
||||
digest: 4450212fae7b9001482aca7eb78b28413c205506eccf08a04e7754a8dfa99004
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
model_archive:
|
||||
taskOutput:
|
||||
outputName: Model archive
|
||||
taskId: Create PyTorch Model Archive with base handler
|
||||
type: PyTorchModelArchive
|
||||
annotations:
|
||||
editor.position: '{"x":380,"y":880,"width":180,"height":70}'
|
||||
Upload XGBoost model to Google Cloud Vertex AI:
|
||||
componentRef:
|
||||
digest: 5a5a273c403670743820986c03a4175b7cb4595a556524fefcce403656286977
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Train XGBoost model on CSV
|
||||
type: XGBoostModel
|
||||
annotations:
|
||||
editor.position: '{"x":720,"y":880,"width":180,"height":54}'
|
||||
outputValues: {}
|
||||
-208
@@ -1,208 +0,0 @@
|
||||
# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
|
||||
from kfp import components
|
||||
|
||||
# %% Loading components
|
||||
download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml")
|
||||
fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
|
||||
split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
|
||||
|
||||
# TensorFlow
|
||||
create_fully_connected_tensorflow_network_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/9ca0f9eecf5f896f65b8538bbd809747052617d1/components/tensorflow/Create_fully_connected_network/component.yaml")
|
||||
train_model_using_Keras_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c504a4010348c50eaaf6d4337586ccc008f4dcef/components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml")
|
||||
predict_with_TensorFlow_model_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/59c759ce6f543184e30db6817d2a703879bc0f39/components/tensorflow/Predict/on_CSV/component.yaml")
|
||||
upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
|
||||
# PyTorch
|
||||
create_fully_connected_pytorch_network_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/1a2ef3eeb77bc278f33cad0dd29008ea2431e191/components/PyTorch/Create_fully_connected_network/component.yaml")
|
||||
train_pytorch_model_from_csv_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml")
|
||||
create_pytorch_model_archive_with_base_handler_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/46d51383e6554b7f3ab4fd8cf614d8c2b422fb22/components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml")
|
||||
upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
|
||||
# XGBoost
|
||||
train_XGBoost_model_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/58d3a47f904f32a64af8403330ba7e2134cae46d/components/XGBoost/Train/component.yaml")
|
||||
xgboost_predict_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/4694ec97baccf59284c2a1db4aa2250c22291eab/components/XGBoost/Predict/component.yaml")
|
||||
upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
|
||||
# Scikit-learn
|
||||
train_linear_regression_model_using_scikit_learn_from_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/f807e02b54d4886c65a05f40848fd51c72407f40/components/ML_frameworks/Scikit_learn/Train_linear_regression_model/from_CSV/component.yaml")
|
||||
upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
|
||||
# Vertex AI
|
||||
deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
|
||||
# %% Pipeline definition
|
||||
def train_tabular_regression_model_using_all_frameworks_pipeline():
|
||||
dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
|
||||
feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"] # Excluded "trip_total"
|
||||
label_column = "tips"
|
||||
training_set_fraction = 0.8
|
||||
# Deploying the model might incur additional costs over time
|
||||
deploy_model = False
|
||||
|
||||
all_columns = [label_column] + feature_columns
|
||||
|
||||
dataset = download_from_gcs_op(
|
||||
gcs_path=dataset_gcs_uri
|
||||
).outputs["Data"]
|
||||
|
||||
dataset = select_columns_using_Pandas_on_CSV_data_op(
|
||||
table=dataset,
|
||||
column_names=all_columns,
|
||||
).outputs["transformed_table"]
|
||||
|
||||
dataset = fill_all_missing_values_using_Pandas_on_CSV_data_op(
|
||||
table=dataset,
|
||||
replacement_value="0",
|
||||
# # Optional:
|
||||
# column_names=None, # =[...]
|
||||
).outputs["transformed_table"]
|
||||
|
||||
split_task = split_rows_into_subsets_op(
|
||||
table=dataset,
|
||||
fraction_1=training_set_fraction,
|
||||
)
|
||||
training_data = split_task.outputs["split_1"]
|
||||
testing_data = split_task.outputs["split_2"]
|
||||
|
||||
# TensorFlow
|
||||
tensorflow_network = create_fully_connected_tensorflow_network_op(
|
||||
input_size=len(feature_columns),
|
||||
# Optional:
|
||||
hidden_layer_sizes=[10],
|
||||
activation_name="elu",
|
||||
# output_activation_name=None,
|
||||
# output_size=1,
|
||||
).outputs["model"]
|
||||
|
||||
tensorflow_model = train_model_using_Keras_on_CSV_op(
|
||||
training_data=training_data,
|
||||
model=tensorflow_network,
|
||||
label_column_name=label_column,
|
||||
# Optional:
|
||||
#loss_function_name="mean_squared_error",
|
||||
number_of_epochs=10,
|
||||
#learning_rate=0.1,
|
||||
#optimizer_name="Adadelta",
|
||||
#optimizer_parameters={},
|
||||
#batch_size=32,
|
||||
metric_names=["mean_absolute_error"],
|
||||
#random_seed=0,
|
||||
).outputs["trained_model"]
|
||||
|
||||
tensorflow_predictions = predict_with_TensorFlow_model_on_CSV_data_op(
|
||||
dataset=testing_data,
|
||||
model=tensorflow_model,
|
||||
# label_column_name needs to be set when doing prediction on a dataset that has labels
|
||||
label_column_name=label_column,
|
||||
# Optional:
|
||||
# batch_size=1000,
|
||||
).outputs["predictions"]
|
||||
|
||||
tensorflow_vertex_model_name = upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op(
|
||||
model=tensorflow_model,
|
||||
).outputs["model_name"]
|
||||
|
||||
# Deploying the model might incur additional costs over time
|
||||
if deploy_model:
|
||||
tensorflow_vertex_endpoint_name = deploy_model_to_endpoint_op(
|
||||
model_name=tensorflow_vertex_model_name,
|
||||
).outputs["endpoint_name"]
|
||||
|
||||
# PyTorch
|
||||
pytorch_network = create_fully_connected_pytorch_network_op(
|
||||
input_size=len(feature_columns),
|
||||
# Optional:
|
||||
hidden_layer_sizes=[10],
|
||||
activation_name="elu",
|
||||
# output_activation_name=None,
|
||||
# output_size=1,
|
||||
).outputs["model"]
|
||||
|
||||
pytorch_model = train_pytorch_model_from_csv_op(
|
||||
model=pytorch_network,
|
||||
training_data=training_data,
|
||||
label_column_name=label_column,
|
||||
# Optional:
|
||||
#loss_function_name="mse_loss",
|
||||
#number_of_epochs=1,
|
||||
#learning_rate=0.1,
|
||||
#optimizer_name="Adadelta",
|
||||
#optimizer_parameters={},
|
||||
#batch_size=32,
|
||||
#batch_log_interval=100,
|
||||
#random_seed=0,
|
||||
).outputs["trained_model"]
|
||||
|
||||
pytorch_model_archive = create_pytorch_model_archive_with_base_handler_op(
|
||||
model=pytorch_model,
|
||||
# Optional:
|
||||
# model_name="model",
|
||||
# model_version="1.0",
|
||||
).outputs["Model archive"]
|
||||
|
||||
pytorch_vertex_model_name = upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op(
|
||||
model_archive=pytorch_model_archive,
|
||||
).outputs["model_name"]
|
||||
|
||||
# Deploying the model might incur additional costs over time
|
||||
if deploy_model:
|
||||
pytorch_vertex_endpoint_name = deploy_model_to_endpoint_op(
|
||||
model_name=pytorch_vertex_model_name,
|
||||
).outputs["endpoint_name"]
|
||||
|
||||
# XGBoost
|
||||
xgboost_model = train_XGBoost_model_on_CSV_op(
|
||||
training_data=training_data,
|
||||
label_column_name=label_column,
|
||||
# Optional:
|
||||
#starting_model=None,
|
||||
#num_iterations=10,
|
||||
#booster_params={},
|
||||
#objective="reg:squarederror",
|
||||
#booster="gbtree",
|
||||
#learning_rate=0.3,
|
||||
#min_split_loss=0,
|
||||
#max_depth=6,
|
||||
).outputs["model"]
|
||||
|
||||
# Predicting on the testing data
|
||||
xgboost_predictions = xgboost_predict_on_CSV_op(
|
||||
data=testing_data,
|
||||
model=xgboost_model,
|
||||
# label_column needs to be set when doing prediction on a dataset that has labels
|
||||
label_column_name=label_column,
|
||||
).outputs["predictions"]
|
||||
|
||||
xgboost_vertex_model_name = upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op(
|
||||
model=xgboost_model,
|
||||
).outputs["model_name"]
|
||||
|
||||
# Deploying the model might incur additional costs over time
|
||||
if deploy_model:
|
||||
xgboost_vertex_endpoint_name = deploy_model_to_endpoint_op(
|
||||
model_name=xgboost_vertex_model_name,
|
||||
).outputs["endpoint_name"]
|
||||
|
||||
# Scikit-learn
|
||||
sklearn_model = train_linear_regression_model_using_scikit_learn_from_CSV_op(
|
||||
dataset=training_data,
|
||||
label_column_name=label_column,
|
||||
).outputs["model"]
|
||||
|
||||
sklearn_vertex_model_name = upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op(
|
||||
model=sklearn_model,
|
||||
).outputs["model_name"]
|
||||
|
||||
# Deploying the model might incur additional costs over time
|
||||
if deploy_model:
|
||||
sklearn_vertex_endpoint_name = deploy_model_to_endpoint_op(
|
||||
model_name=sklearn_vertex_model_name,
|
||||
).outputs["endpoint_name"]
|
||||
|
||||
pipeline_func=train_tabular_regression_model_using_all_frameworks_pipeline
|
||||
|
||||
# %% Pipeline submission
|
||||
if __name__ == '__main__':
|
||||
from google.cloud import aiplatform
|
||||
aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()
|
||||
@@ -34,4 +34,3 @@
|
||||
/notebooks/community/vertex-ai-samples/notebooks/community/model_registry/vertex_ai_model_registry_bqml_custom_model_versioning.ipynb @inardini
|
||||
/notebooks/community/vertex-ai-samples/notebooks/community/model_registry/vertex_ai_model_registry_automl_model_versioning.ipynb @inardini
|
||||
/notebooks/community/vizier/conversions_vertex_vizier_and_open_source_vizier.ipynb @halio-g
|
||||
/notebooks/community/experiments/vertex_ai_model_experimentation.ipynb @inardini @asobran
|
||||
File diff suppressed because it is too large
Load Diff
-974
@@ -1,974 +0,0 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "copyright"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Copyright 2022 Google LLC\n",
|
||||
"#\n",
|
||||
"# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
|
||||
"# you may not use this file except in compliance with the License.\n",
|
||||
"# You may obtain a copy of the License at\n",
|
||||
"#\n",
|
||||
"# https://www.apache.org/licenses/LICENSE-2.0\n",
|
||||
"#\n",
|
||||
"# Unless required by applicable law or agreed to in writing, software\n",
|
||||
"# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
|
||||
"# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
|
||||
"# See the License for the specific language governing permissions and\n",
|
||||
"# limitations under the License."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "title"
|
||||
},
|
||||
"source": [
|
||||
"# E2E ML on GCP: MLOps stage 2 : Get started with autologging using Vertex AI Experiments for TensorFlow models\n",
|
||||
"\n",
|
||||
"<table align=\"left\">\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_with_vertex_experiments_autologging_tf.ipynb\">\n",
|
||||
" <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_with_vertex_experiments_autologging_tf.ipynb\">\n",
|
||||
" <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/community/ml_ops/stage2/get_started_with_vertex_experiments_autologging_tf.ipynb\">\n",
|
||||
" <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
|
||||
" Open in Vertex AI Workbench\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
"</table>\n",
|
||||
"<br/><br/><br/>"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "overview:automl"
|
||||
},
|
||||
"source": [
|
||||
"## Overview\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"This tutorial demonstrates how to use the `Vertex AI Experiments` with DIY code to implement automatic logging of parameters and metrics for experiments."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "objective:automl,training,batch_prediction"
|
||||
},
|
||||
"source": [
|
||||
"### Objective\n",
|
||||
"\n",
|
||||
"In this tutorial, you learn how to create an experiment for training a TensorFlow model, and automatically log parameters and metrics using the enclosed do-it-yourself (DIY) code.\n",
|
||||
"\n",
|
||||
"This tutorial uses the following Google Cloud ML services and resources:\n",
|
||||
"\n",
|
||||
"- `Vertex AI Experiments`\n",
|
||||
"\n",
|
||||
"The steps performed include:\n",
|
||||
"\n",
|
||||
"- Construct the DIY autologging code.\n",
|
||||
"- Construct training package with call to autologging.\n",
|
||||
"- Train a model.\n",
|
||||
"- View the experiment\n",
|
||||
"- Delete the experiment."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "2739272aae1b"
|
||||
},
|
||||
"source": [
|
||||
"### Model\n",
|
||||
"\n",
|
||||
"The model used for this tutorial is a pretrain TensorFlow model that was trained on the [Boston Housing Prices dataset](https://www.cs.toronto.edu/~delve/data/boston/bostonDetail.html). The version of the dataset this tutorial is built into TensorFlow. The trained model predicts the median price of a house in units of 1K USD."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "costs"
|
||||
},
|
||||
"source": [
|
||||
"### Costs\n",
|
||||
"\n",
|
||||
"This tutorial uses billable components of Google Cloud:\n",
|
||||
"\n",
|
||||
"* Vertex AI\n",
|
||||
"* Cloud Storage\n",
|
||||
"\n",
|
||||
"Learn about [Vertex AI\n",
|
||||
"pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n",
|
||||
"pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n",
|
||||
"Calculator](https://cloud.google.com/products/calculator/)\n",
|
||||
"to generate a cost estimate based on your projected usage."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "setup_local"
|
||||
},
|
||||
"source": [
|
||||
"### Set up your local development environment\n",
|
||||
"\n",
|
||||
"If you are using Colab or Vertex Workbench AI Notebooks, your environment already meets all the requirements to run this notebook. You can skip this step.\n",
|
||||
"\n",
|
||||
"Otherwise, make sure your environment meets this notebook's requirements. You need the following:\n",
|
||||
"\n",
|
||||
"- The Cloud Storage SDK\n",
|
||||
"- Git\n",
|
||||
"- Python 3\n",
|
||||
"- virtualenv\n",
|
||||
"- Jupyter notebook running in a virtual environment with Python 3\n",
|
||||
"\n",
|
||||
"The Cloud Storage guide to [Setting up a Python development environment](https://cloud.google.com/python/setup) and the [Jupyter installation guide](https://jupyter.org/install) provide detailed instructions for meeting these requirements. The following steps provide a condensed set of instructions:\n",
|
||||
"\n",
|
||||
"1. [Install and initialize the SDK](https://cloud.google.com/sdk/docs/).\n",
|
||||
"\n",
|
||||
"2. [Install Python 3](https://cloud.google.com/python/setup#installing_python).\n",
|
||||
"\n",
|
||||
"3. [Install virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv) and create a virtual environment that uses Python 3. Activate the virtual environment.\n",
|
||||
"\n",
|
||||
"4. To install Jupyter, run `pip3 install jupyter` on the command-line in a terminal shell.\n",
|
||||
"\n",
|
||||
"5. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
|
||||
"\n",
|
||||
"6. Open this notebook in the Jupyter Notebook Dashboard.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "install_aip:mbsdk"
|
||||
},
|
||||
"source": [
|
||||
"## Installation\n",
|
||||
"\n",
|
||||
"Install the following packages to execute this notebook."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "install_aip:mbsdk"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"\n",
|
||||
"# The Vertex AI Workbench Notebook product has specific requirements\n",
|
||||
"IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
|
||||
"IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
|
||||
" \"/opt/deeplearning/metadata/env_version\"\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
|
||||
"USER_FLAG = \"\"\n",
|
||||
"if IS_WORKBENCH_NOTEBOOK:\n",
|
||||
" USER_FLAG = \"--user\"\n",
|
||||
"\n",
|
||||
"! pip3 install --upgrade --quiet {USER_FLAG} google-cloud-aiplatform \\\n",
|
||||
" tensorflow==2.5 \\\n",
|
||||
" numpy"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "restart"
|
||||
},
|
||||
"source": [
|
||||
"### Restart the kernel\n",
|
||||
"\n",
|
||||
"After you install the additional packages, you need to restart the notebook kernel so it can find the packages."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "restart"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"\n",
|
||||
"if not os.getenv(\"IS_TESTING\"):\n",
|
||||
" # Automatically restart kernel after installs\n",
|
||||
" import IPython\n",
|
||||
"\n",
|
||||
" app = IPython.Application.instance()\n",
|
||||
" app.kernel.do_shutdown(True)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "before_you_begin:nogpu"
|
||||
},
|
||||
"source": [
|
||||
"## Before you begin\n",
|
||||
"\n",
|
||||
"### GPU runtime\n",
|
||||
"\n",
|
||||
"This tutorial does not require a GPU runtime.\n",
|
||||
"\n",
|
||||
"### Set up your Google Cloud project\n",
|
||||
"\n",
|
||||
"**The following steps are required, regardless of your notebook environment.**\n",
|
||||
"\n",
|
||||
"1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
|
||||
"\n",
|
||||
"2. [Make sure that billing is enabled for your project.](https://cloud.google.com/billing/docs/how-to/modify-project)\n",
|
||||
"\n",
|
||||
"3. [Enable the following APIs: Vertex AI APIs, Compute Engine APIs, and Cloud Storage.](https://console.cloud.google.com/flows/enableapi?apiid=ml.googleapis.com,compute_component,storage-component.googleapis.com)\n",
|
||||
"\n",
|
||||
"4. If you are running this notebook locally, you need to install the [Cloud SDK]((https://cloud.google.com/sdk)).\n",
|
||||
"\n",
|
||||
"5. Enter your project ID in the cell below. Then run the cell to make sure the\n",
|
||||
"Cloud SDK uses the right project for all the commands in this notebook.\n",
|
||||
"\n",
|
||||
"**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$`."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "project_id"
|
||||
},
|
||||
"source": [
|
||||
"#### Set your project ID\n",
|
||||
"\n",
|
||||
"**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "set_project_id"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"PROJECT_ID = \"[your-project-id]\" # @param {type:\"string\"}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "autoset_project_id"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
|
||||
" # Get your GCP project id from gcloud\n",
|
||||
" shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
|
||||
" PROJECT_ID = shell_output[0]\n",
|
||||
" print(\"Project ID:\", PROJECT_ID)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "set_gcloud_project_id"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gcloud config set project $PROJECT_ID"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "region"
|
||||
},
|
||||
"source": [
|
||||
"#### Region\n",
|
||||
"\n",
|
||||
"You can also change the `REGION` variable, which is used for operations\n",
|
||||
"throughout the rest of this notebook. Below are regions supported for Vertex AI. We recommend that you choose the region closest to you.\n",
|
||||
"\n",
|
||||
"- Americas: `us-central1`\n",
|
||||
"- Europe: `europe-west4`\n",
|
||||
"- Asia Pacific: `asia-east1`\n",
|
||||
"\n",
|
||||
"You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
|
||||
"\n",
|
||||
"Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "region"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"REGION = \"[your-region]\" # @param {type: \"string\"}\n",
|
||||
"\n",
|
||||
"if REGION == \"[your-region]\":\n",
|
||||
" REGION = \"us-central1\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "timestamp"
|
||||
},
|
||||
"source": [
|
||||
"#### UUID\n",
|
||||
"\n",
|
||||
"If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "timestamp"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import random\n",
|
||||
"import string\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Generate a uuid of a specifed length(default=8)\n",
|
||||
"def generate_uuid(length: int = 8) -> str:\n",
|
||||
" return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"UUID = generate_uuid()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "gcp_authenticate"
|
||||
},
|
||||
"source": [
|
||||
"### Authenticate your Google Cloud account\n",
|
||||
"\n",
|
||||
"**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. \n",
|
||||
"\n",
|
||||
"**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
|
||||
"\n",
|
||||
"**Otherwise**, follow these steps:\n",
|
||||
"\n",
|
||||
"In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
|
||||
"\n",
|
||||
"**Click Create service account**.\n",
|
||||
"\n",
|
||||
"In the **Service account name** field, enter a name, and click **Create**.\n",
|
||||
"\n",
|
||||
"In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex\" into the filter box, and select **Vertex Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
|
||||
"\n",
|
||||
"Click Create. A JSON file that contains your key downloads to your local environment.\n",
|
||||
"\n",
|
||||
"Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "gcp_authenticate"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# If you are running this notebook in Colab, run this cell and follow the\n",
|
||||
"# instructions to authenticate your GCP account. This provides access to your\n",
|
||||
"# Cloud Storage bucket and lets you submit training jobs and prediction\n",
|
||||
"# requests.\n",
|
||||
"\n",
|
||||
"import os\n",
|
||||
"import sys\n",
|
||||
"\n",
|
||||
"# If on Vertex AI Workbench, then don't execute this code\n",
|
||||
"IS_COLAB = \"google.colab\" in sys.modules\n",
|
||||
"if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
|
||||
" \"DL_ANACONDA_HOME\"\n",
|
||||
"):\n",
|
||||
" if \"google.colab\" in sys.modules:\n",
|
||||
" from google.colab import auth as google_auth\n",
|
||||
"\n",
|
||||
" google_auth.authenticate_user()\n",
|
||||
"\n",
|
||||
" # If you are running this notebook locally, replace the string below with the\n",
|
||||
" # path to your service account key and run this cell to authenticate your GCP\n",
|
||||
" # account.\n",
|
||||
" elif not os.getenv(\"IS_TESTING\"):\n",
|
||||
" %env GOOGLE_APPLICATION_CREDENTIALS ''"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "setup_vars"
|
||||
},
|
||||
"source": [
|
||||
"### Import libraries and define constants"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "59963fb7178f"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import google.cloud.aiplatform as aiplatform\n",
|
||||
"import numpy as np\n",
|
||||
"import tensorflow as tf"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "init_aip:mbsdk"
|
||||
},
|
||||
"source": [
|
||||
"## Initialize Vertex AI SDK for Python\n",
|
||||
"\n",
|
||||
"Initialize the Vertex AI SDK for Python for your project and corresponding bucket."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "init_aip:mbsdk"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"aiplatform.init(project=PROJECT_ID, location=REGION)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "ae8f31c8c617"
|
||||
},
|
||||
"source": [
|
||||
"## DIY code for autologging TensorFlow Keras models\n",
|
||||
"\n",
|
||||
"The code below implements autologging for TensorFlow models using the Keras API.\n",
|
||||
"\n",
|
||||
"- `autologging()`: Initializes the experiment and uses heap injection to replace TF.keras `Sequential` and `Model` symbols on the heap with the redirect wrapper classes `VertexTFSequential` and `VertexTFModel`, respectively.\n",
|
||||
"\n",
|
||||
"- `VertexTFSequential`: A subclass of the tf.keras.Sequential class.\n",
|
||||
" - `compile()`: overridden method of super class. Automatically logs specified hyperparameters and calls the underlying `compile()` method.\n",
|
||||
" - `fit()`: overridden method of super class. Automatically logs specified hyperparameters, calls the underlying `fit()` method, and logs the resulting metrics.\n",
|
||||
" - `evaluate()`: overridden method of super class. Calls the underlying `evaluate()` method, and logs the resulting metrics.\n",
|
||||
"- `VertexTFModel`: A subclass of the tf.keras.Model class."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "8eb012e5d7ef"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def autolog(\n",
|
||||
" project: str = None,\n",
|
||||
" location: str = None,\n",
|
||||
" staging_bucket: str = None,\n",
|
||||
" experiment: str = None,\n",
|
||||
" run: str = None,\n",
|
||||
" framework: str = \"tf\",\n",
|
||||
"):\n",
|
||||
" \"\"\"\n",
|
||||
" Enable automatic logging of parameters and metrics in Vertex AI Experiments,\n",
|
||||
" for corresponding framework.\n",
|
||||
"\n",
|
||||
" project: The project ID\n",
|
||||
" location : The region\n",
|
||||
" staging_bucket: temporary bucket\n",
|
||||
" experiment: The name of the experiment\n",
|
||||
" run: The name of the run within the experiment\n",
|
||||
" framework: The ML framework for which a model is being trained.\n",
|
||||
" \"\"\"\n",
|
||||
" # autologging\n",
|
||||
" if framework == \"tf\":\n",
|
||||
" try:\n",
|
||||
" globals()[\"Sequential\"] = VertexTFSequential\n",
|
||||
" if \"tf\" in globals():\n",
|
||||
" tf.keras.Sequential = VertexTFSequential\n",
|
||||
" if \"tensorflow\" in globals():\n",
|
||||
" tensorflow.keras.Sequential = VertexTFSequential\n",
|
||||
" except:\n",
|
||||
" pass\n",
|
||||
"\n",
|
||||
" try:\n",
|
||||
" globals()[\"Model\"] = VertexTFModel\n",
|
||||
" if \"tf\" in globals():\n",
|
||||
" tf.keras.Model = VertexTFModel\n",
|
||||
" if \"tensorflow\" in globals():\n",
|
||||
" tensorflow.keras.Model = VertexTFModel\n",
|
||||
" except:\n",
|
||||
" pass\n",
|
||||
"\n",
|
||||
" if project:\n",
|
||||
" aiplatform.init(\n",
|
||||
" project=project, location=location, staging_bucket=staging_bucket\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" if experiment:\n",
|
||||
" aiplatform.init(experiment=experiment)\n",
|
||||
" if run:\n",
|
||||
" aiplatform.start_run(run)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"class VertexTFSequential(tf.keras.Sequential):\n",
|
||||
" \"\"\"\n",
|
||||
" Sublass of the tensorflow.keras.Sequential model type. Overrides with pass thru of\n",
|
||||
" underlying super class methods to automatically log parameters/metrics for Vertex AI experiments.\n",
|
||||
"\n",
|
||||
" compile():\n",
|
||||
" fit():\n",
|
||||
" evaluate():\n",
|
||||
" \"\"\"\n",
|
||||
"\n",
|
||||
" def __init__(self, layers):\n",
|
||||
" return super().__init__(layers)\n",
|
||||
"\n",
|
||||
" def compile(\n",
|
||||
" self,\n",
|
||||
" optimizer=\"rmsprop\",\n",
|
||||
" loss=None,\n",
|
||||
" metrics=None,\n",
|
||||
" loss_weights=None,\n",
|
||||
" weighted_metrics=None,\n",
|
||||
" run_eagerly=None,\n",
|
||||
" steps_per_execution=None,\n",
|
||||
" ):\n",
|
||||
" try:\n",
|
||||
" learning_rate = optimizer.learning_rate.numpy()\n",
|
||||
" aiplatform.log_params({\"train.learning_rate\": float(learning_rate)})\n",
|
||||
" except:\n",
|
||||
" pass\n",
|
||||
" return super().compile(\n",
|
||||
" loss=loss,\n",
|
||||
" optimizer=optimizer,\n",
|
||||
" metrics=metrics,\n",
|
||||
" loss_weights=loss_weights,\n",
|
||||
" weighted_metrics=weighted_metrics,\n",
|
||||
" run_eagerly=run_eagerly,\n",
|
||||
" steps_per_execution=steps_per_execution,\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" def fit(\n",
|
||||
" self,\n",
|
||||
" x=None,\n",
|
||||
" y=None,\n",
|
||||
" batch_size=None,\n",
|
||||
" epochs=1,\n",
|
||||
" verbose=\"auto\",\n",
|
||||
" callbacks=None,\n",
|
||||
" validation_split=0.0,\n",
|
||||
" validation_data=None,\n",
|
||||
" shuffle=True,\n",
|
||||
" class_weight=None,\n",
|
||||
" sample_weight=None,\n",
|
||||
" initial_epoch=0,\n",
|
||||
" steps_per_epoch=None,\n",
|
||||
" validation_steps=None,\n",
|
||||
" validation_batch_size=None,\n",
|
||||
" validation_freq=1,\n",
|
||||
" max_queue_size=10,\n",
|
||||
" workers=1,\n",
|
||||
" use_multiprocessing=False,\n",
|
||||
" ):\n",
|
||||
" aiplatform.log_params({\"train.epochs\": int(epochs)})\n",
|
||||
" if batch_size:\n",
|
||||
" aiplatform.log_params({\"train.batch_size\": int(batch_size)})\n",
|
||||
" if steps_per_epoch:\n",
|
||||
" aiplatform.log_params({\"train.steps\": int(steps_per_epoch)})\n",
|
||||
"\n",
|
||||
" history = super().fit(\n",
|
||||
" x=x,\n",
|
||||
" y=y,\n",
|
||||
" batch_size=batch_size,\n",
|
||||
" epochs=epochs,\n",
|
||||
" verbose=verbose,\n",
|
||||
" callbacks=callbacks,\n",
|
||||
" validation_split=validation_split,\n",
|
||||
" validation_data=validation_data,\n",
|
||||
" shuffle=shuffle,\n",
|
||||
" class_weight=class_weight,\n",
|
||||
" sample_weight=sample_weight,\n",
|
||||
" initial_epoch=initial_epoch,\n",
|
||||
" steps_per_epoch=steps_per_epoch,\n",
|
||||
" validation_steps=validation_steps,\n",
|
||||
" validation_batch_size=validation_batch_size,\n",
|
||||
" validation_freq=validation_freq,\n",
|
||||
" max_queue_size=max_queue_size,\n",
|
||||
" workers=workers,\n",
|
||||
" use_multiprocessing=use_multiprocessing,\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" for key, val in history.history.items():\n",
|
||||
" aiplatform.log_metrics({f\"train.{key}\": val[-1]})\n",
|
||||
" return history\n",
|
||||
"\n",
|
||||
" def evaluate(\n",
|
||||
" self,\n",
|
||||
" x=None,\n",
|
||||
" y=None,\n",
|
||||
" batch_size=None,\n",
|
||||
" verbose=1,\n",
|
||||
" sample_weight=None,\n",
|
||||
" steps=None,\n",
|
||||
" callbacks=None,\n",
|
||||
" max_queue_size=10,\n",
|
||||
" workers=1,\n",
|
||||
" use_multiprocessing=False,\n",
|
||||
" return_dict=False,\n",
|
||||
" ):\n",
|
||||
"\n",
|
||||
" metrics = super().evaluate(\n",
|
||||
" x=x,\n",
|
||||
" y=y,\n",
|
||||
" batch_size=batch_size,\n",
|
||||
" verbose=verbose,\n",
|
||||
" sample_weight=sample_weight,\n",
|
||||
" steps=steps,\n",
|
||||
" callbacks=callbacks,\n",
|
||||
" max_queue_size=max_queue_size,\n",
|
||||
" workers=workers,\n",
|
||||
" use_multiprocessing=use_multiprocessing,\n",
|
||||
" return_dict=return_dict,\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" aiplatform.log_metrics({\"eval.loss\": metrics[0]})\n",
|
||||
" for _ in range(1, len(metrics)):\n",
|
||||
" aiplatform.log_metrics({\"eval.metric\": metrics[_]})\n",
|
||||
" return metrics\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"class VertexTFModel(tf.keras.Model):\n",
|
||||
" \"\"\"\n",
|
||||
" Sublass of the tensorflow.keras.Model model type. Overrides with pass thru of\n",
|
||||
" underlying super class methods to automatically log parameters/metrics for Vertex AI experiments.\n",
|
||||
"\n",
|
||||
" compile():\n",
|
||||
" fit():\n",
|
||||
" evaluate():\n",
|
||||
" \"\"\"\n",
|
||||
"\n",
|
||||
" def __init__(self, inputs, outputs):\n",
|
||||
" return super().__init__(inputs, outputs)\n",
|
||||
"\n",
|
||||
" def compile(\n",
|
||||
" self,\n",
|
||||
" optimizer=\"rmsprop\",\n",
|
||||
" loss=None,\n",
|
||||
" metrics=None,\n",
|
||||
" loss_weights=None,\n",
|
||||
" weighted_metrics=None,\n",
|
||||
" run_eagerly=None,\n",
|
||||
" steps_per_execution=None,\n",
|
||||
" ):\n",
|
||||
" try:\n",
|
||||
" learning_rate = optimizer.learning_rate.numpy()\n",
|
||||
" aiplatform.log_params({\"train.learning_rate\": float(learning_rate)})\n",
|
||||
" except:\n",
|
||||
" pass\n",
|
||||
" return super().compile(\n",
|
||||
" loss=loss,\n",
|
||||
" optimizer=optimizer,\n",
|
||||
" metrics=metrics,\n",
|
||||
" loss_weights=loss_weights,\n",
|
||||
" weighted_metrics=weighted_metrics,\n",
|
||||
" run_eagerly=run_eagerly,\n",
|
||||
" steps_per_execution=steps_per_execution,\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" def fit(\n",
|
||||
" self,\n",
|
||||
" x=None,\n",
|
||||
" y=None,\n",
|
||||
" batch_size=None,\n",
|
||||
" epochs=1,\n",
|
||||
" verbose=\"auto\",\n",
|
||||
" callbacks=None,\n",
|
||||
" validation_split=0.0,\n",
|
||||
" validation_data=None,\n",
|
||||
" shuffle=True,\n",
|
||||
" class_weight=None,\n",
|
||||
" sample_weight=None,\n",
|
||||
" initial_epoch=0,\n",
|
||||
" steps_per_epoch=None,\n",
|
||||
" validation_steps=None,\n",
|
||||
" validation_batch_size=None,\n",
|
||||
" validation_freq=1,\n",
|
||||
" max_queue_size=10,\n",
|
||||
" workers=1,\n",
|
||||
" use_multiprocessing=False,\n",
|
||||
" ):\n",
|
||||
" aiplatform.log_params({\"train.epochs\": int(epochs)})\n",
|
||||
" if batch_size:\n",
|
||||
" aiplatform.log_params({\"train.batch_size\": int(batch_size)})\n",
|
||||
" if steps_per_epoch:\n",
|
||||
" aiplatform.log_params({\"train.steps\": int(steps_per_epoch)})\n",
|
||||
"\n",
|
||||
" history = super().fit(\n",
|
||||
" x=x,\n",
|
||||
" y=y,\n",
|
||||
" batch_size=batch_size,\n",
|
||||
" epochs=epochs,\n",
|
||||
" verbose=verbose,\n",
|
||||
" callbacks=callbacks,\n",
|
||||
" validation_split=validation_split,\n",
|
||||
" validation_data=validation_data,\n",
|
||||
" shuffle=shuffle,\n",
|
||||
" class_weight=class_weight,\n",
|
||||
" sample_weight=sample_weight,\n",
|
||||
" initial_epoch=initial_epoch,\n",
|
||||
" steps_per_epoch=steps_per_epoch,\n",
|
||||
" validation_steps=validation_steps,\n",
|
||||
" validation_batch_size=validation_batch_size,\n",
|
||||
" validation_freq=validation_freq,\n",
|
||||
" max_queue_size=max_queue_size,\n",
|
||||
" workers=workers,\n",
|
||||
" use_multiprocessing=use_multiprocessing,\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" for key, val in history.history.items():\n",
|
||||
" aiplatform.log_metrics({f\"train.{key}\": val[-1]})\n",
|
||||
" return history\n",
|
||||
"\n",
|
||||
" def evaluate(\n",
|
||||
" self,\n",
|
||||
" x=None,\n",
|
||||
" y=None,\n",
|
||||
" batch_size=None,\n",
|
||||
" verbose=1,\n",
|
||||
" sample_weight=None,\n",
|
||||
" steps=None,\n",
|
||||
" callbacks=None,\n",
|
||||
" max_queue_size=10,\n",
|
||||
" workers=1,\n",
|
||||
" use_multiprocessing=False,\n",
|
||||
" return_dict=False,\n",
|
||||
" ):\n",
|
||||
"\n",
|
||||
" metrics = super().evaluate(\n",
|
||||
" x=x,\n",
|
||||
" y=y,\n",
|
||||
" batch_size=batch_size,\n",
|
||||
" verbose=verbose,\n",
|
||||
" sample_weight=sample_weight,\n",
|
||||
" steps=steps,\n",
|
||||
" callbacks=callbacks,\n",
|
||||
" max_queue_size=max_queue_size,\n",
|
||||
" workers=workers,\n",
|
||||
" use_multiprocessing=use_multiprocessing,\n",
|
||||
" return_dict=return_dict,\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" aiplatform.log_metrics({\"eval.loss\": metrics[0]})\n",
|
||||
" for _ in range(1, len(metrics)):\n",
|
||||
" aiplatform.log_metrics({\"eval.metric\": metrics[_]})\n",
|
||||
" return metrics"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "ce76826902c0"
|
||||
},
|
||||
"source": [
|
||||
"### Train the model with Vertex AI Experiments\n",
|
||||
"\n",
|
||||
"In the following code, you build, train and evaluate a TensorFlow tabular model. The Python script includes the following calls to integrate `Vertex AI Experiments`:\n",
|
||||
"\n",
|
||||
"- command-line arguments: The arguments `experiment` and `run` are used to pass in the experiment and run names for the experiment.\n",
|
||||
"- `autologging()`: Initializes the experiment and does the heap injection.\n",
|
||||
"- `aiplatform.start_execution()`: Initializes a context for linking artifacts.\n",
|
||||
"- `aiplatform.end_run()`: Ends the experiment.\n",
|
||||
"\n",
|
||||
"*Note:* The initializer `Sequential` will be redirected to `VertexTFSequential` by heap injection. When subsequent calls are made to the compile(), fit() and evaluate() methods, they will be executed as the corresponding `VertexTFSequential` methods."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "427846783ed6"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"EXPERIMENT_NAME = f\"myexperiment{UUID}\"\n",
|
||||
"RUN_NAME = \"run-1\"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def make_dataset():\n",
|
||||
"\n",
|
||||
" # Scaling Boston Housing data features\n",
|
||||
" def scale(feature):\n",
|
||||
" max = np.max(feature)\n",
|
||||
" feature = (feature / max).astype(np.float)\n",
|
||||
" return feature, max\n",
|
||||
"\n",
|
||||
" (x_train, y_train), (x_test, y_test) = tf.keras.datasets.boston_housing.load_data(\n",
|
||||
" path=\"boston_housing.npz\", test_split=0.2, seed=113\n",
|
||||
" )\n",
|
||||
" params = []\n",
|
||||
"\n",
|
||||
" for _ in range(13):\n",
|
||||
" x_train[_], max = scale(x_train[_])\n",
|
||||
" x_test[_], _ = scale(x_test[_])\n",
|
||||
" params.append(max)\n",
|
||||
"\n",
|
||||
" return (x_train, y_train), (x_test, y_test)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Build the Keras model\n",
|
||||
"def build_and_compile_dnn_model(lr):\n",
|
||||
" model = tf.keras.Sequential(\n",
|
||||
" [\n",
|
||||
" tf.keras.layers.Dense(128, activation=\"relu\", input_shape=(13,)),\n",
|
||||
" tf.keras.layers.Dense(128, activation=\"relu\"),\n",
|
||||
" tf.keras.layers.Dense(1, activation=\"linear\"),\n",
|
||||
" ]\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" model.compile(\n",
|
||||
" loss=\"mse\",\n",
|
||||
" optimizer=tf.keras.optimizers.RMSprop(learning_rate=lr),\n",
|
||||
" metrics=[tf.keras.metrics.RootMeanSquaredError()],\n",
|
||||
" )\n",
|
||||
" return model\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# autologging\n",
|
||||
"autolog(experiment=EXPERIMENT_NAME, run=RUN_NAME)\n",
|
||||
"\n",
|
||||
"with aiplatform.start_execution(\n",
|
||||
" schema_title=\"system.ContainerExecution\", display_name=\"example_training\"\n",
|
||||
") as execution:\n",
|
||||
" BATCH_SIZE = 16\n",
|
||||
"\n",
|
||||
" model = build_and_compile_dnn_model(lr=0.01)\n",
|
||||
"\n",
|
||||
" # Train the model\n",
|
||||
" (x_train, y_train), (x_test, y_test) = make_dataset()\n",
|
||||
" model.fit(x_train, y_train, epochs=10, batch_size=BATCH_SIZE)\n",
|
||||
"\n",
|
||||
" model.evaluate(x_test, y_test)\n",
|
||||
"\n",
|
||||
"aiplatform.end_run()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "5f40912e6500"
|
||||
},
|
||||
"source": [
|
||||
"#### Get the experiment results\n",
|
||||
"\n",
|
||||
"Next, you use the experiment name as a parameter to the method `get_experiment_df()` to get the results of the experiment as a pandas dataframe."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "7e9671712230"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"experiment_df = aiplatform.get_experiment_df()\n",
|
||||
"experiment_df = experiment_df[experiment_df.experiment_name == EXPERIMENT_NAME]\n",
|
||||
"experiment_df.T"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "e508c159d712"
|
||||
},
|
||||
"source": [
|
||||
"#### Delete the experiment\n",
|
||||
"\n",
|
||||
"Since the experiment was created within a training script, to delete the experiment you use the `list()` method to obtain all the experiments for the project, and then filter on the experiment name."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "1a1b5fcbfde0"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"experiments = aiplatform.Experiment.list()\n",
|
||||
"for experiment in experiments:\n",
|
||||
" if experiment.name == EXPERIMENT_NAME:\n",
|
||||
" experiment.delete()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "cleanup:mbsdk"
|
||||
},
|
||||
"source": [
|
||||
"# Cleaning up\n",
|
||||
"\n",
|
||||
"To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
|
||||
"project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
|
||||
"\n",
|
||||
"Otherwise, you can delete the individual resources you created in this tutorial."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "9eb897e0e728"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# There are no resources to cleanup"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"name": "get_started_with_vertex_experiments_autologging_tf.ipynb",
|
||||
"toc_visible": true
|
||||
},
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"name": "python3"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 0
|
||||
}
|
||||
-1349
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
+372
-101
@@ -31,8 +31,6 @@
|
||||
"source": [
|
||||
"# [TODO] Add your H1 title heading here\n",
|
||||
"\n",
|
||||
"{TODO: Update the links below.} \n",
|
||||
"\n",
|
||||
"<table align=\"left\">\n",
|
||||
"\n",
|
||||
" <td>\n",
|
||||
@@ -55,17 +53,6 @@
|
||||
"</table>"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "24743cf4a1e1"
|
||||
},
|
||||
"source": [
|
||||
"**_NOTE_**: This notebook has been tested in the following environment:\n",
|
||||
"\n",
|
||||
"* Python version = 3.9"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
@@ -124,7 +111,7 @@
|
||||
"This tutorial uses billable components of Google Cloud:\n",
|
||||
"\n",
|
||||
"* Vertex AI\n",
|
||||
"* {TODO: BigQuery}\n",
|
||||
"* {TODO: BigQyuery}\n",
|
||||
"* Cloud Storage\n",
|
||||
"\n",
|
||||
"{TODO: Include links to pricing documentation for each product you listed above.\n",
|
||||
@@ -138,6 +125,62 @@
|
||||
"to generate a cost estimate based on your projected usage."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "gCuSR8GkAgzl"
|
||||
},
|
||||
"source": [
|
||||
"### Set up your local development environment\n",
|
||||
"\n",
|
||||
"**If you are using Colab or Vertex AI Workbench Notebooks**, your environment already meets\n",
|
||||
"all the requirements to run this notebook. You can skip this step."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "24743cf4a1e1"
|
||||
},
|
||||
"source": [
|
||||
"**_NOTE_**: This notebook has been tested in the following environment:\n",
|
||||
"\n",
|
||||
"* Python version = 3.9"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "gCuSR8GkAgzl"
|
||||
},
|
||||
"source": [
|
||||
"**Otherwise**, make sure your environment meets this notebook's requirements.\n",
|
||||
"You need the following:\n",
|
||||
"\n",
|
||||
"* The Google Cloud SDK\n",
|
||||
"\n",
|
||||
"The Google Cloud guide to [Setting up a Python development\n",
|
||||
"environment](https://cloud.google.com/python/setup) and the [Jupyter\n",
|
||||
"installation guide](https://jupyter.org/install) provide detailed instructions\n",
|
||||
"for meeting these requirements. The following steps provide a condensed set of\n",
|
||||
"instructions:\n",
|
||||
"\n",
|
||||
"1. [Install and initialize the Cloud SDK.](https://cloud.google.com/sdk/docs/)\n",
|
||||
"\n",
|
||||
"1. [Install Python 3.](https://cloud.google.com/python/setup#installing_python)\n",
|
||||
"\n",
|
||||
"1. [Install\n",
|
||||
" virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv)\n",
|
||||
" and create a virtual environment that uses Python 3. Activate the virtual environment.\n",
|
||||
"\n",
|
||||
"1. To install Jupyter, run `pip3 install jupyter` on the\n",
|
||||
"command-line in a terminal shell.\n",
|
||||
"\n",
|
||||
"1. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
|
||||
"\n",
|
||||
"1. Open this notebook in the Jupyter Notebook Dashboard."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
@@ -159,32 +202,51 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Install the packages\n",
|
||||
"! pip3 install --user --upgrade google-cloud-aiplatform"
|
||||
"import os\n",
|
||||
"\n",
|
||||
"# The Vertex AI Workbench Notebook product has specific requirements\n",
|
||||
"IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
|
||||
"IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
|
||||
" \"/opt/deeplearning/metadata/env_version\"\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
|
||||
"USER_FLAG = \"\"\n",
|
||||
"if IS_WORKBENCH_NOTEBOOK:\n",
|
||||
" USER_FLAG = \"--user\"\n",
|
||||
"\n",
|
||||
"! pip3 install --upgrade google-cloud-aiplatform {USER_FLAG} -q\n",
|
||||
"# TODO: Add remaining package installs here. All packages should be on a single pip install to resolve dependencies"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "58707a750154"
|
||||
"id": "hhq5zEbGg0XX"
|
||||
},
|
||||
"source": [
|
||||
"### Colab only: Uncomment the following cell to restart the kernel."
|
||||
"### Restart the kernel\n",
|
||||
"\n",
|
||||
"After you install the additional packages, you need to restart the notebook kernel so it can find the packages."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "f200f10a1da3"
|
||||
"id": "EzrelQZ22IZj"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Automatically restart kernel after installs so that your environment can access the new packages\n",
|
||||
"# import IPython\n",
|
||||
"# Automatically restart kernel after installs\n",
|
||||
"import os\n",
|
||||
"\n",
|
||||
"# app = IPython.Application.instance()\n",
|
||||
"# app.kernel.do_shutdown(True)"
|
||||
"if not os.getenv(\"IS_TESTING\"):\n",
|
||||
" # Automatically restart kernel after installs\n",
|
||||
" import IPython\n",
|
||||
"\n",
|
||||
" app = IPython.Application.instance()\n",
|
||||
" app.kernel.do_shutdown(True)"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -201,11 +263,16 @@
|
||||
"\n",
|
||||
"1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
|
||||
"\n",
|
||||
"2. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
|
||||
"1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
|
||||
"\n",
|
||||
"3. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com). {TODO: Update the APIs needed for your tutorial. Edit the API names, and update the link to append the API IDs, separating each one with a comma. For example, container.googleapis.com,cloudbuild.googleapis.com}\n",
|
||||
"1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com). {TODO: Update the APIs needed for your tutorial. Edit the API names, and update the link to append the API IDs, separating each one with a comma. For example, container.googleapis.com,cloudbuild.googleapis.com}\n",
|
||||
"\n",
|
||||
"4. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk)."
|
||||
"1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
|
||||
"\n",
|
||||
"1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
|
||||
"Cloud SDK uses the right project for all the commands in this notebook.\n",
|
||||
"\n",
|
||||
"**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -216,10 +283,7 @@
|
||||
"source": [
|
||||
"#### Set your project ID\n",
|
||||
"\n",
|
||||
"**If you don't know your project ID**, try the following:\n",
|
||||
"* Run `gcloud config list`.\n",
|
||||
"* Run `gcloud projects list`.\n",
|
||||
"* See the support page: [Locate the project ID](https://support.google.com/googleapi/answer/7014113)"
|
||||
"**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -230,10 +294,57 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"PROJECT_ID = \"[your-project-id]\" # @param {type:\"string\"}\n",
|
||||
"PROJECT_ID = \"[your-project-id]\" # @param {type:\"string\"}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "riG_qUokg0XZ"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
|
||||
" # Get your GCP project id from gcloud\n",
|
||||
" shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
|
||||
" PROJECT_ID = shell_output[0]\n",
|
||||
" print(\"Project ID:\", PROJECT_ID)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "set_gcloud_project_id"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gcloud config set project $PROJECT_ID"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "23988890fef6"
|
||||
},
|
||||
"source": [
|
||||
"#### Get your project number {TODO: Include these cells if the notebook uses a project number}\n",
|
||||
"\n",
|
||||
"# Set the project id\n",
|
||||
"! gcloud config set project {PROJECT_ID}"
|
||||
"Now that the project ID is set, you get your corresponding project number."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "2d6950574e1d"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"shell_output = ! gcloud projects list --filter=\"PROJECT_ID:'{PROJECT_ID}'\" --format='value(PROJECT_NUMBER)'\n",
|
||||
"PROJECT_NUMBER = shell_output[0]\n",
|
||||
"print(\"Project Number:\", PROJECT_NUMBER)"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -244,18 +355,63 @@
|
||||
"source": [
|
||||
"#### Region\n",
|
||||
"\n",
|
||||
"You can also change the `REGION` variable used by Vertex AI. Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
|
||||
"You can also change the `REGION` variable, which is used for operations\n",
|
||||
"throughout the rest of this notebook. Below are regions supported for Vertex AI. It is recommended that you choose the region closest to you.\n",
|
||||
"\n",
|
||||
"- Americas: `us-central1`\n",
|
||||
"- Europe: `europe-west4`\n",
|
||||
"- Asia Pacific: `asia-east1`\n",
|
||||
"\n",
|
||||
"You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
|
||||
"\n",
|
||||
"Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "region"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"REGION = \"[your-region]\" # @param {type: \"string\"}"
|
||||
"REGION = \"[your-region]\" # @param {type: \"string\"}\n",
|
||||
"\n",
|
||||
"if REGION == \"[your-region]\":\n",
|
||||
" REGION = \"us-central1\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "06571eb4063b"
|
||||
},
|
||||
"source": [
|
||||
"#### UUID\n",
|
||||
"\n",
|
||||
"If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial.\n",
|
||||
"\n",
|
||||
"{TODO: replace the `TIMESTAMP` with `UUID` in official notebooks}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "697568e92bd6"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import random\n",
|
||||
"import string\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Generate a uuid of a specifed length(default=8)\n",
|
||||
"def generate_uuid(length: int = 8) -> str:\n",
|
||||
" return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"UUID = generate_uuid()"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -266,68 +422,64 @@
|
||||
"source": [
|
||||
"### Authenticate your Google Cloud account\n",
|
||||
"\n",
|
||||
"Depending on your Jupyter environment, you may have to manually authenticate. Follow the relevant instructions below."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "74ccc9e52986"
|
||||
},
|
||||
"source": [
|
||||
"**1. Vertex AI Workbench**\n",
|
||||
"* Do nothing as you are already authenticated."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "de775a3773ba"
|
||||
},
|
||||
"source": [
|
||||
"**2. Local JupyterLab instance, uncomment and run:**"
|
||||
"**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
|
||||
"authenticated. \n",
|
||||
"\n",
|
||||
"**If you are using Colab**, run the cell below and follow the instructions\n",
|
||||
"when prompted to authenticate your account via oAuth.\n",
|
||||
"\n",
|
||||
"**Otherwise**, follow these steps:\n",
|
||||
"\n",
|
||||
"1. In the Cloud Console, go to the [**Create service account key**\n",
|
||||
" page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
|
||||
"\n",
|
||||
"2. Click **Create service account**.\n",
|
||||
"\n",
|
||||
"3. In the **Service account name** field, enter a name, and\n",
|
||||
" click **Create**.\n",
|
||||
"\n",
|
||||
"4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
|
||||
"into the filter box, and select\n",
|
||||
" **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
|
||||
"\n",
|
||||
"5. Click *Create*. A JSON file that contains your key downloads to your\n",
|
||||
"local environment.\n",
|
||||
"\n",
|
||||
"6. Enter the path to your service account key as the\n",
|
||||
"`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "254614fa0c46"
|
||||
"id": "PyQmSRbKA8r-"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# ! gcloud auth login"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "ef21552ccea8"
|
||||
},
|
||||
"source": [
|
||||
"**3. Colab, uncomment and run:**"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"metadata": {
|
||||
"id": "603adbbf0532"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# from google.colab import auth\n",
|
||||
"# auth.authenticate_user()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "f6b2ccc891ed"
|
||||
},
|
||||
"source": [
|
||||
"**4. Service account or other**\n",
|
||||
"* See all authentication options here: [Google Cloud Platform Jupyter Notebook Authentication Guide](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/notebook_authentication_guide.ipynb)"
|
||||
"# If you are running this notebook in Colab, run this cell and follow the\n",
|
||||
"# instructions to authenticate your GCP account. This provides access to your\n",
|
||||
"# Cloud Storage bucket and lets you submit training jobs and prediction\n",
|
||||
"# requests.\n",
|
||||
"\n",
|
||||
"import os\n",
|
||||
"import sys\n",
|
||||
"\n",
|
||||
"# If on Vertex AI Workbench, then don't execute this code\n",
|
||||
"IS_COLAB = \"google.colab\" in sys.modules\n",
|
||||
"if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
|
||||
" \"DL_ANACONDA_HOME\"\n",
|
||||
"):\n",
|
||||
" if \"google.colab\" in sys.modules:\n",
|
||||
" from google.colab import auth as google_auth\n",
|
||||
"\n",
|
||||
" google_auth.authenticate_user()\n",
|
||||
"\n",
|
||||
" # If you are running this notebook locally, replace the string below with the\n",
|
||||
" # path to your service account key and run this cell to authenticate your GCP\n",
|
||||
" # account.\n",
|
||||
" elif not os.getenv(\"IS_TESTING\"):\n",
|
||||
" %env GOOGLE_APPLICATION_CREDENTIALS '[your-service-account-key-path]'"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -338,9 +490,20 @@
|
||||
"source": [
|
||||
"### Create a Cloud Storage bucket\n",
|
||||
"\n",
|
||||
"Create a storage bucket to store intermediate artifacts such as datasets.\n",
|
||||
"**The following steps are required, regardless of your notebook environment.**\n",
|
||||
"\n",
|
||||
"- *{Note to notebook author: For any user-provided strings that need to be unique (like bucket names or model ID's), append \"-unique\" to the end so proper testing can occur}*"
|
||||
"\n",
|
||||
"{TODO: Adjust wording in the first paragraph to fit your use case - explain how your tutorial uses the Cloud Storage bucket. The example below shows how Vertex AI uses the bucket for training.}\n",
|
||||
"\n",
|
||||
"When you submit a training job using the Vertex AI SDK, you upload a Python package\n",
|
||||
"containing your training code to a Cloud Storage bucket. Vertex AI runs\n",
|
||||
"the code from this package. In this tutorial, Vertex AI also saves the\n",
|
||||
"trained model that results from your job in the same bucket. Using this model artifact, you can then\n",
|
||||
"create Vertex AI model and endpoint resources in order to serve\n",
|
||||
"online predictions.\n",
|
||||
"\n",
|
||||
"Set the name of your Cloud Storage bucket below. It must be unique across all\n",
|
||||
"Cloud Storage buckets."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -351,7 +514,21 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"BUCKET_URI = \"gs://your-bucket-name-unique\" # @param {type:\"string\"}"
|
||||
"BUCKET_NAME = \"[your-bucket-name]\" # @param {type:\"string\"}\n",
|
||||
"BUCKET_URI = f\"gs://{BUCKET_NAME}\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "cf221059d072"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
|
||||
" BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
|
||||
" BUCKET_URI = f\"gs://{BUCKET_NAME}\""
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -377,7 +554,99 @@
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "960505627ddf"
|
||||
"id": "ucvCsknMCims"
|
||||
},
|
||||
"source": [
|
||||
"Finally, validate access to your Cloud Storage bucket by examining its contents:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "vhOb7YnwClBb"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil ls -al $BUCKET_URI"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "set_service_account"
|
||||
},
|
||||
"source": [
|
||||
"#### Service Account {TODO: Include these cells if the notebook specifies a service account}\n",
|
||||
"\n",
|
||||
"{TODO: What uses service account in the notebook; e.g., You use a service account to create Vertex AI Pipeline jobs.}. If you do not want to use your project's Compute Engine service account, set `SERVICE_ACCOUNT` to another service account ID."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "set_service_account"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"SERVICE_ACCOUNT = \"[your-service-account]\" # @param {type:\"string\"}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "autoset_service_account"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"if (\n",
|
||||
" SERVICE_ACCOUNT == \"\"\n",
|
||||
" or SERVICE_ACCOUNT is None\n",
|
||||
" or SERVICE_ACCOUNT == \"[your-service-account]\"\n",
|
||||
"):\n",
|
||||
" # Get your service account from gcloud\n",
|
||||
" if not IS_COLAB:\n",
|
||||
" shell_output = !gcloud auth list 2>/dev/null\n",
|
||||
" SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
|
||||
"\n",
|
||||
" else: # IS_COLAB:\n",
|
||||
" shell_output = ! gcloud projects describe $PROJECT_ID\n",
|
||||
" project_number = shell_output[-1].split(\":\")[1].strip().replace(\"'\", \"\")\n",
|
||||
" SERVICE_ACCOUNT = f\"{project_number}-compute@developer.gserviceaccount.com\"\n",
|
||||
"\n",
|
||||
" print(\"Service Account:\", SERVICE_ACCOUNT)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "set_service_account:pipelines"
|
||||
},
|
||||
"source": [
|
||||
"#### Set service account access for {TODO; e.g., Vertex AI Pipelines}\n",
|
||||
"\n",
|
||||
"Run the following commands to grant your service account access to {TODO; i.e., read and write pipeline artifacts} in the bucket that you created in the previous step. You only need to run this step once per service account."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "set_service_account:pipelines"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
|
||||
"\n",
|
||||
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "XoEqT2Y4DJmf"
|
||||
},
|
||||
"source": [
|
||||
"### Import libraries"
|
||||
@@ -387,11 +656,13 @@
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "PyQmSRbKA8r-"
|
||||
"id": "pRUOFELefqf1"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from google.cloud import aiplatform"
|
||||
"import google.cloud.aiplatform as aiplatform\n",
|
||||
"\n",
|
||||
"# TODO: import remaining libraries; e.g., tensorflow"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -434,21 +705,21 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "sx_vKniMq9ZX"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"\n",
|
||||
"# Delete endpoint resource\n",
|
||||
"# e.g. `endpoint.delete()`\n",
|
||||
"! gcloud ai endpoints delete $ENDPOINT_NAME --quiet --region $REGION\n",
|
||||
"\n",
|
||||
"# Delete model resource\n",
|
||||
"# e.g. `model.delete()`\n",
|
||||
"! gcloud ai models delete $MODEL_NAME --quiet\n",
|
||||
"\n",
|
||||
"# Delete Cloud Storage objects that were created\n",
|
||||
"! gsutil -m rm -r $JOB_DIR\n",
|
||||
"\n",
|
||||
"delete_bucket = False\n",
|
||||
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
|
||||
" ! gsutil -m rm -r $BUCKET_URI"
|
||||
|
||||
@@ -29,7 +29,7 @@
|
||||
"id": "title"
|
||||
},
|
||||
"source": [
|
||||
"# Vertex AI SDK: Training an AutoML text sentiment analysis model for online predictions\n",
|
||||
"# Vertex SDK: AutoML training text sentiment analysis model for online prediction\n",
|
||||
"\n",
|
||||
"<table align=\"left\">\n",
|
||||
" <td>\n",
|
||||
@@ -45,7 +45,6 @@
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_text_sentiment_analysis_online.ipynb\">\n",
|
||||
" <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
|
||||
" Open in Vertex AI Workbench\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
@@ -61,7 +60,8 @@
|
||||
"source": [
|
||||
"## Overview\n",
|
||||
"\n",
|
||||
"This tutorial demonstrates how to use the Vertex AI SDK to train and deploy an [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) text sentiment analysis model and get online predictions from it."
|
||||
"\n",
|
||||
"This tutorial demonstrates how to use the Vertex AI SDK to create text sentiment analysis models and do online prediction using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) model."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -72,23 +72,16 @@
|
||||
"source": [
|
||||
"### Objective\n",
|
||||
"\n",
|
||||
"In this tutorial, you learn how to create an AutoML text sentiment analysis model and deploy it for online predictions from a Python script using the Vertex AI SDK. You can alternatively create and deploy models using the `gcloud` command-line tool or online using the Cloud Console.\n",
|
||||
"\n",
|
||||
"This tutorial uses the following Google Cloud ML services and resources:\n",
|
||||
"- Vertex AI Datasets\n",
|
||||
"- Vertex AI Training (AutoML)\n",
|
||||
"- Vertex AI Model Registry\n",
|
||||
"- Vertex AI Endpoints\n",
|
||||
"In this tutorial, you learn how to create an AutoML text sentiment analysis model and deploy for online prediction from a Python script using the Vertex SDK. You can alternatively create and deploy models using the `gcloud` command-line tool or online using the Cloud Console.\n",
|
||||
"\n",
|
||||
"The steps performed include:\n",
|
||||
"\n",
|
||||
"- Create a `Vertex AI Dataset` resource.\n",
|
||||
"- Create a training job for the AutoML model on the dataset.\n",
|
||||
"- View the model evaluation metrics.\n",
|
||||
"- Deploy the `Vertex AI Model` resource to a serving `Vertex AI Endpoint`.\n",
|
||||
"- Make a prediction request to the deployed model.\n",
|
||||
"- Undeploy the model from endpoint.\n",
|
||||
"- Perform clean up process."
|
||||
"- Create a Vertex `Dataset` resource.\n",
|
||||
"- Create a training job for the model.\n",
|
||||
"- View the model evaluation.\n",
|
||||
"- Deploy the `Model` resource to a serving `Endpoint` resource.\n",
|
||||
"- Make a prediction.\n",
|
||||
"- Undeploy the `Model`."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -99,7 +92,7 @@
|
||||
"source": [
|
||||
"### Dataset\n",
|
||||
"\n",
|
||||
"The dataset used for this tutorial is the [Crowdflower Claritin-Twitter dataset](https://data.world/crowdflower/claritin-twitter) that consists of tweets tagged with sentiment, the author's gender, and whether or not they mention any of the top 10 adverse events reported to the FDA. The version of the dataset you use in this tutorial is stored in a public Cloud Storage bucket. In this tutorial, you use the tweets data to build an AutoML text sentiment analysis model on Google Cloud platform."
|
||||
"The dataset used for this tutorial is the [Crowdflower Claritin-Twitter dataset](https://data.world/crowdflower/claritin-twitter) that consists of tweets tagged with sentiment, the author's gender, and whether or not they mention any of the top 10 adverse events reported to the FDA. The version of the dataset you will use in this tutorial is stored in a public Cloud Storage bucket. In this tutorial, you will use the tweets' data to build an AutoML-text-sentiment-analysis model on Google Cloud platform."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -130,34 +123,29 @@
|
||||
"source": [
|
||||
"### Set up your local development environment\n",
|
||||
"\n",
|
||||
"**If you are using Colab or Vertex AI Workbench Notebooks**, your environment already meets\n",
|
||||
"all the requirements to run this notebook. You can skip this step.\n",
|
||||
"If you are using Colab or Google Cloud Notebooks, your environment already meets all the requirements to run this notebook. You can skip this step.\n",
|
||||
"\n",
|
||||
"**Otherwise**, make sure your environment meets this notebook's requirements.\n",
|
||||
"You need the following:\n",
|
||||
"Otherwise, make sure your environment meets this notebook's requirements. You need the following:\n",
|
||||
"\n",
|
||||
"* The Google Cloud SDK\n",
|
||||
"- The Cloud Storage SDK\n",
|
||||
"- Git\n",
|
||||
"- Python 3\n",
|
||||
"- virtualenv\n",
|
||||
"- Jupyter notebook running in a virtual environment with Python 3\n",
|
||||
"\n",
|
||||
"The Google Cloud guide to [Setting up a Python development\n",
|
||||
"environment](https://cloud.google.com/python/setup) and the [Jupyter\n",
|
||||
"installation guide](https://jupyter.org/install) provide detailed instructions\n",
|
||||
"for meeting these requirements. The following steps provide a condensed set of\n",
|
||||
"instructions:\n",
|
||||
"The Cloud Storage guide to [Setting up a Python development environment](https://cloud.google.com/python/setup) and the [Jupyter installation guide](https://jupyter.org/install) provide detailed instructions for meeting these requirements. The following steps provide a condensed set of instructions:\n",
|
||||
"\n",
|
||||
"1. [Install and initialize the Cloud SDK.](https://cloud.google.com/sdk/docs/)\n",
|
||||
"1. [Install and initialize the SDK](https://cloud.google.com/sdk/docs/).\n",
|
||||
"\n",
|
||||
"1. [Install Python 3.](https://cloud.google.com/python/setup#installing_python)\n",
|
||||
"2. [Install Python 3](https://cloud.google.com/python/setup#installing_python).\n",
|
||||
"\n",
|
||||
"1. [Install\n",
|
||||
" virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv)\n",
|
||||
" and create a virtual environment that uses Python 3. Activate the virtual environment.\n",
|
||||
"3. [Install virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv) and create a virtual environment that uses Python 3. Activate the virtual environment.\n",
|
||||
"\n",
|
||||
"1. To install Jupyter, run `pip3 install jupyter` on the\n",
|
||||
"command-line in a terminal shell.\n",
|
||||
"4. To install Jupyter, run `pip3 install jupyter` on the command-line in a terminal shell.\n",
|
||||
"\n",
|
||||
"1. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
|
||||
"5. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
|
||||
"\n",
|
||||
"1. Open this notebook in the Jupyter Notebook Dashboard."
|
||||
"6. Open this notebook in the Jupyter Notebook Dashboard.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -168,7 +156,7 @@
|
||||
"source": [
|
||||
"## Installation\n",
|
||||
"\n",
|
||||
"Install the latest version of Vertex AI SDK for Python."
|
||||
"Install the latest version of Vertex SDK for Python."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -181,19 +169,35 @@
|
||||
"source": [
|
||||
"import os\n",
|
||||
"\n",
|
||||
"# The Vertex AI Workbench Notebook product has specific requirements\n",
|
||||
"IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
|
||||
"IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
|
||||
" \"/opt/deeplearning/metadata/env_version\"\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
|
||||
"USER_FLAG = \"\"\n",
|
||||
"if IS_WORKBENCH_NOTEBOOK:\n",
|
||||
"# Google Cloud Notebook\n",
|
||||
"if os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
|
||||
" USER_FLAG = \"--user\"\n",
|
||||
"else:\n",
|
||||
" USER_FLAG = \"\"\n",
|
||||
"\n",
|
||||
"! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG -q\n",
|
||||
"! pip3 install -U google-cloud-storage $USER_FLAG -q"
|
||||
"! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "install_storage"
|
||||
},
|
||||
"source": [
|
||||
"Install the latest GA version of *google-cloud-storage* library as well.\n",
|
||||
"\n",
|
||||
"**Note**: You may encounter a PIP dependency error during the installation of the Google Cloud Storage package. This can be ignored as it will not affect the proper running of this script."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "install_storage"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! pip3 install -U google-cloud-storage $USER_FLAG"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -204,7 +208,7 @@
|
||||
"source": [
|
||||
"### Restart the kernel\n",
|
||||
"\n",
|
||||
"After you install the additional packages, you need to restart the notebook kernel so it can find the packages."
|
||||
"Once you've installed the additional packages, you need to restart the notebook kernel so it can find the packages. The following cell will restart the kernel."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -215,7 +219,6 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Automatically restart kernel after installs\n",
|
||||
"import os\n",
|
||||
"\n",
|
||||
"if not os.getenv(\"IS_TESTING\"):\n",
|
||||
@@ -234,33 +237,26 @@
|
||||
"source": [
|
||||
"## Before you begin\n",
|
||||
"\n",
|
||||
"### GPU runtime\n",
|
||||
"\n",
|
||||
"This tutorial does not require a GPU runtime.\n",
|
||||
"\n",
|
||||
"### Set up your Google Cloud project\n",
|
||||
"\n",
|
||||
"**The following steps are required, regardless of your notebook environment.**\n",
|
||||
"\n",
|
||||
"1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
|
||||
"\n",
|
||||
"1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
|
||||
"2. [Make sure that billing is enabled for your project.](https://cloud.google.com/billing/docs/how-to/modify-project)\n",
|
||||
"\n",
|
||||
"1. [Enable the Vertex AI, Compute Engine, and Cloud Storage APIs.](https://console.cloud.google.com/flows/enableapi?apiid=ml.googleapis.com,compute_component,storage-component.googleapis.com)\n",
|
||||
"3. [Enable the following APIs: Vertex AI APIs, Compute Engine APIs, and Cloud Storage.](https://console.cloud.google.com/flows/enableapi?apiid=ml.googleapis.com,compute_component,storage-component.googleapis.com)\n",
|
||||
"\n",
|
||||
"1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
|
||||
"4. If you are running this notebook locally, you will need to install the [Cloud SDK]((https://cloud.google.com/sdk)).\n",
|
||||
"\n",
|
||||
"1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
|
||||
"5. Enter your project ID in the cell below. Then run the cell to make sure the\n",
|
||||
"Cloud SDK uses the right project for all the commands in this notebook.\n",
|
||||
"\n",
|
||||
"**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "af794e75b7e3"
|
||||
},
|
||||
"source": [
|
||||
"#### Set your project ID\n",
|
||||
"\n",
|
||||
"**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
|
||||
"**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$`."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -340,9 +336,9 @@
|
||||
"id": "timestamp"
|
||||
},
|
||||
"source": [
|
||||
"#### UUID\n",
|
||||
"#### Timestamp\n",
|
||||
"\n",
|
||||
"If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
|
||||
"If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -353,16 +349,9 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import random\n",
|
||||
"import string\n",
|
||||
"from datetime import datetime\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Generate a uuid of a specifed length(default=8)\n",
|
||||
"def generate_uuid(length: int = 8) -> str:\n",
|
||||
" return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"UUID = generate_uuid()"
|
||||
"TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -373,31 +362,23 @@
|
||||
"source": [
|
||||
"### Authenticate your Google Cloud account\n",
|
||||
"\n",
|
||||
"**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
|
||||
"authenticated. \n",
|
||||
"**If you are using Google Cloud Notebooks**, your environment is already authenticated. Skip this step.\n",
|
||||
"\n",
|
||||
"**If you are using Colab**, run the cell below and follow the instructions\n",
|
||||
"when prompted to authenticate your account via oAuth.\n",
|
||||
"**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
|
||||
"\n",
|
||||
"**Otherwise**, follow these steps:\n",
|
||||
"\n",
|
||||
"1. In the Cloud Console, go to the [**Create service account key**\n",
|
||||
" page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
|
||||
"In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
|
||||
"\n",
|
||||
"2. Click **Create service account**.\n",
|
||||
"**Click Create service account**.\n",
|
||||
"\n",
|
||||
"3. In the **Service account name** field, enter a name, and\n",
|
||||
" click **Create**.\n",
|
||||
"In the **Service account name** field, enter a name, and click **Create**.\n",
|
||||
"\n",
|
||||
"4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
|
||||
"into the filter box, and select\n",
|
||||
" **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
|
||||
"In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex\" into the filter box, and select **Vertex Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
|
||||
"\n",
|
||||
"5. Click *Create*. A JSON file that contains your key downloads to your\n",
|
||||
"local environment.\n",
|
||||
"Click Create. A JSON file that contains your key downloads to your local environment.\n",
|
||||
"\n",
|
||||
"6. Enter the path to your service account key as the\n",
|
||||
"`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
|
||||
"Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -416,11 +397,8 @@
|
||||
"import os\n",
|
||||
"import sys\n",
|
||||
"\n",
|
||||
"# If on Vertex AI Workbench, then don't execute this code\n",
|
||||
"IS_COLAB = \"google.colab\" in sys.modules\n",
|
||||
"if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
|
||||
" \"DL_ANACONDA_HOME\"\n",
|
||||
"):\n",
|
||||
"# If on Google Cloud Notebook, then don't execute this code\n",
|
||||
"if not os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
|
||||
" if \"google.colab\" in sys.modules:\n",
|
||||
" from google.colab import auth as google_auth\n",
|
||||
"\n",
|
||||
@@ -430,7 +408,7 @@
|
||||
" # path to your service account key and run this cell to authenticate your GCP\n",
|
||||
" # account.\n",
|
||||
" elif not os.getenv(\"IS_TESTING\"):\n",
|
||||
" %env GOOGLE_APPLICATION_CREDENTIALS '[your-service-account-key-path]'"
|
||||
" %env GOOGLE_APPLICATION_CREDENTIALS ''"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -443,9 +421,9 @@
|
||||
"\n",
|
||||
"**The following steps are required, regardless of your notebook environment.**\n",
|
||||
"\n",
|
||||
"When you initialize your Vertex AI SDK, you provide a Cloud Storage bucket to the SDK to serve as a staging bucket for the session. \n",
|
||||
"When you initialize the Vertex SDK for Python, you specify a Cloud Storage staging bucket. The staging bucket is where all the data associated with your dataset and model resources are retained across sessions.\n",
|
||||
"\n",
|
||||
"Set the name of your Cloud Storage bucket below. It must be unique across all Cloud Storage buckets."
|
||||
"Set the name of your Cloud Storage bucket below. Bucket names must be globally unique across all Google Cloud projects, including those outside of your organization."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -456,8 +434,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"BUCKET_NAME = \"[your-bucket-name]\" # @param {type:\"string\"}\n",
|
||||
"BUCKET_URI = f\"gs://{BUCKET_NAME}\""
|
||||
"BUCKET_NAME = \"gs://[your-bucket-name]\" # @param {type:\"string\"}"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -468,9 +445,8 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
|
||||
" BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
|
||||
" BUCKET_URI = f\"gs://{BUCKET_NAME}\""
|
||||
"if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
|
||||
" BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -490,7 +466,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI"
|
||||
"! gsutil mb -l $REGION $BUCKET_NAME"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -510,7 +486,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil ls -al $BUCKET_URI"
|
||||
"! gsutil ls -al $BUCKET_NAME"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -519,7 +495,10 @@
|
||||
"id": "setup_vars"
|
||||
},
|
||||
"source": [
|
||||
"### Import libraries"
|
||||
"### Set up variables\n",
|
||||
"\n",
|
||||
"Next, set up some variables used throughout the tutorial.\n",
|
||||
"### Import libraries and define constants"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -539,9 +518,9 @@
|
||||
"id": "init_aip:mbsdk"
|
||||
},
|
||||
"source": [
|
||||
"### Initialize Vertex AI SDK for Python\n",
|
||||
"## Initialize Vertex SDK for Python\n",
|
||||
"\n",
|
||||
"Initialize the Vertex AI SDK for Python for your project and corresponding bucket."
|
||||
"Initialize the Vertex SDK for Python for your project and corresponding bucket."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -552,7 +531,18 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"aiplatform.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
|
||||
"aiplatform.init(project=PROJECT_ID, staging_bucket=BUCKET_NAME)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "tutorial_start:automl"
|
||||
},
|
||||
"source": [
|
||||
"# Tutorial\n",
|
||||
"\n",
|
||||
"Now you are ready to start creating your own AutoML text sentiment analysis model."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -561,9 +551,9 @@
|
||||
"id": "import_file:u_dataset,csv"
|
||||
},
|
||||
"source": [
|
||||
"### Define the constants\n",
|
||||
"#### Location of Cloud Storage training data.\n",
|
||||
"\n",
|
||||
"Set the constants that you use in this tutorial."
|
||||
"Now set the variable `IMPORT_FILE` to the location of the CSV index file in Cloud Storage."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -574,9 +564,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Set the location of the CSV index file in Cloud Storage.\n",
|
||||
"IMPORT_FILE = \"gs://cloud-samples-data/language/claritin.csv\"\n",
|
||||
"# Set the max. sentiment score\n",
|
||||
"SENTIMENT_MAX = 4"
|
||||
]
|
||||
},
|
||||
@@ -586,11 +574,11 @@
|
||||
"id": "quick_peek:csv"
|
||||
},
|
||||
"source": [
|
||||
"## Take a quick peek at your data\n",
|
||||
"#### Quick peek at your data\n",
|
||||
"\n",
|
||||
"This tutorial uses a version of the `Crowdflower Claritin-Twitter` dataset which is stored in a public Cloud Storage bucket, using a CSV index file.\n",
|
||||
"This tutorial uses a version of the Crowdflower Claritin-Twitter dataset that is stored in a public Cloud Storage bucket, using a CSV index file.\n",
|
||||
"\n",
|
||||
"Start by taking a quick peek at the data. Further, count the number of examples by counting the number of rows in the CSV index file (`wc -l`) and then print the first few rows."
|
||||
"Start by doing a quick peek at the data. You count the number of examples by counting the number of rows in the CSV index file (`wc -l`) and then peek at the first few rows."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -616,12 +604,12 @@
|
||||
"id": "create_dataset:text,tst"
|
||||
},
|
||||
"source": [
|
||||
"## Create the Dataset\n",
|
||||
"### Create the Dataset\n",
|
||||
"\n",
|
||||
"Now, create a `Vertex AI Dataset` resource using the `create` method of the `TextDataset` class, which takes the following parameters:\n",
|
||||
"Next, create the `Dataset` resource using the `create` method for the `TextDataset` class, which takes the following parameters:\n",
|
||||
"\n",
|
||||
"- `display_name`: The human readable name for the dataset resource.\n",
|
||||
"- `gcs_source`: A list of one or more dataset index files to import the data items into the dataset resource.\n",
|
||||
"- `display_name`: The human readable name for the `Dataset` resource.\n",
|
||||
"- `gcs_source`: A list of one or more dataset index files to import the data items into the `Dataset` resource.\n",
|
||||
"- `import_schema_uri`: The data labeling schema for the data items.\n",
|
||||
"\n",
|
||||
"This operation may take several minutes."
|
||||
@@ -636,7 +624,7 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"dataset = aiplatform.TextDataset.create(\n",
|
||||
" display_name=\"Crowdflower Claritin-Twitter\" + \"_\" + UUID,\n",
|
||||
" display_name=\"Crowdflower Claritin-Twitter\" + \"_\" + TIMESTAMP,\n",
|
||||
" gcs_source=[IMPORT_FILE],\n",
|
||||
" import_schema_uri=aiplatform.schema.dataset.ioformat.text.sentiment,\n",
|
||||
")\n",
|
||||
@@ -650,18 +638,15 @@
|
||||
"id": "create_automl_pipeline:text,tst"
|
||||
},
|
||||
"source": [
|
||||
"## Create and run training job\n",
|
||||
"### Create and run training pipeline\n",
|
||||
"\n",
|
||||
"In this section, to train an AutoML model, you perform these steps:\n",
|
||||
"To train an AutoML model, you perform two steps: 1) create a training pipeline, and 2) run the pipeline.\n",
|
||||
"\n",
|
||||
"1) create a training job.\n",
|
||||
"2) run the job.\n",
|
||||
"#### Create training pipeline\n",
|
||||
"\n",
|
||||
"### Create a training job\n",
|
||||
"An AutoML training pipeline is created with the `AutoMLTextTrainingJob` class, with the following parameters:\n",
|
||||
"\n",
|
||||
"An AutoML training job is created with the `AutoMLTextTrainingJob` class, with the following parameters:\n",
|
||||
"\n",
|
||||
"- `display_name`: The human readable name for the training job resource.\n",
|
||||
"- `display_name`: The human readable name for the `TrainingJob` resource.\n",
|
||||
"- `prediction_type`: The type task to train the model for.\n",
|
||||
" - `classification`: A text classification model.\n",
|
||||
" - `sentiment`: A text sentiment analysis model.\n",
|
||||
@@ -679,7 +664,7 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"job = aiplatform.AutoMLTextTrainingJob(\n",
|
||||
" display_name=\"claritin_\" + UUID,\n",
|
||||
" display_name=\"claritin_\" + TIMESTAMP,\n",
|
||||
" prediction_type=\"sentiment\",\n",
|
||||
" sentiment_max=SENTIMENT_MAX,\n",
|
||||
")\n",
|
||||
@@ -693,7 +678,7 @@
|
||||
"id": "run_automl_pipeline:text"
|
||||
},
|
||||
"source": [
|
||||
"### Run the training job\n",
|
||||
"#### Run the training pipeline\n",
|
||||
"\n",
|
||||
"Next, you run the training job by invoking the method `run`, with the following parameters:\n",
|
||||
"\n",
|
||||
@@ -705,7 +690,7 @@
|
||||
"\n",
|
||||
"The `run` method when completed returns the `Model` resource.\n",
|
||||
"\n",
|
||||
"The execution of the training pipeline take upto 180 minutes."
|
||||
"The execution of the training pipeline will take upto 180 minutes."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -718,7 +703,7 @@
|
||||
"source": [
|
||||
"model = job.run(\n",
|
||||
" dataset=dataset,\n",
|
||||
" model_display_name=\"claritin_\" + UUID,\n",
|
||||
" model_display_name=\"claritin_\" + TIMESTAMP,\n",
|
||||
" training_fraction_split=0.8,\n",
|
||||
" validation_fraction_split=0.1,\n",
|
||||
" test_fraction_split=0.1,\n",
|
||||
@@ -732,10 +717,9 @@
|
||||
},
|
||||
"source": [
|
||||
"## Review model evaluation scores\n",
|
||||
"After your model has finished training, you can review the evaluation scores for it.\n",
|
||||
"\n",
|
||||
"Once your model training has finished, you can review the evaluation scores.\n",
|
||||
"\n",
|
||||
"Firstly, you need to get a reference to the newly created model. As with datasets, you can either use the reference to the model variable you created when you deployed the model or you can list all of the models in your project and filter."
|
||||
"First, you need to get a reference to the new model. As with datasets, you can either use the reference to the model variable you created when you deployed the model or you can list all of the models in your project."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -747,7 +731,7 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Get model resource ID\n",
|
||||
"models = aiplatform.Model.list(filter=\"display_name=claritin_\" + UUID)\n",
|
||||
"models = aiplatform.Model.list(filter=\"display_name=claritin_\" + TIMESTAMP)\n",
|
||||
"\n",
|
||||
"# Get a reference to the Model Service client\n",
|
||||
"client_options = {\"api_endpoint\": f\"{REGION}-aiplatform.googleapis.com\"}\n",
|
||||
@@ -770,9 +754,7 @@
|
||||
"source": [
|
||||
"## Deploy the model\n",
|
||||
"\n",
|
||||
"Next, deploy your model to serve online predictions. To deploy the model, you invoke the `deploy` method of the model resource which in turn returns you the deployed endpoint.\n",
|
||||
"\n",
|
||||
"**Note:** Normally, an endpoint is created beforehand and is given as a reference while model deployment. By default, `deploy()` method creates an endpoint when an endpoint reference is not given."
|
||||
"Next, deploy your model for online prediction. To deploy the model, you invoke the `deploy` method."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -792,9 +774,9 @@
|
||||
"id": "make_prediction"
|
||||
},
|
||||
"source": [
|
||||
"## Send online prediction requests\n",
|
||||
"## Send a online prediction request\n",
|
||||
"\n",
|
||||
"In this step, you prepare some test instances from the dataset and send an online prediction request to your deployed model."
|
||||
"Send a online prediction to your deployed model."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -803,9 +785,9 @@
|
||||
"id": "get_test_item"
|
||||
},
|
||||
"source": [
|
||||
"### Create test instances\n",
|
||||
"### Get test item\n",
|
||||
"\n",
|
||||
"You use an arbitrary example out of the dataset as a test item. Don't be concerned that the example was likely used in training the model. It is just to demonstrate how to make a prediction."
|
||||
"You will use an arbitrary example out of the dataset as a test item. Don't be concerned that the example was likely used in training the model -- we just want to demonstrate how to make a prediction."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -831,21 +813,21 @@
|
||||
"id": "predict_request:mbsdk,tst"
|
||||
},
|
||||
"source": [
|
||||
"### Make the prediction request\n",
|
||||
"### Make the prediction\n",
|
||||
"\n",
|
||||
"Now that your model is deployed to an endpoint, you can send online prediction requests to the endpoint resource.\n",
|
||||
"Now that your `Model` resource is deployed to an `Endpoint` resource, you can do online predictions by sending prediction requests to the `Endpoint` resource.\n",
|
||||
"\n",
|
||||
"#### Request format\n",
|
||||
"#### Request\n",
|
||||
"\n",
|
||||
"The format of each instance should be in JSON as below:\n",
|
||||
"The format of each instance is:\n",
|
||||
"\n",
|
||||
" { 'content': text_string }\n",
|
||||
"\n",
|
||||
"Since the `predict()` method can take multiple instances, send your request as a list of one test instance.\n",
|
||||
"Since the predict() method can take multiple items (instances), send your single test item as a list of one test item.\n",
|
||||
"\n",
|
||||
"#### Response\n",
|
||||
"\n",
|
||||
"The response from the `predict()` call is a Python dictionary with the following entries:\n",
|
||||
"The response from the predict() call is a Python dictionary with the following entries:\n",
|
||||
"\n",
|
||||
"- `ids`: The internal assigned unique identifiers for each prediction request.\n",
|
||||
"- `sentiment`: The sentiment value.\n",
|
||||
@@ -874,7 +856,7 @@
|
||||
"source": [
|
||||
"## Undeploy the model\n",
|
||||
"\n",
|
||||
"After you explore the predictions, you undeploy the model from the `Endpoint` resouce. This deprovisions all compute resources and ends billing for the deployed model."
|
||||
"When you are done doing predictions, you undeploy the model from the `Endpoint` resouce. This deprovisions all compute resources and ends billing for the deployed model."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -901,11 +883,11 @@
|
||||
"\n",
|
||||
"Otherwise, you can delete the individual resources you created in this tutorial:\n",
|
||||
"\n",
|
||||
"- Vertex AI Dataset\n",
|
||||
"- Vertex AI Model\n",
|
||||
"- Vertex AI Endpoint\n",
|
||||
"- Dataset\n",
|
||||
"- Model\n",
|
||||
"- Endpoint\n",
|
||||
"- AutoML Training Job\n",
|
||||
"- Cloud Storage Bucket (set `delete_bucket` to **True** to delete the bucket)"
|
||||
"- Cloud Storage Bucket"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -916,8 +898,6 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"delete_bucket = False\n",
|
||||
"\n",
|
||||
"# Delete the dataset using the Vertex dataset object\n",
|
||||
"dataset.delete()\n",
|
||||
"\n",
|
||||
@@ -931,8 +911,8 @@
|
||||
"job.delete()\n",
|
||||
"\n",
|
||||
"# Delete the Cloud storage bucket\n",
|
||||
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
|
||||
" ! gsutil rm -r $BUCKET_URI"
|
||||
"if os.getenv(\"IS_TESTING\"):\n",
|
||||
" ! gsutil rm -r $BUCKET_NAME"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
"#\n",
|
||||
"# Unless required by applicable law or agreed to in writing, software\n",
|
||||
"# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
|
||||
"# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
|
||||
"# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
|
||||
"# See the License for the specific language governing permissions and\n",
|
||||
"# limitations under the License."
|
||||
]
|
||||
@@ -73,11 +73,7 @@
|
||||
"source": [
|
||||
"### Objective\n",
|
||||
"\n",
|
||||
"In this tutorial, you learn how to create an AutoML video object tracking model from a Python script, and then do a batch prediction using the Vertex AI SDK. You can alternatively create and deploy models using the `gcloud` command-line tool or online using the Cloud Console.\n",
|
||||
"\n",
|
||||
"This tutorial uses the following Google Cloud ML services and resources:\n",
|
||||
"\n",
|
||||
"- Vertex AI\n",
|
||||
"In this tutorial, you learn how to create an AutoML video object tracking model from a Python script, and then do a batch prediction using the Vertex SDK. You can alternatively create and deploy models using the `gcloud` command-line tool or online using the Cloud Console.\n",
|
||||
"\n",
|
||||
"The steps performed include:\n",
|
||||
"\n",
|
||||
@@ -101,7 +97,7 @@
|
||||
"source": [
|
||||
"### Dataset\n",
|
||||
"\n",
|
||||
"The dataset used for this tutorial is the [Traffic](https://storage.googleapis.com/automl-video-demo-data/traffic_videos/traffic_videos_labels.csv) dataset. The version of the dataset you use in this tutorial is stored in a public Cloud Storage bucket."
|
||||
"The dataset used for this tutorial is the [Traffic](https://storage.googleapis.com/automl-video-demo-data/traffic_videos/traffic_videos_labels.csv) dataset. The version of the dataset you will use in this tutorial is stored in a public Cloud Storage bucket."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -132,11 +128,12 @@
|
||||
"source": [
|
||||
"### Set up your local development environment\n",
|
||||
"\n",
|
||||
"**If you are using Colab or Vertex AI Workbench Notebooks**, your environment already meets all the requirements to run this notebook. You can skip this step.\n",
|
||||
"If you are using Colab or Google Cloud Notebooks, your environment already meets all the requirements to run this notebook. You can skip this step.\n",
|
||||
"\n",
|
||||
"**Otherwise**, make sure your environment meets this notebook's requirements. You need the following:\n",
|
||||
"Otherwise, make sure your environment meets this notebook's requirements. You need the following:\n",
|
||||
"\n",
|
||||
"- The Cloud Storage SDK\n",
|
||||
"- Git\n",
|
||||
"- Python 3\n",
|
||||
"- virtualenv\n",
|
||||
"- Jupyter notebook running in a virtual environment with Python 3\n",
|
||||
@@ -164,7 +161,7 @@
|
||||
"source": [
|
||||
"## Installation\n",
|
||||
"\n",
|
||||
"Install the following packages required to execute this notebook."
|
||||
"Install the latest version of Vertex SDK for Python."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -177,19 +174,33 @@
|
||||
"source": [
|
||||
"import os\n",
|
||||
"\n",
|
||||
"# The Vertex AI Workbench Notebook product has specific requirements\n",
|
||||
"IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
|
||||
"IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
|
||||
" \"/opt/deeplearning/metadata/env_version\"\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
|
||||
"if IS_WORKBENCH_NOTEBOOK:\n",
|
||||
"# Google Cloud Notebook\n",
|
||||
"if os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
|
||||
" USER_FLAG = \"--user\"\n",
|
||||
"else:\n",
|
||||
" USER_FLAG = \"\"\n",
|
||||
"\n",
|
||||
"! pip3 install --upgrade google-cloud-aiplatform google-cloud-storage $USER_FLAG -q"
|
||||
"! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "install_storage"
|
||||
},
|
||||
"source": [
|
||||
"Install the latest GA version of *google-cloud-storage* library as well."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "install_storage"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! pip3 install -U google-cloud-storage $USER_FLAG"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -200,9 +211,7 @@
|
||||
"source": [
|
||||
"### Restart the kernel\n",
|
||||
"\n",
|
||||
"Once you've installed the additional packages, you need to restart the notebook kernel so it can find the packages.\n",
|
||||
"\n",
|
||||
"**Note: You may get a message saying \"Your session crashed for an unknown reason.\", this is expected. Once this cell has finished running, continue on. You do not need to re-run any of the cells above.**"
|
||||
"Once you've installed the additional packages, you need to restart the notebook kernel so it can find the packages."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -245,43 +254,30 @@
|
||||
"\n",
|
||||
"3. [Enable the following APIs: Vertex AI APIs, Compute Engine APIs, and Cloud Storage.](https://console.cloud.google.com/flows/enableapi?apiid=ml.googleapis.com,compute_component,storage-component.googleapis.com)\n",
|
||||
"\n",
|
||||
"4. If you are running this notebook locally, you need to install the [Cloud SDK]((https://cloud.google.com/sdk)).\n",
|
||||
"4. If you are running this notebook locally, you will need to install the [Cloud SDK]((https://cloud.google.com/sdk)).\n",
|
||||
"\n",
|
||||
"5. Enter your project ID in the cell below. Then run the cell to make sure the\n",
|
||||
"Cloud SDK uses the right project for all the commands in this notebook.\n",
|
||||
"\n",
|
||||
"6. (optional) You may also specify a service account to use to run Vertex AI Pipelines in the project.\n",
|
||||
"\n",
|
||||
"**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$`."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "5aee4379e8e5"
|
||||
},
|
||||
"source": [
|
||||
"#### Set your project ID\n",
|
||||
"\n",
|
||||
"**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "dcdfccf50581"
|
||||
"id": "set_project_id"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"PROJECT_ID = \"[your-project-id]\" # @param {type:\"string\"}"
|
||||
"PROJECT_ID = \"\" # @param {type:\"string\"}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "5bf9979b96ff"
|
||||
"id": "autoset_project_id"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -296,7 +292,7 @@
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "09021c90b34c"
|
||||
"id": "set_gcloud_project_id"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -342,9 +338,9 @@
|
||||
"id": "timestamp"
|
||||
},
|
||||
"source": [
|
||||
"#### UUID\n",
|
||||
"#### Timestamp\n",
|
||||
"\n",
|
||||
"If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
|
||||
"If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -355,16 +351,9 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import random\n",
|
||||
"import string\n",
|
||||
"from datetime import datetime\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Generate a uuid of a specifed length(default=8)\n",
|
||||
"def generate_uuid(length: int = 8) -> str:\n",
|
||||
" return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"UUID = generate_uuid()"
|
||||
"TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -375,23 +364,23 @@
|
||||
"source": [
|
||||
"### Authenticate your Google Cloud account\n",
|
||||
"\n",
|
||||
"**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated.\n",
|
||||
"**If you are using Google Cloud Notebooks**, your environment is already authenticated. Skip this step.\n",
|
||||
"\n",
|
||||
"**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
|
||||
"\n",
|
||||
"**Otherwise**, follow these steps:\n",
|
||||
"\n",
|
||||
"- In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
|
||||
"In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
|
||||
"\n",
|
||||
"- **Click Create service account**.\n",
|
||||
"**Click Create service account**.\n",
|
||||
"\n",
|
||||
"- In the **Service account name** field, enter a name, and click **Create**.\n",
|
||||
"In the **Service account name** field, enter a name, and click **Create**.\n",
|
||||
"\n",
|
||||
"- In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex\" into the filter box, and select **Vertex Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
|
||||
"In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex\" into the filter box, and select **Vertex Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
|
||||
"\n",
|
||||
"- Click Create. A JSON file that contains your key downloads to your local environment.\n",
|
||||
"Click Create. A JSON file that contains your key downloads to your local environment.\n",
|
||||
"\n",
|
||||
"- Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
|
||||
"Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -410,11 +399,8 @@
|
||||
"import os\n",
|
||||
"import sys\n",
|
||||
"\n",
|
||||
"# If on Vertex AI Workbench, then don't execute this code\n",
|
||||
"IS_COLAB = \"google.colab\" in sys.modules\n",
|
||||
"if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
|
||||
" \"DL_ANACONDA_HOME\"\n",
|
||||
"):\n",
|
||||
"# If on Google Cloud Notebook, then don't execute this code\n",
|
||||
"if not os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
|
||||
" if \"google.colab\" in sys.modules:\n",
|
||||
" from google.colab import auth as google_auth\n",
|
||||
"\n",
|
||||
@@ -424,7 +410,7 @@
|
||||
" # path to your service account key and run this cell to authenticate your GCP\n",
|
||||
" # account.\n",
|
||||
" elif not os.getenv(\"IS_TESTING\"):\n",
|
||||
" %env GOOGLE_APPLICATION_CREDENTIALS '[your-service-account-key-path]'"
|
||||
" %env GOOGLE_APPLICATION_CREDENTIALS ''"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -437,7 +423,7 @@
|
||||
"\n",
|
||||
"**The following steps are required, regardless of your notebook environment.**\n",
|
||||
"\n",
|
||||
"When you initialize the Vertex AI SDK for Python, you specify a Cloud Storage staging bucket. The staging bucket is where all the data associated with your dataset and model resources are retained across sessions.\n",
|
||||
"When you initialize the Vertex SDK for Python, you specify a Cloud Storage staging bucket. The staging bucket is where all the data associated with your dataset and model resources are retained across sessions.\n",
|
||||
"\n",
|
||||
"Set the name of your Cloud Storage bucket below. Bucket names must be globally unique across all Google Cloud projects, including those outside of your organization."
|
||||
]
|
||||
@@ -446,31 +432,29 @@
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "f78cf4290843"
|
||||
"id": "bucket"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"BUCKET_NAME = \"[your-bucket-name]\" # @param {type:\"string\"}\n",
|
||||
"BUCKET_URI = f\"gs://{BUCKET_NAME}\""
|
||||
"BUCKET_URI = \"\" # @param {type:\"string\"}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "219a24ea078b"
|
||||
"id": "autoset_bucket"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
|
||||
" BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
|
||||
" BUCKET_URI = f\"gs://{BUCKET_NAME}\""
|
||||
"if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
|
||||
" BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "a8a62bec0259"
|
||||
"id": "create_bucket"
|
||||
},
|
||||
"source": [
|
||||
"**Only if your bucket doesn't already exist**: Run the following cell to create your Cloud Storage bucket."
|
||||
@@ -480,7 +464,7 @@
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "91c46850b49b"
|
||||
"id": "create_bucket"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -490,7 +474,7 @@
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "4e69d430073b"
|
||||
"id": "validate_bucket"
|
||||
},
|
||||
"source": [
|
||||
"Finally, validate access to your Cloud Storage bucket by examining its contents:"
|
||||
@@ -500,7 +484,7 @@
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "835eaacd691f"
|
||||
"id": "validate_bucket"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -513,6 +497,9 @@
|
||||
"id": "setup_vars"
|
||||
},
|
||||
"source": [
|
||||
"### Set up variables\n",
|
||||
"\n",
|
||||
"Next, set up some variables used throughout the tutorial.\n",
|
||||
"### Import libraries and define constants"
|
||||
]
|
||||
},
|
||||
@@ -537,9 +524,9 @@
|
||||
"id": "init_aip:mbsdk"
|
||||
},
|
||||
"source": [
|
||||
"## Initialize Vertex AI SDK for Python\n",
|
||||
"## Initialize Vertex SDK for Python\n",
|
||||
"\n",
|
||||
"Initialize the Vertex AI SDK for Python for your project and corresponding bucket."
|
||||
"Initialize the Vertex SDK for Python for your project and corresponding bucket."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -640,7 +627,7 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"dataset = aiplatform.VideoDataset.create(\n",
|
||||
" display_name=\"Traffic\" + \"_\" + UUID,\n",
|
||||
" display_name=\"Traffic\" + \"_\" + TIMESTAMP,\n",
|
||||
" gcs_source=[IMPORT_FILE],\n",
|
||||
" import_schema_uri=aiplatform.schema.dataset.ioformat.video.object_tracking,\n",
|
||||
")\n",
|
||||
@@ -678,7 +665,7 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"job = aiplatform.AutoMLVideoTrainingJob(\n",
|
||||
" display_name=\"traffic_\" + UUID,\n",
|
||||
" display_name=\"traffic_\" + TIMESTAMP,\n",
|
||||
" prediction_type=\"object_tracking\",\n",
|
||||
")\n",
|
||||
"\n",
|
||||
@@ -702,7 +689,7 @@
|
||||
"\n",
|
||||
"The `run` method when completed returns the `Model` resource.\n",
|
||||
"\n",
|
||||
"The execution of the training pipeline will take upto 4 hours."
|
||||
"The execution of the training pipeline will take upto 5 hours."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -715,7 +702,7 @@
|
||||
"source": [
|
||||
"model = job.run(\n",
|
||||
" dataset=dataset,\n",
|
||||
" model_display_name=\"traffic_\" + UUID,\n",
|
||||
" model_display_name=\"traffic_\" + TIMESTAMP,\n",
|
||||
" training_fraction_split=0.8,\n",
|
||||
" test_fraction_split=0.2,\n",
|
||||
")"
|
||||
@@ -741,28 +728,22 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"model_evaluations = model.list_model_evaluations()\n",
|
||||
"# Get model resource ID\n",
|
||||
"models = aiplatform.Model.list(filter=\"display_name=traffic_\" + TIMESTAMP)\n",
|
||||
"\n",
|
||||
"# Get a reference to the Model Service client\n",
|
||||
"client_options = {\"api_endpoint\": f\"{REGION}-aiplatform.googleapis.com\"}\n",
|
||||
"model_service_client = aiplatform.gapic.ModelServiceClient(\n",
|
||||
" client_options=client_options\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"model_evaluations = model_service_client.list_model_evaluations(\n",
|
||||
" parent=models[0].resource_name\n",
|
||||
")\n",
|
||||
"model_evaluation = list(model_evaluations)[0]\n",
|
||||
"print(model_evaluation)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "b3377983702b"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Print the evaluation metrics\n",
|
||||
"for evaluation in model_evaluations:\n",
|
||||
" evaluation = evaluation.to_dict()\n",
|
||||
" print(\"Model's evaluation metrics from Training:\\n\")\n",
|
||||
" metrics = evaluation[\"metrics\"]\n",
|
||||
" for metric in metrics.keys():\n",
|
||||
" print(f\"metric: {metric}, value: {metrics[metric]}\\n\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
@@ -893,7 +874,7 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"batch_predict_job = model.batch_predict(\n",
|
||||
" job_display_name=\"traffic_\" + UUID,\n",
|
||||
" job_display_name=\"traffic_\" + TIMESTAMP,\n",
|
||||
" gcs_source=gcs_input_uri,\n",
|
||||
" gcs_destination_prefix=BUCKET_URI,\n",
|
||||
" sync=False,\n",
|
||||
@@ -985,9 +966,13 @@
|
||||
"Otherwise, you can delete the individual resources you created in this tutorial:\n",
|
||||
"\n",
|
||||
"- Dataset\n",
|
||||
"- Pipeline\n",
|
||||
"- Model\n",
|
||||
"- Endpoint\n",
|
||||
"- AutoML Training Job\n",
|
||||
"- Batch Job\n",
|
||||
"- Custom Job\n",
|
||||
"- Hyperparameter Tuning Job\n",
|
||||
"- Cloud Storage Bucket"
|
||||
]
|
||||
},
|
||||
@@ -1011,8 +996,7 @@
|
||||
"# Delete the batch prediction job using the Vertex batch prediction object\n",
|
||||
"batch_predict_job.delete()\n",
|
||||
"\n",
|
||||
"delete_bucket = False\n",
|
||||
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
|
||||
"if os.getenv(\"IS_TESTING\"):\n",
|
||||
" ! gsutil -m rm -r $BUCKET_URI"
|
||||
]
|
||||
}
|
||||
|
||||
@@ -282,17 +282,6 @@
|
||||
"**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "3c8049930470"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"PROJECT_ID = \"[your-project-id]\" # @param {type:\"string\"}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
@@ -301,11 +290,15 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
|
||||
" # Get your GCP project id from gcloud\n",
|
||||
" shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
|
||||
"PROJECT_ID = \"[YOUR-PROJECT-ID]\"\n",
|
||||
"\n",
|
||||
"# Get your Google Cloud project ID from gcloud\n",
|
||||
"import os\n",
|
||||
"\n",
|
||||
"if not os.getenv(\"IS_TESTING\"):\n",
|
||||
" shell_output = !gcloud config list --format 'value(core.project)' 2>/dev/null\n",
|
||||
" PROJECT_ID = shell_output[0]\n",
|
||||
" print(\"Project ID:\", PROJECT_ID)"
|
||||
" print(\"Project ID: \", PROJECT_ID)"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -325,7 +318,8 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gcloud config set project $PROJECT_ID"
|
||||
"if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
|
||||
" PROJECT_ID = \"[your-project-id]\" # @param {type:\"string\"}"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
@@ -29,7 +29,7 @@
|
||||
"id": "JAPoU8Sm5E6e"
|
||||
},
|
||||
"source": [
|
||||
"# Using Vertex AI Feature Store with Pandas Dataframe\n",
|
||||
"# Using Vertex AI Feature Store with pandas DataFrame\n",
|
||||
"\n",
|
||||
"<table align=\"left\">\n",
|
||||
" <td>\n",
|
||||
@@ -73,17 +73,13 @@
|
||||
"source": [
|
||||
"### Objective\n",
|
||||
"\n",
|
||||
"In this notebook, you learn how to use `Vertex AI Feature Store` with pandas Dataframe.\n",
|
||||
"\n",
|
||||
"This tutorial uses the following Google Cloud ML services and resources:\n",
|
||||
"\n",
|
||||
"- Vertex AI Feature Store\n",
|
||||
"In this notebook, you learn how to use `Vertex AI Feature Store` with pandas DataFrame.\n",
|
||||
"\n",
|
||||
"The steps performed include:\n",
|
||||
"\n",
|
||||
"- Ingest Feature values from Pandas DataFrame into Feature Store's Entity types.\n",
|
||||
"- Read Entity feature values from Online Feature Store into Pandas DataFrame.\n",
|
||||
"- Batch serve feature values from your Feature Store into Pandas DataFrame.\n",
|
||||
"- Read Entity Feature values from Online Feature Store into Pandas DataFrame.\n",
|
||||
"- Batch serve Feature values from your Feature Store into Pandas DataFrame.\n",
|
||||
"\n",
|
||||
"You also learn how Vertex AI Feature Store can be useful in the below scenarios:\n",
|
||||
"\n",
|
||||
@@ -99,7 +95,7 @@
|
||||
"source": [
|
||||
"### Dataset\n",
|
||||
"\n",
|
||||
"This tutorial is a part of the Feature Store tutorial notebooks. It uses a movie recommendation dataset as an example for demonstrating various functionalities of Feature Store. The original task is to train a model to predict if a user is going to watch a movie, and serve the model online."
|
||||
"This tutorial uses a movie recommendation dataset as an example throughout all the notebooks including this one. The original task is to train a model to predict if a user is going to watch a movie and serve the model online."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -150,15 +146,12 @@
|
||||
"source": [
|
||||
"import os\n",
|
||||
"\n",
|
||||
"# The Vertex AI Workbench Notebook product has specific requirements\n",
|
||||
"IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
|
||||
"IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
|
||||
" \"/opt/deeplearning/metadata/env_version\"\n",
|
||||
")\n",
|
||||
"# The Google Cloud Notebook product has specific requirements\n",
|
||||
"IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
|
||||
"\n",
|
||||
"# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
|
||||
"# Google Cloud Notebook requires dependencies to be installed with '--user'\n",
|
||||
"USER_FLAG = \"\"\n",
|
||||
"if IS_WORKBENCH_NOTEBOOK:\n",
|
||||
"if IS_GOOGLE_CLOUD_NOTEBOOK:\n",
|
||||
" USER_FLAG = \"--user\"\n",
|
||||
" \n",
|
||||
"! pip install -U {USER_FLAG} --upgrade google-cloud-aiplatform \\\n",
|
||||
@@ -215,7 +208,7 @@
|
||||
"\n",
|
||||
"1. [Enable the Vertex AI API and Compute Engine API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component).\n",
|
||||
"\n",
|
||||
"1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
|
||||
"1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
|
||||
"\n",
|
||||
"1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
|
||||
"Cloud SDK uses the right project for all the commands in this notebook.\n",
|
||||
@@ -280,7 +273,7 @@
|
||||
"#### Region\n",
|
||||
"\n",
|
||||
"You can also change the `REGION` variable, which is used for operations\n",
|
||||
"throughout the rest of this notebook. Below are regions supported for Vertex AI. It is recommended that you choose the region closest to you.\n",
|
||||
"throughout the rest of this notebook. Below are regions supported for Vertex AI. We recommend that you choose the region closest to you.\n",
|
||||
"\n",
|
||||
"- Americas: `us-central1`\n",
|
||||
"- Europe: `europe-west4`\n",
|
||||
@@ -288,7 +281,7 @@
|
||||
"\n",
|
||||
"You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
|
||||
"\n",
|
||||
"Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
|
||||
"Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -311,29 +304,10 @@
|
||||
"id": "dr--iN2kAylZ"
|
||||
},
|
||||
"source": [
|
||||
"#### UUID\n",
|
||||
"### Authenticate your Google Cloud account\n",
|
||||
"\n",
|
||||
"If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "4e166d927e36"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import random\n",
|
||||
"import string\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Generate a uuid of a specifed length(default=8)\n",
|
||||
"def generate_uuid(length: int = 8) -> str:\n",
|
||||
" return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"UUID = generate_uuid()"
|
||||
"**If you are using Google Cloud Notebooks**, your environment is already\n",
|
||||
"authenticated. Skip this step."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -342,11 +316,6 @@
|
||||
"id": "sBCra4QMA2wR"
|
||||
},
|
||||
"source": [
|
||||
"### Authenticate your Google Cloud account\n",
|
||||
"\n",
|
||||
"**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
|
||||
"authenticated. \n",
|
||||
"\n",
|
||||
"**If you are using Colab**, run the cell below and follow the instructions\n",
|
||||
"when prompted to authenticate your account via oAuth.\n",
|
||||
"\n",
|
||||
@@ -364,7 +333,7 @@
|
||||
"into the filter box, and select\n",
|
||||
" **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
|
||||
"\n",
|
||||
"5. Click *Create*. A JSON file that contains your key downloads to your\n",
|
||||
"5. Click **Create**. A JSON file that contains your key downloads to your\n",
|
||||
"local environment.\n",
|
||||
"\n",
|
||||
"6. Enter the path to your service account key as the\n",
|
||||
@@ -379,19 +348,19 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"import sys\n",
|
||||
"\n",
|
||||
"# If you are running this notebook in Colab, run this cell and follow the\n",
|
||||
"# instructions to authenticate your GCP account. This provides access to your\n",
|
||||
"# Cloud Storage bucket and lets you submit training jobs and prediction\n",
|
||||
"# requests.\n",
|
||||
"\n",
|
||||
"import os\n",
|
||||
"import sys\n",
|
||||
"# The Google Cloud Notebook product has specific requirements\n",
|
||||
"IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
|
||||
"\n",
|
||||
"# If on Vertex AI Workbench, then don't execute this code\n",
|
||||
"IS_COLAB = \"google.colab\" in sys.modules\n",
|
||||
"if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
|
||||
" \"DL_ANACONDA_HOME\"\n",
|
||||
"):\n",
|
||||
"# If on Google Cloud Notebooks, then don't execute this code\n",
|
||||
"if not IS_GOOGLE_CLOUD_NOTEBOOK:\n",
|
||||
" if \"google.colab\" in sys.modules:\n",
|
||||
" from google.colab import auth as google_auth\n",
|
||||
"\n",
|
||||
@@ -401,7 +370,7 @@
|
||||
" # path to your service account key and run this cell to authenticate your GCP\n",
|
||||
" # account.\n",
|
||||
" elif not os.getenv(\"IS_TESTING\"):\n",
|
||||
" %env GOOGLE_APPLICATION_CREDENTIALS '[your-service-account-key-path]'"
|
||||
" %env GOOGLE_APPLICATION_CREDENTIALS ''"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -410,7 +379,7 @@
|
||||
"id": "XoEqT2Y4DJmf"
|
||||
},
|
||||
"source": [
|
||||
"### Import libraries"
|
||||
"### Import libraries and define constants"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -424,31 +393,18 @@
|
||||
"import datetime\n",
|
||||
"\n",
|
||||
"import pandas as pd\n",
|
||||
"from avro.datafile import DataFileReader\n",
|
||||
"from avro.io import DatumReader\n",
|
||||
"from google.cloud import aiplatform"
|
||||
"from google.cloud import aiplatform\n",
|
||||
"\n",
|
||||
"aiplatform.init(project=PROJECT_ID, location=REGION)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "138407556b22"
|
||||
"id": "9UvxYyGUimKw"
|
||||
},
|
||||
"source": [
|
||||
"### Initialize Vertex AI SDK for Python\n",
|
||||
"\n",
|
||||
"Initialize the Vertex AI SDK for Python for your project and region."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "8d2077ffee78"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"aiplatform.init(project=PROJECT_ID, location=REGION)"
|
||||
"## Create Feature Store Resources"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -457,12 +413,11 @@
|
||||
"id": "buQBIv3ZL3A0"
|
||||
},
|
||||
"source": [
|
||||
"## Create a Feature Store\n",
|
||||
"### Create Feature Store\n",
|
||||
"\n",
|
||||
"The method to create a Feature Store in Vertex AI returns a\n",
|
||||
"[long-running operation](https://google.aip.dev/151) (LRO). An LRO starts an asynchronous job. LROs are returned for other API methods too, such as updating or deleting a featurestore. \n",
|
||||
"\n",
|
||||
"Running the code cell below creates a featurestore and prints the process' logs."
|
||||
"The method to create a Feature Store returns a\n",
|
||||
"[long-running operation](https://google.aip.dev/151) (LRO). An LRO starts an asynchronous job. LROs are returned for other API\n",
|
||||
"methods too, such as updating or deleting a featurestore. Running the code cell creates a featurestore and prints the process logs."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -473,9 +428,9 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Create featurestore\n",
|
||||
"movie_predictions_feature_store = aiplatform.Featurestore.create(\n",
|
||||
" featurestore_id=f\"movie_predictions_{UUID}\", online_store_fixed_node_count=1\n",
|
||||
" featurestore_id=\"movie_predictions\",\n",
|
||||
" online_store_fixed_node_count=1,\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
@@ -485,7 +440,7 @@
|
||||
"id": "EpmJq75zXjmT"
|
||||
},
|
||||
"source": [
|
||||
"## Create Entity types\n",
|
||||
"### Create Entity Types\n",
|
||||
"\n",
|
||||
"Entity types can be created within the Featurestore class. Below, you create the `Users` entity type and `Movies` entity type. Process logs are printed in the output for each cell."
|
||||
]
|
||||
@@ -498,7 +453,6 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Create users entity type\n",
|
||||
"users_entity_type = movie_predictions_feature_store.create_entity_type(\n",
|
||||
" entity_type_id=\"users\",\n",
|
||||
" description=\"Users entity\",\n",
|
||||
@@ -513,7 +467,6 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Create movies entity type\n",
|
||||
"movies_entity_type = movie_predictions_feature_store.create_entity_type(\n",
|
||||
" entity_type_id=\"movies\",\n",
|
||||
" description=\"Movies entity\",\n",
|
||||
@@ -526,11 +479,8 @@
|
||||
"id": "FJW4q-0jO2Xf"
|
||||
},
|
||||
"source": [
|
||||
"## Create Features\n",
|
||||
"Features can be created within each entity type. Add defined features to the `Users` entity type and `Movies` entity type by using the following methods.\n",
|
||||
"\n",
|
||||
"### Add features using *create_feature* method\n",
|
||||
"Provide the feature information like id, type and description to the `create_feature` method of entity type."
|
||||
"### Create Features\n",
|
||||
"Features can be created within each entity type. Add defining features to the `Users` entity type and `Movies` entity type by using the following methods."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -541,21 +491,18 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Create age feature\n",
|
||||
"users_feature_age = users_entity_type.create_feature(\n",
|
||||
" feature_id=\"age\",\n",
|
||||
" value_type=\"INT64\",\n",
|
||||
" description=\"User age\",\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# Create gender feature\n",
|
||||
"users_feature_gender = users_entity_type.create_feature(\n",
|
||||
" feature_id=\"gender\",\n",
|
||||
" value_type=\"STRING\",\n",
|
||||
" description=\"User gender\",\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# Create liked_genres feature\n",
|
||||
"users_feature_liked_genres = users_entity_type.create_feature(\n",
|
||||
" feature_id=\"liked_genres\",\n",
|
||||
" value_type=\"STRING_ARRAY\",\n",
|
||||
@@ -563,18 +510,6 @@
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "ecb141839033"
|
||||
},
|
||||
"source": [
|
||||
"### Add features using batch method\n",
|
||||
"You can also create features using a config map in a dictionary format and the `batch_create_features` method. This way, you can add multiple features at once. \n",
|
||||
"\n",
|
||||
"Below, you define and create *title*, *genres* and *average_rating* features using the batch method."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
@@ -596,8 +531,17 @@
|
||||
" \"value_type\": \"DOUBLE\",\n",
|
||||
" \"description\": \"The average rating for the movie, range is [1.0-5.0]\",\n",
|
||||
" },\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
"}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "YhfOKJL_BvuM"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"movie_features = movies_entity_type.batch_create_features(\n",
|
||||
" feature_configs=movies_feature_configs,\n",
|
||||
")"
|
||||
@@ -609,15 +553,18 @@
|
||||
"id": "K3n5XdK8Xjmw"
|
||||
},
|
||||
"source": [
|
||||
"## Ingest Feature values into Entity types from dataframes\n",
|
||||
"## Ingest Feature Values into Entity Type from a Pandas DataFrame\n",
|
||||
"\n",
|
||||
"You need to ingest feature values into your entity type containing the features. It is so that you can later `read` (online) or `batch serve` (offline) the feature values from the entity type. \n",
|
||||
"\n",
|
||||
"In this step, you learn how to ingest feature values from a Pandas dataframe into an entity type. You can also import feature values from BigQuery or Google Cloud Storage.\n",
|
||||
"\n",
|
||||
"### Get data from source\n",
|
||||
"\n",
|
||||
"Define the public data sources for users and movies and copy them locally into *avro* files."
|
||||
"You need to ingest feature values into your entity type containing the features, so you can later `read` (online) or `batch serve` (offline) the feature values from the entity type. In this step, you will learn how to ingest feature values from a Pandas DataFrame into an entity type. We can also import feature values from BigQuery or Google Cloud Storage.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "BlqJ-QdTcs6W"
|
||||
},
|
||||
"source": [
|
||||
"#### Get data from source files"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -636,8 +583,17 @@
|
||||
")\n",
|
||||
"\n",
|
||||
"USERS_AVRO_FN = \"users.avro\"\n",
|
||||
"MOVIES_AVRO_FN = \"movies.avro\"\n",
|
||||
"\n",
|
||||
"MOVIES_AVRO_FN = \"movies.avro\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "KqIH_bS-5OW5"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil cp $GCS_USERS_AVRO_URI $USERS_AVRO_FN\n",
|
||||
"! gsutil cp $GCS_MOVIES_AVRO_URI $MOVIES_AVRO_FN"
|
||||
]
|
||||
@@ -648,9 +604,7 @@
|
||||
"id": "Fd6Z0jfR5OW5"
|
||||
},
|
||||
"source": [
|
||||
"### Load data from avro files \n",
|
||||
"\n",
|
||||
"Load users and movies data from avro files into Pandas dataframes."
|
||||
"#### Load Avro Files into Pandas DataFrames"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -661,7 +615,10 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Define a class for reading the avro data\n",
|
||||
"from avro.datafile import DataFileReader\n",
|
||||
"from avro.io import DatumReader\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"class AvroReader:\n",
|
||||
" def __init__(self, data_file):\n",
|
||||
" self.avro_reader = DataFileReader(open(data_file, \"rb\"), DatumReader())\n",
|
||||
@@ -679,7 +636,6 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Load users data from avro file\n",
|
||||
"users_avro_reader = AvroReader(data_file=USERS_AVRO_FN)\n",
|
||||
"users_source_df = users_avro_reader.to_dataframe()\n",
|
||||
"print(users_source_df)"
|
||||
@@ -693,7 +649,6 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Load movies data from avro file\n",
|
||||
"movies_avro_reader = AvroReader(data_file=MOVIES_AVRO_FN)\n",
|
||||
"movies_source_df = movies_avro_reader.to_dataframe()\n",
|
||||
"print(movies_source_df)"
|
||||
@@ -705,9 +660,7 @@
|
||||
"id": "bgb0WGwX5OW6"
|
||||
},
|
||||
"source": [
|
||||
"### Ingest Feature values into Entity types\n",
|
||||
"\n",
|
||||
"Load the feature values into `users` entity type providing the id fields and time field."
|
||||
"#### Ingest Feature Values into _Users_ Entity Type"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -732,7 +685,7 @@
|
||||
"id": "PCAdQ3cF5OW6"
|
||||
},
|
||||
"source": [
|
||||
"Load the feature values into `movie` entity type providing the id fields and time field."
|
||||
"#### Ingest Feature Values into _Movies_ Entity Type"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -757,12 +710,10 @@
|
||||
"id": "pIYLZwao5OW6"
|
||||
},
|
||||
"source": [
|
||||
"## Read/serve Entity's feature values online from Feature Store\n",
|
||||
"## Read/Online Serve Entity's Feature Values from Vertex AI Online Feature Store\n",
|
||||
"\n",
|
||||
"Feature Store allows [online serving](https://cloud.google.com/vertex-ai/docs/featurestore/serving-online)\n",
|
||||
"which lets you read feature values for small batches of entities. It works well when you want to read values of selected features from an entity or multiple entities in an entity type.\n",
|
||||
"\n",
|
||||
"### Read feature values for users"
|
||||
"which lets you read feature values for small batches of entities. It works well when you want to read values of selected features from an entity or multiple entities in an entity type."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -779,15 +730,6 @@
|
||||
"print(users_read_df)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "b2cfa09ef11d"
|
||||
},
|
||||
"source": [
|
||||
"### Read feature values for movies"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
@@ -809,13 +751,18 @@
|
||||
"id": "AK2Glzkq5OW7"
|
||||
},
|
||||
"source": [
|
||||
"## Batch serve feature values from Feature Store\n",
|
||||
"## Batch Serve Feature Values from Vertex AI Feature Store\n",
|
||||
"\n",
|
||||
"Batch Serving is used to fetch a large batch of feature values for high-throughput, and is typically used for training a model or batch prediction. In this section, you learn how to prepare training examples by using the Feature Store's batch serve function.\n",
|
||||
"\n",
|
||||
"### Read instances from source file\n",
|
||||
"\n",
|
||||
"Define the source file and destination file. "
|
||||
"Batch Serving is used to fetch a large batch of feature values for high-throughput, and is typically used for training a model or batch prediction. In this section, you learn how to prepare training examples by using the Feature Store's batch serve function."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "hxsotHUe5OW7"
|
||||
},
|
||||
"source": [
|
||||
"#### Read instances from source file"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -830,15 +777,6 @@
|
||||
"READ_INSTANCES_CSV_FN = \"data.csv\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "3f2c558b649f"
|
||||
},
|
||||
"source": [
|
||||
"Copy the instances from the source file to the destination file locally."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
@@ -856,9 +794,7 @@
|
||||
"id": "T5DW1MFt5OW7"
|
||||
},
|
||||
"source": [
|
||||
"### Load the instances\n",
|
||||
"\n",
|
||||
"Load the instances from CSV file into a Pandas dataframe."
|
||||
"#### Load CSV file into a Pandas DataFrame"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -879,9 +815,7 @@
|
||||
"id": "LsgNNH8G5OW8"
|
||||
},
|
||||
"source": [
|
||||
"### Change the data type\n",
|
||||
"\n",
|
||||
"Change the data type of the timestamp field from `Timestamp` to `Datetime64`."
|
||||
"#### Change the Dtype of `Timestamp` to `Datetime64`"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -903,9 +837,7 @@
|
||||
"id": "ao1dC5Pc5OW8"
|
||||
},
|
||||
"source": [
|
||||
"### Batch serve feature values from Feature Store\n",
|
||||
"\n",
|
||||
"Serve the batch response to a dataframe and display the data."
|
||||
"#### Batch Serve Feature Values from Movie Predictions Feature Store"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -926,21 +858,43 @@
|
||||
"movie_predictions_df"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "29gLNORP5OW8"
|
||||
},
|
||||
"source": [
|
||||
"## Read the Updated Feature Values"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "XN84znoI5OW8"
|
||||
},
|
||||
"source": [
|
||||
"## Read the latest feature values\n",
|
||||
"\n",
|
||||
"In Feature Store, you access the latest or the last available feature values unless a specific time is provided. Now, you test this feature by ingesting new data to the entity types and reading it from the Feature Store.\n",
|
||||
"\n",
|
||||
"### Ingest updated feature values\n",
|
||||
"\n",
|
||||
"Now, you update the feature values by running the following cell. \n",
|
||||
"\n",
|
||||
"**Note:** For comparison, you can try printing the feature values read from the entity types earlier (those in `movies_read_df` variable). "
|
||||
"#### Feature Values from last ingestion\n",
|
||||
"Recall read from the Entity Type shows Feature Values from the last ingestion."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "wtmshq_n5OW9"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"print(movies_read_df)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "feTUJjqG5OW9"
|
||||
},
|
||||
"source": [
|
||||
"#### Ingest updated Feature Values"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -951,13 +905,21 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Create a dataframe for the new data\n",
|
||||
"update_movies_df = pd.DataFrame(\n",
|
||||
" data=[[\"movie_03\", 4.3], [\"movie_04\", 4.8]],\n",
|
||||
" columns=[\"movie_id\", \"average_rating\"],\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# Ingest the new data from the dataframe\n",
|
||||
"print(update_movies_df)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "aKKhSzUc5OW9"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"movies_entity_type.ingest_from_df(\n",
|
||||
" feature_ids=[\"average_rating\"],\n",
|
||||
" feature_time=datetime.datetime.now(),\n",
|
||||
@@ -972,9 +934,8 @@
|
||||
"id": "s47WCIvL5OW9"
|
||||
},
|
||||
"source": [
|
||||
"### Fetch the latest feature values\n",
|
||||
"\n",
|
||||
"Reading from the entity type gives you the updated feature values from the latest ingestion."
|
||||
"#### Latest Feature Values\n",
|
||||
"Read from the Entity Type shows updated Feature values from the latest ingestion."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -992,18 +953,23 @@
|
||||
"print(update_movies_read_df)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "wsvCRzn_5OW9"
|
||||
},
|
||||
"source": [
|
||||
"## Point-in-Time Correctness"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "R1YGRNsW5OW9"
|
||||
},
|
||||
"source": [
|
||||
"## Point-in-time correctness\n",
|
||||
"\n",
|
||||
"Vertex AI Feature Store captures feature values for a feature at a specific point in time. In case there are missing values in your past data, you can backfill them using batch serving.\n",
|
||||
"\n",
|
||||
"### Missing data\n",
|
||||
"Recall that response from the batch serve from last ingestion has some missing data in it."
|
||||
"#### Missing data\n",
|
||||
"Recall Batch Serve from the last ingestion has some missing data in it."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1014,7 +980,6 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Print the response\n",
|
||||
"print(movie_predictions_df)"
|
||||
]
|
||||
},
|
||||
@@ -1024,9 +989,7 @@
|
||||
"id": "abQRF6mx5OW-"
|
||||
},
|
||||
"source": [
|
||||
"### Backfill/correct point-in-time data\n",
|
||||
"\n",
|
||||
"Impute the missing data based on the timestamps."
|
||||
"#### Backfill/Correct point-in-time data"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1037,7 +1000,6 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Impute the users data\n",
|
||||
"backfill_users_df = pd.DataFrame(\n",
|
||||
" data=[[\"bob\", 34, \"Male\", [\"Drama\"], \"2020-02-13 09:35:15\"]],\n",
|
||||
" columns=[\"user_id\", \"age\", \"gender\", \"liked_genres\", \"update_time\"],\n",
|
||||
@@ -1054,7 +1016,6 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Impute the movies data\n",
|
||||
"backfill_movies_df = pd.DataFrame(\n",
|
||||
" data=[[\"movie_04\", 4.2, \"The Dark Knight\", \"Action\", \"2020-02-13 09:35:15\"]],\n",
|
||||
" columns=[\"movie_id\", \"average_rating\", \"title\", \"genres\", \"update_time\"],\n",
|
||||
@@ -1069,9 +1030,7 @@
|
||||
"id": "WXb4JUhu5OW-"
|
||||
},
|
||||
"source": [
|
||||
"### Ingest the backfilled/corrected data\n",
|
||||
"\n",
|
||||
"Ingest the imputed point-in-time data from dataframe to the entity types in feature store."
|
||||
"#### Ingest backfilled/corrected point-in-time data from dataframe"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1082,7 +1041,6 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Ingest the users data\n",
|
||||
"users_entity_type.ingest_from_df(\n",
|
||||
" feature_ids=[\"age\", \"gender\", \"liked_genres\"],\n",
|
||||
" feature_time=\"update_time\",\n",
|
||||
@@ -1099,7 +1057,6 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Ingest the users data\n",
|
||||
"movies_entity_type.ingest_from_df(\n",
|
||||
" feature_ids=[\"average_rating\", \"title\", \"genres\"],\n",
|
||||
" feature_time=\"update_time\",\n",
|
||||
@@ -1114,8 +1071,8 @@
|
||||
"id": "1e62Ku6W5OW_"
|
||||
},
|
||||
"source": [
|
||||
"### Fetch the latest data\n",
|
||||
"Batch serve the latest ingested data with backfill/correction to a dataframe to ensure the feature store is updated. "
|
||||
"#### Latest ingestion with imputed missing data\n",
|
||||
"Batch Serve from the latest ingestion with backfill/correction has reduced missing data."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1147,7 +1104,7 @@
|
||||
"To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
|
||||
"project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
|
||||
"\n",
|
||||
"Otherwise, you can delete the individual resources you created in this tutorial:"
|
||||
"You can also keep the project but delete the featurestore by running the code below:"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1158,7 +1115,6 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Delete the feature store\n",
|
||||
"movie_predictions_feature_store.delete(force=True)"
|
||||
]
|
||||
}
|
||||
|
||||
@@ -758,7 +758,7 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"tree_ah_index = aiplatform.MatchingEngineIndex.create_tree_ah_index(\n",
|
||||
" display_name=DISPLAY_NAME_BRUTE_FORCE,\n",
|
||||
" display_name=DISPLAY_NAME,\n",
|
||||
" contents_delta_uri=EMBEDDINGS_INITIAL_URI,\n",
|
||||
" dimensions=DIMENSIONS,\n",
|
||||
" approximate_neighbors_count=150,\n",
|
||||
|
||||
+182
-189
@@ -33,67 +33,20 @@
|
||||
"\n",
|
||||
"<table align=\"left\">\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ10%20Vertex%20SDK%20Custom%20Scikit-Learn%20with%20pre-built%20training%20container.ipynb\">\n",
|
||||
" <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/ai-platform-samples/blob/master/vertex-ai-samples/tree/master/notebooks/official/migration/UJ10%20Vertex%20SDK%20Custom%20Scikit-Learn%20with%20pre-built%20training%20container.ipynb\">\n",
|
||||
" <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ10%20Vertex%20SDK%20Custom%20Scikit-Learn%20with%20pre-built%20training%20container.ipynb\">\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/ai-platform-samples/blob/master/vertex-ai-samples/tree/master/notebooks/official/migration/UJ10%20Vertex%20SDK%20Custom%20Scikit-Learn%20with%20pre-built%20training%20container.ipynb\">\n",
|
||||
" <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/migration/UJ10%20Vertex%20SDK%20Custom%20Scikit-Learn%20with%20pre-built%20training%20container.ipynb\">\n",
|
||||
" <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
|
||||
" Open in Vertex AI Workbench\n",
|
||||
" </a>\n",
|
||||
" </td> \n",
|
||||
"</table>\n",
|
||||
"<br/><br/><br/>"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "7a8a13b86a8b"
|
||||
},
|
||||
"source": [
|
||||
"## Overview\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"This tutorial demonstrates how to use the Vertex AI SDK for Python to train and deploy a custom tabular classification scikit-learn model for batch prediction."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "618cfedf829a"
|
||||
},
|
||||
"source": [
|
||||
"### Objective\n",
|
||||
"\n",
|
||||
"In this tutorial, you learn to use `Vertex AI Training` to create a custom trained model and use `Vertex AI Batch Prediction` to do a batch prediction on the trained model.\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"You learn how to create a custom-trained model from a Python script in a Docker container using the Vertex AI SDK for Python, and then do a prediction on the deployed model by sending data.\n",
|
||||
"\n",
|
||||
"This tutorial uses the following Google Cloud ML services:\n",
|
||||
"\n",
|
||||
"- `Vertex AI Training`\n",
|
||||
"- `Vertex AI Batch Prediction`\n",
|
||||
"- `Vertex AI Model` resource\n",
|
||||
"- `Vertex AI Endpoint` resource\n",
|
||||
"\n",
|
||||
"The steps performed include:\n",
|
||||
"\n",
|
||||
"- Create a `Vertex AI` custom job for training a scikit-learn model.\n",
|
||||
"- Upload the trained model artifacts as a `Model` resource.\n",
|
||||
"- Make a batch prediction.\n",
|
||||
"- Deploy model to a endpoint\n",
|
||||
"- Make a online prediction"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
@@ -102,7 +55,7 @@
|
||||
"source": [
|
||||
"### Dataset\n",
|
||||
"\n",
|
||||
"The dataset used for this tutorial is the UCI Machine Learning [US Census Data (1990) dataset](https://archive.ics.uci.edu/ml/datasets/US+Census+Data+(1990)).The version of the dataset you use in this tutorial is stored in a public Cloud Storage bucket.\n",
|
||||
"The dataset used for this tutorial is the UCI Machine Learning [US Census Data (1990) dataset](https://archive.ics.uci.edu/ml/datasets/US+Census+Data+(1990)).The version of the dataset you will use in this tutorial is stored in a public Cloud Storage bucket.\n",
|
||||
"\n",
|
||||
"The dataset predicts whether a persons income will be above $50K USD."
|
||||
]
|
||||
@@ -135,37 +88,29 @@
|
||||
"source": [
|
||||
"### Set up your local development environment\n",
|
||||
"\n",
|
||||
"**If you are using Colab or Vertex AI Workbench Notebooks**, your environment already meets all the requirements to run this notebook. You can skip this step.\n",
|
||||
"If you are using Colab or Google Cloud Notebooks, your environment already meets all the requirements to run this notebook. You can skip this step.\n",
|
||||
"\n",
|
||||
"**_NOTE_**: This notebook has been tested in the following environment:\n",
|
||||
"Otherwise, make sure your environment meets this notebook's requirements. You need the following:\n",
|
||||
"\n",
|
||||
"* Python version = 3.9\n",
|
||||
"- The Cloud Storage SDK\n",
|
||||
"- Git\n",
|
||||
"- Python 3\n",
|
||||
"- virtualenv\n",
|
||||
"- Jupyter notebook running in a virtual environment with Python 3\n",
|
||||
"\n",
|
||||
"**Otherwise**, make sure your environment meets this notebook's requirements.\n",
|
||||
"You need the following:\n",
|
||||
"The Cloud Storage guide to [Setting up a Python development environment](https://cloud.google.com/python/setup) and the [Jupyter installation guide](https://jupyter.org/install) provide detailed instructions for meeting these requirements. The following steps provide a condensed set of instructions:\n",
|
||||
"\n",
|
||||
"* The Google Cloud SDK\n",
|
||||
"1. [Install and initialize the SDK](https://cloud.google.com/sdk/docs/).\n",
|
||||
"\n",
|
||||
"The Google Cloud guide to [Setting up a Python development\n",
|
||||
"environment](https://cloud.google.com/python/setup) and the [Jupyter\n",
|
||||
"installation guide](https://jupyter.org/install) provide detailed instructions\n",
|
||||
"for meeting these requirements. The following steps provide a condensed set of\n",
|
||||
"instructions:\n",
|
||||
"2. [Install Python 3](https://cloud.google.com/python/setup#installing_python).\n",
|
||||
"\n",
|
||||
"1. [Install and initialize the Cloud SDK.](https://cloud.google.com/sdk/docs/)\n",
|
||||
"3. [Install virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv) and create a virtual environment that uses Python 3. Activate the virtual environment.\n",
|
||||
"\n",
|
||||
"1. [Install Python 3.](https://cloud.google.com/python/setup#installing_python)\n",
|
||||
"4. To install Jupyter, run `pip3 install jupyter` on the command-line in a terminal shell.\n",
|
||||
"\n",
|
||||
"1. [Install\n",
|
||||
" virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv)\n",
|
||||
" and create a virtual environment that uses Python 3. Activate the virtual environment.\n",
|
||||
"5. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
|
||||
"\n",
|
||||
"1. To install Jupyter, run `pip3 install jupyter` on the\n",
|
||||
"command-line in a terminal shell.\n",
|
||||
"\n",
|
||||
"1. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
|
||||
"\n",
|
||||
"1. Open this notebook in the Jupyter Notebook Dashboard."
|
||||
"6. Open this notebook in the Jupyter Notebook Dashboard.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -176,7 +121,7 @@
|
||||
"source": [
|
||||
"## Installation\n",
|
||||
"\n",
|
||||
"Install the following packages required to execute this notebook. "
|
||||
"Install the latest version of Vertex SDK for Python."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -189,18 +134,45 @@
|
||||
"source": [
|
||||
"import os\n",
|
||||
"\n",
|
||||
"# The Vertex AI Workbench Notebook product has specific requirements\n",
|
||||
"IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
|
||||
"IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
|
||||
" \"/opt/deeplearning/metadata/env_version\"\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
|
||||
"USER_FLAG = \"\"\n",
|
||||
"if IS_WORKBENCH_NOTEBOOK:\n",
|
||||
"# Google Cloud Notebook\n",
|
||||
"if os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
|
||||
" USER_FLAG = \"--user\"\n",
|
||||
"else:\n",
|
||||
" USER_FLAG = \"\"\n",
|
||||
"\n",
|
||||
"! pip3 install --upgrade google-cloud-aiplatform google-cloud-storage tensorflow $USER_FLAG -q"
|
||||
"! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "install_storage"
|
||||
},
|
||||
"source": [
|
||||
"Install the latest GA version of *google-cloud-storage* library as well."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "install_storage"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! pip3 install -U google-cloud-storage $USER_FLAG"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "install_tensorflow"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"if os.getenv(\"IS_TESTING\"):\n",
|
||||
" ! pip3 install --upgrade tensorflow $USER_FLAG"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -222,7 +194,6 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Automatically restart kernel after installs\n",
|
||||
"import os\n",
|
||||
"\n",
|
||||
"if not os.getenv(\"IS_TESTING\"):\n",
|
||||
@@ -241,33 +212,26 @@
|
||||
"source": [
|
||||
"## Before you begin\n",
|
||||
"\n",
|
||||
"### GPU runtime\n",
|
||||
"\n",
|
||||
"This tutorial does not require a GPU runtime.\n",
|
||||
"\n",
|
||||
"### Set up your Google Cloud project\n",
|
||||
"\n",
|
||||
"**The following steps are required, regardless of your notebook environment.**\n",
|
||||
"\n",
|
||||
"1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
|
||||
"\n",
|
||||
"1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
|
||||
"2. [Make sure that billing is enabled for your project.](https://cloud.google.com/billing/docs/how-to/modify-project)\n",
|
||||
"\n",
|
||||
"1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com). \n",
|
||||
"3. [Enable the following APIs: Vertex AI APIs, Compute Engine APIs, and Cloud Storage.](https://console.cloud.google.com/flows/enableapi?apiid=ml.googleapis.com,compute_component,storage-component.googleapis.com)\n",
|
||||
"\n",
|
||||
"1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
|
||||
"4. If you are running this notebook locally, you will need to install the [Cloud SDK]((https://cloud.google.com/sdk)).\n",
|
||||
"\n",
|
||||
"1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
|
||||
"5. Enter your project ID in the cell below. Then run the cell to make sure the\n",
|
||||
"Cloud SDK uses the right project for all the commands in this notebook.\n",
|
||||
"\n",
|
||||
"**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "5aee4379e8e5"
|
||||
},
|
||||
"source": [
|
||||
"#### Set your project ID\n",
|
||||
"\n",
|
||||
"**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
|
||||
"**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$`."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -335,10 +299,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"REGION = \"[your-region]\" # @param {type: \"string\"}\n",
|
||||
"\n",
|
||||
"if REGION == \"[your-region]\":\n",
|
||||
" REGION = \"us-central1\""
|
||||
"REGION = \"us-central1\" # @param {type: \"string\"}"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -347,9 +308,9 @@
|
||||
"id": "timestamp"
|
||||
},
|
||||
"source": [
|
||||
"#### UUID\n",
|
||||
"#### Timestamp\n",
|
||||
"\n",
|
||||
"If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
|
||||
"If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -360,16 +321,9 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import random\n",
|
||||
"import string\n",
|
||||
"from datetime import datetime\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Generate a uuid of a specifed length(default=8)\n",
|
||||
"def generate_uuid(length: int = 8) -> str:\n",
|
||||
" return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"UUID = generate_uuid()"
|
||||
"TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -380,31 +334,23 @@
|
||||
"source": [
|
||||
"### Authenticate your Google Cloud account\n",
|
||||
"\n",
|
||||
"**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
|
||||
"authenticated. \n",
|
||||
"**If you are using Google Cloud Notebooks**, your environment is already authenticated. Skip this step.\n",
|
||||
"\n",
|
||||
"**If you are using Colab**, run the cell below and follow the instructions\n",
|
||||
"when prompted to authenticate your account via oAuth.\n",
|
||||
"**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
|
||||
"\n",
|
||||
"**Otherwise**, follow these steps:\n",
|
||||
"\n",
|
||||
"1. In the Cloud Console, go to the [**Create service account key**\n",
|
||||
" page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
|
||||
"In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
|
||||
"\n",
|
||||
"2. Click **Create service account**.\n",
|
||||
"**Click Create service account**.\n",
|
||||
"\n",
|
||||
"3. In the **Service account name** field, enter a name, and\n",
|
||||
" click **Create**.\n",
|
||||
"In the **Service account name** field, enter a name, and click **Create**.\n",
|
||||
"\n",
|
||||
"4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
|
||||
"into the filter box, and select\n",
|
||||
" **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
|
||||
"In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex\" into the filter box, and select **Vertex Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
|
||||
"\n",
|
||||
"5. Click *Create*. A JSON file that contains your key downloads to your\n",
|
||||
"local environment.\n",
|
||||
"Click Create. A JSON file that contains your key downloads to your local environment.\n",
|
||||
"\n",
|
||||
"6. Enter the path to your service account key as the\n",
|
||||
"`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
|
||||
"Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -423,11 +369,8 @@
|
||||
"import os\n",
|
||||
"import sys\n",
|
||||
"\n",
|
||||
"# If on Vertex AI Workbench, then don't execute this code\n",
|
||||
"IS_COLAB = \"google.colab\" in sys.modules\n",
|
||||
"if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
|
||||
" \"DL_ANACONDA_HOME\"\n",
|
||||
"):\n",
|
||||
"# If on Google Cloud Notebook, then don't execute this code\n",
|
||||
"if not os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
|
||||
" if \"google.colab\" in sys.modules:\n",
|
||||
" from google.colab import auth as google_auth\n",
|
||||
"\n",
|
||||
@@ -437,7 +380,7 @@
|
||||
" # path to your service account key and run this cell to authenticate your GCP\n",
|
||||
" # account.\n",
|
||||
" elif not os.getenv(\"IS_TESTING\"):\n",
|
||||
" %env GOOGLE_APPLICATION_CREDENTIALS '[your-service-account-key-path]'"
|
||||
" %env GOOGLE_APPLICATION_CREDENTIALS ''"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -463,8 +406,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"BUCKET_NAME = \"[your-bucket-name]\" # @param {type:\"string\"}\n",
|
||||
"BUCKET_URI = f\"gs://{BUCKET_NAME}\""
|
||||
"BUCKET_NAME = \"gs://[your-bucket-name]\" # @param {type:\"string\"}"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -475,9 +417,8 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
|
||||
" BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
|
||||
" BUCKET_URI = f\"gs://{BUCKET_NAME}\""
|
||||
"if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
|
||||
" BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -497,7 +438,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI"
|
||||
"! gsutil mb -l $REGION $BUCKET_NAME"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -517,7 +458,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil ls -al $BUCKET_URI"
|
||||
"! gsutil ls -al $BUCKET_NAME"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -526,6 +467,9 @@
|
||||
"id": "setup_vars"
|
||||
},
|
||||
"source": [
|
||||
"### Set up variables\n",
|
||||
"\n",
|
||||
"Next, set up some variables used throughout the tutorial.\n",
|
||||
"### Import libraries and define constants"
|
||||
]
|
||||
},
|
||||
@@ -537,11 +481,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import json\n",
|
||||
"import os\n",
|
||||
"\n",
|
||||
"import google.cloud.aiplatform as aip\n",
|
||||
"import tensorflow as tf"
|
||||
"import google.cloud.aiplatform as aip"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -563,7 +503,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"aip.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
|
||||
"aip.init(project=PROJECT_ID, staging_bucket=BUCKET_NAME)"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -592,13 +532,10 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"TRAIN_VERSION = \"scikit-learn-cpu.0-23\"\n",
|
||||
"DEPLOY_VERSION = \"sklearn-cpu.1-0\"\n",
|
||||
"DEPLOY_VERSION = \"sklearn-cpu.0-23\"\n",
|
||||
"\n",
|
||||
"TRAIN_IMAGE = \"us-docker.pkg.dev/vertex-ai/training/{}:latest\".format(TRAIN_VERSION)\n",
|
||||
"DEPLOY_IMAGE = \"us-docker.pkg.dev/vertex-ai/prediction/{}:latest\".format(DEPLOY_VERSION)\n",
|
||||
"\n",
|
||||
"print(\"Training:\", TRAIN_IMAGE)\n",
|
||||
"print(\"Deployment:\", DEPLOY_IMAGE)"
|
||||
"TRAIN_IMAGE = \"gcr.io/cloud-aiplatform/training/{}:latest\".format(TRAIN_VERSION)\n",
|
||||
"DEPLOY_IMAGE = \"gcr.io/cloud-aiplatform/prediction/{}:latest\".format(DEPLOY_VERSION)"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -611,7 +548,7 @@
|
||||
"\n",
|
||||
"Next, set the machine type to use for training and prediction.\n",
|
||||
"\n",
|
||||
"- Set the variables `TRAIN_COMPUTE` and `DEPLOY_COMPUTE` to configure the compute resources for the VMs you use for for training and prediction.\n",
|
||||
"- Set the variables `TRAIN_COMPUTE` and `DEPLOY_COMPUTE` to configure the compute resources for the VMs you will use for for training and prediction.\n",
|
||||
" - `machine type`\n",
|
||||
" - `n1-standard`: 3.75GB of memory per vCPU.\n",
|
||||
" - `n1-highmem`: 6.5GB of memory per vCPU\n",
|
||||
@@ -663,7 +600,7 @@
|
||||
"\n",
|
||||
"#### Package layout\n",
|
||||
"\n",
|
||||
"Before you start the training, you look at how a Python package is assembled for a custom training job. When unarchived, the package contains the following directory/file layout.\n",
|
||||
"Before you start the training, you will look at how a Python package is assembled for a custom training job. When unarchived, the package contains the following directory/file layout.\n",
|
||||
"\n",
|
||||
"- PKG-INFO\n",
|
||||
"- README.md\n",
|
||||
@@ -679,7 +616,7 @@
|
||||
"\n",
|
||||
"#### Package Assembly\n",
|
||||
"\n",
|
||||
"In the following cells, you assemble the training package."
|
||||
"In the following cells, you will assemble the training package."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -700,7 +637,7 @@
|
||||
"setup_cfg = \"[egg_info]\\n\\ntag_build =\\n\\ntag_date = 0\"\n",
|
||||
"! echo \"$setup_cfg\" > custom/setup.cfg\n",
|
||||
"\n",
|
||||
"setup_py = \"import setuptools\\n\\nsetuptools.setup(\\n\\n install_requires=[\\n\\n 'tensorflow_datasets',\\n\\n ],\\n\\n packages=setuptools.find_packages())\"\n",
|
||||
"setup_py = \"import setuptools\\n\\nsetuptools.setup(\\n\\n install_requires=[\\n\\n 'tensorflow_datasets==1.3.0',\\n\\n ],\\n\\n packages=setuptools.find_packages())\"\n",
|
||||
"! echo \"$setup_py\" > custom/setup.py\n",
|
||||
"\n",
|
||||
"pkg_info = \"Metadata-Version: 1.0\\n\\nName: US Census Data (1990) tabular binary classification\\n\\nVersion: 0.0.0\\n\\nSummary: Demostration training script\\n\\nHome-page: www.google.com\\n\\nAuthor: Google\\n\\nAuthor-email: aferlitsch@google.com\\n\\nLicense: Public\\n\\nDescription: Demo\\n\\nPlatform: Vertex\"\n",
|
||||
@@ -742,6 +679,7 @@
|
||||
"parser.add_argument('--model-dir', dest='model_dir',\n",
|
||||
" default=os.getenv('AIP_MODEL_DIR'), type=str, help='Model dir.')\n",
|
||||
"args = parser.parse_args()\n",
|
||||
"\n",
|
||||
"print('Python Version = {}'.format(sys.version))\n",
|
||||
"\n",
|
||||
"# Public bucket holding the census data\n",
|
||||
@@ -857,9 +795,6 @@
|
||||
"subdirs = args.model_dir.split('/')[3:]\n",
|
||||
"subdir = subdirs[0]\n",
|
||||
"subdirs.pop(0)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"for comp in subdirs:\n",
|
||||
" subdir = os.path.join(subdir, comp)\n",
|
||||
"\n",
|
||||
@@ -868,7 +803,7 @@
|
||||
"\n",
|
||||
"# Upload the model to GCS\n",
|
||||
"bucket = storage.Client().bucket(bucket)\n",
|
||||
"blob = bucket.blob(subdir + 'model.joblib')\n",
|
||||
"blob = bucket.blob(subdir + '/model.joblib')\n",
|
||||
"blob.upload_from_filename('model.joblib')"
|
||||
]
|
||||
},
|
||||
@@ -894,7 +829,7 @@
|
||||
"! rm -f custom.tar custom.tar.gz\n",
|
||||
"! tar cvf custom.tar custom\n",
|
||||
"! gzip custom.tar\n",
|
||||
"! gsutil cp custom.tar.gz $BUCKET_URI/trainer_census.tar.gz"
|
||||
"! gsutil cp custom.tar.gz $BUCKET_NAME/trainer_census.tar.gz"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -945,10 +880,10 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"job = aip.CustomTrainingJob(\n",
|
||||
" display_name=\"census_\" + UUID,\n",
|
||||
" display_name=\"census_\" + TIMESTAMP,\n",
|
||||
" script_path=\"custom/trainer/task.py\",\n",
|
||||
" container_uri=TRAIN_IMAGE,\n",
|
||||
" requirements=[\"gcsfs\", \"tensorflow-datasets\"],\n",
|
||||
" requirements=[\"gcsfs==0.7.1\", \"tensorflow-datasets==4.4\"],\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"print(job)"
|
||||
@@ -989,7 +924,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"MODEL_DIR = \"{}/{}\".format(BUCKET_URI, UUID)\n",
|
||||
"MODEL_DIR = \"{}/{}\".format(BUCKET_NAME, TIMESTAMP)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"job.run(\n",
|
||||
@@ -1036,7 +971,7 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"model = aip.Model.upload(\n",
|
||||
" display_name=\"census_\" + UUID,\n",
|
||||
" display_name=\"census_\" + TIMESTAMP,\n",
|
||||
" artifact_uri=MODEL_DIR,\n",
|
||||
" serving_container_image_uri=DEPLOY_IMAGE,\n",
|
||||
" sync=False,\n",
|
||||
@@ -1086,7 +1021,7 @@
|
||||
"source": [
|
||||
"### Make test items\n",
|
||||
"\n",
|
||||
"You use synthetic data as test data items. Don't be concerned that we are using synthetic data -- we just want to demonstrate how to make a prediction."
|
||||
"You will use synthetic data as a test data items. Don't be concerned that we are using synthetic data -- we just want to demonstrate how to make a prediction."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1141,7 +1076,7 @@
|
||||
"source": [
|
||||
"### Make the batch input file\n",
|
||||
"\n",
|
||||
"Now make a batch input file, which you store in your local Cloud Storage bucket. Each instance in the prediction request is a list of the form:\n",
|
||||
"Now make a batch input file, which you will store in your local Cloud Storage bucket. Each instance in the prediction request is a list of the form:\n",
|
||||
"\n",
|
||||
" [ [ content_1], [content_2] ]\n",
|
||||
"\n",
|
||||
@@ -1156,7 +1091,11 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"gcs_input_uri = BUCKET_URI + \"/\" + \"test.jsonl\"\n",
|
||||
"import json\n",
|
||||
"\n",
|
||||
"import tensorflow as tf\n",
|
||||
"\n",
|
||||
"gcs_input_uri = BUCKET_NAME + \"/\" + \"test.jsonl\"\n",
|
||||
"with tf.io.gfile.GFile(gcs_input_uri, \"w\") as f:\n",
|
||||
" for i in INSTANCES:\n",
|
||||
" f.write(json.dumps(i) + \"\\n\")\n",
|
||||
@@ -1195,9 +1134,9 @@
|
||||
"MAX_NODES = 1\n",
|
||||
"\n",
|
||||
"batch_predict_job = model.batch_predict(\n",
|
||||
" job_display_name=\"census_\" + UUID,\n",
|
||||
" job_display_name=\"census_\" + TIMESTAMP,\n",
|
||||
" gcs_source=gcs_input_uri,\n",
|
||||
" gcs_destination_prefix=BUCKET_URI,\n",
|
||||
" gcs_destination_prefix=BUCKET_NAME,\n",
|
||||
" instances_format=\"jsonl\",\n",
|
||||
" predictions_format=\"jsonl\",\n",
|
||||
" model_parameters=None,\n",
|
||||
@@ -1309,6 +1248,8 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import json\n",
|
||||
"\n",
|
||||
"bp_iter_outputs = batch_predict_job.iter_outputs()\n",
|
||||
"\n",
|
||||
"prediction_results = list()\n",
|
||||
@@ -1322,7 +1263,8 @@
|
||||
" with tf.io.gfile.GFile(name=gfile_name, mode=\"r\") as gfile:\n",
|
||||
" for line in gfile.readlines():\n",
|
||||
" line = json.loads(line)\n",
|
||||
" print(line)"
|
||||
" print(line)\n",
|
||||
" break"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1381,7 +1323,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"DEPLOYED_NAME = \"census-\" + UUID\n",
|
||||
"DEPLOYED_NAME = \"census-\" + TIMESTAMP\n",
|
||||
"\n",
|
||||
"TRAFFIC_SPLIT = {\"0\": 100}\n",
|
||||
"\n",
|
||||
@@ -1415,6 +1357,15 @@
|
||||
" INFO:google.cloud.aiplatform.models:Endpoint model deployed. Resource name: projects/759209241365/locations/us-central1/endpoints/4867177336350441472"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "endpoints_predict:migration,new,mbsdk"
|
||||
},
|
||||
"source": [
|
||||
"### [predictions.online-prediction-automl](https://cloud.google.com/vertex-ai/docs/predictions/online-predictions-automl)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
@@ -1423,7 +1374,7 @@
|
||||
"source": [
|
||||
"### Make test item\n",
|
||||
"\n",
|
||||
"You use synthetic data as a test data item. Don't be concerned that we are using synthetic data -- we just want to demonstrate how to make a prediction."
|
||||
"You will use synthetic data as a test data item. Don't be concerned that we are using synthetic data -- we just want to demonstrate how to make a prediction."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1540,10 +1491,13 @@
|
||||
"Otherwise, you can delete the individual resources you created in this tutorial:\n",
|
||||
"\n",
|
||||
"- Dataset\n",
|
||||
"- Pipeline\n",
|
||||
"- Model\n",
|
||||
"- Endpoint\n",
|
||||
"- AutoML Training Job\n",
|
||||
"- Batch Job\n",
|
||||
"- Custom Job\n",
|
||||
"- Hyperparameter Tuning Job\n",
|
||||
"- Cloud Storage Bucket"
|
||||
]
|
||||
},
|
||||
@@ -1551,25 +1505,64 @@
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "74ddbf65df4c"
|
||||
"id": "cleanup:mbsdk"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# delete endpoint\n",
|
||||
"endpoint.delete()\n",
|
||||
"delete_all = True\n",
|
||||
"\n",
|
||||
"# Delete the model using the Vertex model object\n",
|
||||
"model.delete()\n",
|
||||
"if delete_all:\n",
|
||||
" # Delete the dataset using the Vertex dataset object\n",
|
||||
" try:\n",
|
||||
" if \"dataset\" in globals():\n",
|
||||
" dataset.delete()\n",
|
||||
" except Exception as e:\n",
|
||||
" print(e)\n",
|
||||
"\n",
|
||||
"# Delete the AutoML or Pipeline training job\n",
|
||||
"job.delete()\n",
|
||||
" # Delete the model using the Vertex model object\n",
|
||||
" try:\n",
|
||||
" if \"model\" in globals():\n",
|
||||
" model.delete()\n",
|
||||
" except Exception as e:\n",
|
||||
" print(e)\n",
|
||||
"\n",
|
||||
"# Delete the batch prediction job using the Vertex batch prediction object\n",
|
||||
"batch_predict_job.delete()\n",
|
||||
" # Delete the endpoint using the Vertex endpoint object\n",
|
||||
" try:\n",
|
||||
" if \"endpoint\" in globals():\n",
|
||||
" endpoint.delete()\n",
|
||||
" except Exception as e:\n",
|
||||
" print(e)\n",
|
||||
"\n",
|
||||
"delete_bucket = False\n",
|
||||
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
|
||||
" ! gsutil -m rm -r $BUCKET_URI"
|
||||
" # Delete the AutoML or Pipeline trainig job\n",
|
||||
" try:\n",
|
||||
" if \"dag\" in globals():\n",
|
||||
" dag.delete()\n",
|
||||
" except Exception as e:\n",
|
||||
" print(e)\n",
|
||||
"\n",
|
||||
" # Delete the custom trainig job\n",
|
||||
" try:\n",
|
||||
" if \"job\" in globals():\n",
|
||||
" job.delete()\n",
|
||||
" except Exception as e:\n",
|
||||
" print(e)\n",
|
||||
"\n",
|
||||
" # Delete the batch prediction job using the Vertex batch prediction object\n",
|
||||
" try:\n",
|
||||
" if \"batch_predict_job\" in globals():\n",
|
||||
" batch_predict_job.delete()\n",
|
||||
" except Exception as e:\n",
|
||||
" print(e)\n",
|
||||
"\n",
|
||||
" # Delete the hyperparameter tuning job using the Vertex hyperparameter tuning object\n",
|
||||
" try:\n",
|
||||
" if \"hpt_job\" in globals():\n",
|
||||
" hpt_job.delete()\n",
|
||||
" except Exception as e:\n",
|
||||
" print(e)\n",
|
||||
"\n",
|
||||
" if \"BUCKET_NAME\" in globals():\n",
|
||||
" ! gsutil rm -r $BUCKET_NAME"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
@@ -43,58 +43,10 @@
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ7%20Vertex%20SDK%20AutoML%20Text%20Entity%20Extraction.ipynb\">\n",
|
||||
" <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
|
||||
" Open in Vertex AI Workbench\n",
|
||||
" </a>\n",
|
||||
" </td> \n",
|
||||
"</table>\n",
|
||||
"<br/><br/><br/>"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "2277f661a148"
|
||||
},
|
||||
"source": [
|
||||
"## Overview\n",
|
||||
"\n",
|
||||
"<a name=\"section-1\"></a>\n",
|
||||
"\n",
|
||||
"This notebook demonstrates how to create an AutoML Text Entity Extrasction Model, with a Vertex AI ncbi disease research dataset, and how to serve the model for batch prediction. It requires you provide a bucket where the dataset will be stored.\n",
|
||||
"\n",
|
||||
"Note: you may incur charges for training, prediction, storage or usage of other GCP products in connection with testing this SDK."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "f926ec7acab3"
|
||||
},
|
||||
"source": [
|
||||
"### Objective\n",
|
||||
"\n",
|
||||
"The objective of this notebook is to build a AutoML Text Entity Extrasction Model. The following steps have been followed:\n",
|
||||
"This tutorial uses the following Google Cloud ML services :\n",
|
||||
"\n",
|
||||
"* Vertex AI Dataset resource\n",
|
||||
"* AutoML Training\n",
|
||||
"* Vertex AI Model resource\n",
|
||||
"* Vertex AI Batch Prediction\n",
|
||||
"\n",
|
||||
"The steps performed include the following:\n",
|
||||
"\n",
|
||||
"* Set your task name, and GCS prefix\n",
|
||||
"* Copy AutoML video demo train data for creating managed dataset\n",
|
||||
"* Create a dataset on Vertex AI.\n",
|
||||
"* Configure a training job\n",
|
||||
"* Launch a training job and create a model on Vertex AI\n",
|
||||
"* Copy AutoML Video Demo Prediction Data for creating batch prediction job\n",
|
||||
"* Perform batch prediction job on the model"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
@@ -167,7 +119,7 @@
|
||||
"source": [
|
||||
"## Installation\n",
|
||||
"\n",
|
||||
"Install the packages required for executing this notebook."
|
||||
"Install the latest version of Vertex SDK for Python."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -186,9 +138,7 @@
|
||||
"else:\n",
|
||||
" USER_FLAG = \"\"\n",
|
||||
"\n",
|
||||
"! pip3 install --upgrade google-cloud-aiplatform \\\n",
|
||||
" cuda-python \\\n",
|
||||
" $USER_FLAG -q"
|
||||
"! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -328,7 +278,7 @@
|
||||
"#### Region\n",
|
||||
"\n",
|
||||
"You can also change the `REGION` variable, which is used for operations\n",
|
||||
"throughout the rest of this notebook. Below are regions supported for Vertex AI. It is recommended that you choose the region closest to you.\n",
|
||||
"throughout the rest of this notebook. Below are regions supported for Vertex AI. We recommend that you choose the region closest to you.\n",
|
||||
"\n",
|
||||
"- Americas: `us-central1`\n",
|
||||
"- Europe: `europe-west4`\n",
|
||||
@@ -336,7 +286,7 @@
|
||||
"\n",
|
||||
"You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
|
||||
"\n",
|
||||
"Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
|
||||
"Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -347,10 +297,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"REGION = \"[your-region]\" # @param {type: \"string\"}\n",
|
||||
"\n",
|
||||
"if REGION == \"[your-region]\":\n",
|
||||
" REGION = \"us-central1\""
|
||||
"REGION = \"us-central1\" # @param {type: \"string\"}"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -359,9 +306,9 @@
|
||||
"id": "timestamp"
|
||||
},
|
||||
"source": [
|
||||
"#### UUID\n",
|
||||
"#### Timestamp\n",
|
||||
"\n",
|
||||
"If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
|
||||
"If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -372,16 +319,9 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import random\n",
|
||||
"import string\n",
|
||||
"from datetime import datetime\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Generate a uuid of a specifed length(default=8)\n",
|
||||
"def generate_uuid(length: int = 8) -> str:\n",
|
||||
" return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"UUID = generate_uuid()"
|
||||
"TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -392,38 +332,23 @@
|
||||
"source": [
|
||||
"### Authenticate your Google Cloud account\n",
|
||||
"\n",
|
||||
"**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
|
||||
"authenticated. Skip this step."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "f1d7a972141f"
|
||||
},
|
||||
"source": [
|
||||
"**If you are using Colab**, run the cell below and follow the instructions\n",
|
||||
"when prompted to authenticate your account via oAuth.\n",
|
||||
"**If you are using Google Cloud Notebooks**, your environment is already authenticated. Skip this step.\n",
|
||||
"\n",
|
||||
"**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
|
||||
"\n",
|
||||
"**Otherwise**, follow these steps:\n",
|
||||
"\n",
|
||||
"1. In the Cloud Console, go to the [**Create service account key**\n",
|
||||
" page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
|
||||
"In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
|
||||
"\n",
|
||||
"2. Click **Create service account**.\n",
|
||||
"**Click Create service account**.\n",
|
||||
"\n",
|
||||
"3. In the **Service account name** field, enter a name, and\n",
|
||||
" click **Create**.\n",
|
||||
"In the **Service account name** field, enter a name, and click **Create**.\n",
|
||||
"\n",
|
||||
"4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
|
||||
"into the filter box, and select\n",
|
||||
" **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
|
||||
"In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex\" into the filter box, and select **Vertex Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
|
||||
"\n",
|
||||
"5. Click *Create*. A JSON file that contains your key downloads to your\n",
|
||||
"local environment.\n",
|
||||
"Click Create. A JSON file that contains your key downloads to your local environment.\n",
|
||||
"\n",
|
||||
"6. Enter the path to your service account key as the\n",
|
||||
"`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
|
||||
"Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -442,11 +367,8 @@
|
||||
"import os\n",
|
||||
"import sys\n",
|
||||
"\n",
|
||||
"# If on Vertex AI Workbench, then don't execute this code\n",
|
||||
"IS_COLAB = \"google.colab\" in sys.modules\n",
|
||||
"if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
|
||||
" \"DL_ANACONDA_HOME\"\n",
|
||||
"):\n",
|
||||
"# If on Google Cloud Notebook, then don't execute this code\n",
|
||||
"if not os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
|
||||
" if \"google.colab\" in sys.modules:\n",
|
||||
" from google.colab import auth as google_auth\n",
|
||||
"\n",
|
||||
@@ -469,16 +391,9 @@
|
||||
"\n",
|
||||
"**The following steps are required, regardless of your notebook environment.**\n",
|
||||
"\n",
|
||||
"When you initialize the Vertex SDK for Python, you specify a Cloud Storage staging bucket. The staging bucket is where all the data associated with your dataset and model resources are retained across sessions.\n",
|
||||
"\n",
|
||||
"When you submit a training job using the Cloud SDK, you upload a Python package\n",
|
||||
"containing your training code to a Cloud Storage bucket. Vertex AI runs\n",
|
||||
"the code from this package. In this tutorial, Vertex AI also saves the\n",
|
||||
"trained model that results from your job in the same bucket. Using this model artifact, you can then\n",
|
||||
"create Vertex AI model and endpoint resources in order to serve\n",
|
||||
"online predictions.\n",
|
||||
"\n",
|
||||
"Set the name of your Cloud Storage bucket below. It must be unique across all\n",
|
||||
"Cloud Storage buckets."
|
||||
"Set the name of your Cloud Storage bucket below. Bucket names must be globally unique across all Google Cloud projects, including those outside of your organization."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -489,8 +404,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"BUCKET_NAME = \"[your-bucket-name]\" # @param {type:\"string\"}\n",
|
||||
"BUCKET_URI = f\"gs://{BUCKET_NAME}\""
|
||||
"BUCKET_NAME = \"gs://[your-bucket-name]\" # @param {type:\"string\"}"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -501,9 +415,8 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
|
||||
" BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
|
||||
" BUCKET_URI = f\"gs://{BUCKET_NAME}\""
|
||||
"if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
|
||||
" BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -523,7 +436,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI"
|
||||
"! gsutil mb -l $REGION $BUCKET_NAME"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -543,7 +456,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil ls -al $BUCKET_URI"
|
||||
"! gsutil ls -al $BUCKET_NAME"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -575,7 +488,7 @@
|
||||
"id": "init_aip:mbsdk"
|
||||
},
|
||||
"source": [
|
||||
"## Initialize Vertex AI SDK for Python\n",
|
||||
"## Initialize Vertex SDK for Python\n",
|
||||
"\n",
|
||||
"Initialize the Vertex SDK for Python for your project and corresponding bucket."
|
||||
]
|
||||
@@ -690,7 +603,7 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"dataset = aip.TextDataset.create(\n",
|
||||
" display_name=\"NCBI Biomedical\" + \"_\" + UUID,\n",
|
||||
" display_name=\"NCBI Biomedical\" + \"_\" + TIMESTAMP,\n",
|
||||
" gcs_source=[IMPORT_FILE],\n",
|
||||
" import_schema_uri=aip.schema.dataset.ioformat.text.extraction,\n",
|
||||
")\n",
|
||||
@@ -769,7 +682,7 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"dag = aip.AutoMLTextTrainingJob(\n",
|
||||
" display_name=\"biomedical_\" + UUID, prediction_type=\"extraction\"\n",
|
||||
" display_name=\"biomedical_\" + TIMESTAMP, prediction_type=\"extraction\"\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"print(dag)"
|
||||
@@ -817,7 +730,7 @@
|
||||
"source": [
|
||||
"model = dag.run(\n",
|
||||
" dataset=dataset,\n",
|
||||
" model_display_name=\"biomedical_\" + UUID,\n",
|
||||
" model_display_name=\"biomedical_\" + TIMESTAMP,\n",
|
||||
" training_fraction_split=0.8,\n",
|
||||
" validation_fraction_split=0.1,\n",
|
||||
" test_fraction_split=0.1,\n",
|
||||
@@ -888,7 +801,7 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Get model resource ID\n",
|
||||
"models = aip.Model.list(filter=\"display_name=biomedical_\" + UUID)\n",
|
||||
"models = aip.Model.list(filter=\"display_name=biomedical_\" + TIMESTAMP)\n",
|
||||
"\n",
|
||||
"# Get a reference to the Model Service client\n",
|
||||
"client_options = {\"api_endpoint\": f\"{REGION}-aiplatform.googleapis.com\"}\n",
|
||||
@@ -1012,14 +925,14 @@
|
||||
"\n",
|
||||
"import tensorflow as tf\n",
|
||||
"\n",
|
||||
"gcs_test_item_1 = BUCKET_URI + \"/test1.txt\"\n",
|
||||
"gcs_test_item_1 = BUCKET_NAME + \"/test1.txt\"\n",
|
||||
"with tf.io.gfile.GFile(gcs_test_item_1, \"w\") as f:\n",
|
||||
" f.write(test_item_1 + \"\\n\")\n",
|
||||
"gcs_test_item_2 = BUCKET_URI + \"/test2.txt\"\n",
|
||||
"gcs_test_item_2 = BUCKET_NAME + \"/test2.txt\"\n",
|
||||
"with tf.io.gfile.GFile(gcs_test_item_2, \"w\") as f:\n",
|
||||
" f.write(test_item_2 + \"\\n\")\n",
|
||||
"\n",
|
||||
"gcs_input_uri = BUCKET_URI + \"/test.jsonl\"\n",
|
||||
"gcs_input_uri = BUCKET_NAME + \"/test.jsonl\"\n",
|
||||
"with tf.io.gfile.GFile(gcs_input_uri, \"w\") as f:\n",
|
||||
" data = {\"content\": gcs_test_item_1, \"mime_type\": \"text/plain\"}\n",
|
||||
" f.write(json.dumps(data) + \"\\n\")\n",
|
||||
@@ -1055,9 +968,9 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"batch_predict_job = model.batch_predict(\n",
|
||||
" job_display_name=\"biomedical_\" + UUID,\n",
|
||||
" job_display_name=\"biomedical_\" + TIMESTAMP,\n",
|
||||
" gcs_source=gcs_input_uri,\n",
|
||||
" gcs_destination_prefix=BUCKET_URI,\n",
|
||||
" gcs_destination_prefix=BUCKET_NAME,\n",
|
||||
" sync=False,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
@@ -1398,31 +1311,60 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Delete the dataset using the Vertex dataset object\n",
|
||||
"delete_all = True\n",
|
||||
"\n",
|
||||
"dataset.delete()\n",
|
||||
"if delete_all:\n",
|
||||
" # Delete the dataset using the Vertex dataset object\n",
|
||||
" try:\n",
|
||||
" if \"dataset\" in globals():\n",
|
||||
" dataset.delete()\n",
|
||||
" except Exception as e:\n",
|
||||
" print(e)\n",
|
||||
"\n",
|
||||
"# Delete the model using the Vertex model object\n",
|
||||
" # Delete the model using the Vertex model object\n",
|
||||
" try:\n",
|
||||
" if \"model\" in globals():\n",
|
||||
" model.delete()\n",
|
||||
" except Exception as e:\n",
|
||||
" print(e)\n",
|
||||
"\n",
|
||||
"model.delete()\n",
|
||||
" # Delete the endpoint using the Vertex endpoint object\n",
|
||||
" try:\n",
|
||||
" if \"endpoint\" in globals():\n",
|
||||
" endpoint.delete()\n",
|
||||
" except Exception as e:\n",
|
||||
" print(e)\n",
|
||||
"\n",
|
||||
"# Delete the endpoint using the Vertex endpoint object\n",
|
||||
" # Delete the AutoML or Pipeline trainig job\n",
|
||||
" try:\n",
|
||||
" if \"dag\" in globals():\n",
|
||||
" dag.delete()\n",
|
||||
" except Exception as e:\n",
|
||||
" print(e)\n",
|
||||
"\n",
|
||||
"endpoint.delete()\n",
|
||||
"# Delete the AutoML or Pipeline trainig job\n",
|
||||
" # Delete the custom trainig job\n",
|
||||
" try:\n",
|
||||
" if \"job\" in globals():\n",
|
||||
" job.delete()\n",
|
||||
" except Exception as e:\n",
|
||||
" print(e)\n",
|
||||
"\n",
|
||||
"dag.delete()\n",
|
||||
" # Delete the batch prediction job using the Vertex batch prediction object\n",
|
||||
" try:\n",
|
||||
" if \"batch_predict_job\" in globals():\n",
|
||||
" batch_predict_job.delete()\n",
|
||||
" except Exception as e:\n",
|
||||
" print(e)\n",
|
||||
"\n",
|
||||
"# Delete the batch prediction job using the Vertex batch prediction object\n",
|
||||
" # Delete the hyperparameter tuning job using the Vertex hyperparameter tuning object\n",
|
||||
" try:\n",
|
||||
" if \"hpt_job\" in globals():\n",
|
||||
" hpt_job.delete()\n",
|
||||
" except Exception as e:\n",
|
||||
" print(e)\n",
|
||||
"\n",
|
||||
"batch_predict_job.delete()\n",
|
||||
"\n",
|
||||
"# Delete the hyperparameter tuning job using the Vertex hyperparameter tuning object\n",
|
||||
"\n",
|
||||
"# Delete GCS bucket.\n",
|
||||
"delete_bucket = False\n",
|
||||
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
|
||||
" ! gsutil -m rm -r $BUCKET_URI"
|
||||
" if \"BUCKET_NAME\" in globals():\n",
|
||||
" ! gsutil rm -r $BUCKET_NAME"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "bdccc50b",
|
||||
"metadata": {
|
||||
"id": "copyright"
|
||||
},
|
||||
@@ -26,7 +25,6 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "c6c22009",
|
||||
"metadata": {
|
||||
"id": "title:migration,new"
|
||||
},
|
||||
@@ -58,19 +56,17 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "b3558cd7",
|
||||
"metadata": {
|
||||
"id": "dataset:claritin,tst"
|
||||
},
|
||||
"source": [
|
||||
"### Dataset\n",
|
||||
"\n",
|
||||
"The dataset used for this tutorial is the [Crowdflower Claritin-Twitter dataset](https://data.world/crowdflower/claritin-twitter) from [data.world Datasets](https://data.world). The version of the dataset you use in this tutorial is stored in a public Cloud Storage bucket."
|
||||
"The dataset used for this tutorial is the [Crowdflower Claritin-Twitter dataset](https://data.world/crowdflower/claritin-twitter) from [data.world Datasets](https://data.world). The version of the dataset you will use in this tutorial is stored in a public Cloud Storage bucket."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "9b9362da",
|
||||
"metadata": {
|
||||
"id": "costs"
|
||||
},
|
||||
@@ -91,7 +87,6 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "05425dbe",
|
||||
"metadata": {
|
||||
"id": "setup_local"
|
||||
},
|
||||
@@ -125,7 +120,6 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "070c64e0",
|
||||
"metadata": {
|
||||
"id": "install_aip:mbsdk"
|
||||
},
|
||||
@@ -138,7 +132,6 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "f6b14b99",
|
||||
"metadata": {
|
||||
"id": "install_aip:mbsdk"
|
||||
},
|
||||
@@ -152,12 +145,42 @@
|
||||
"else:\n",
|
||||
" USER_FLAG = \"\"\n",
|
||||
"\n",
|
||||
"! pip3 install --upgrade google-cloud-aiplatform google-cloud-storage tensorflow $USER_FLAG -q"
|
||||
"! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "install_storage"
|
||||
},
|
||||
"source": [
|
||||
"Install the latest GA version of *google-cloud-storage* library as well."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "install_storage"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! pip3 install -U google-cloud-storage $USER_FLAG"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "install_tensorflow"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! pip3 install --upgrade tensorflow $USER_FLAG"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "81f60b84",
|
||||
"metadata": {
|
||||
"id": "restart"
|
||||
},
|
||||
@@ -170,7 +193,6 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "a95627f0",
|
||||
"metadata": {
|
||||
"id": "restart"
|
||||
},
|
||||
@@ -188,7 +210,6 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "b7f6b038",
|
||||
"metadata": {
|
||||
"id": "before_you_begin:nogpu"
|
||||
},
|
||||
@@ -209,7 +230,7 @@
|
||||
"\n",
|
||||
"3. [Enable the following APIs: Vertex AI APIs, Compute Engine APIs, and Cloud Storage.](https://console.cloud.google.com/flows/enableapi?apiid=ml.googleapis.com,compute_component,storage-component.googleapis.com)\n",
|
||||
"\n",
|
||||
"4. If you are running this notebook locally, you need to install the [Cloud SDK]((https://cloud.google.com/sdk)).\n",
|
||||
"4. If you are running this notebook locally, you will need to install the [Cloud SDK]((https://cloud.google.com/sdk)).\n",
|
||||
"\n",
|
||||
"5. Enter your project ID in the cell below. Then run the cell to make sure the\n",
|
||||
"Cloud SDK uses the right project for all the commands in this notebook.\n",
|
||||
@@ -220,7 +241,6 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "f55cca7c",
|
||||
"metadata": {
|
||||
"id": "set_project_id"
|
||||
},
|
||||
@@ -232,7 +252,6 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "6917314c",
|
||||
"metadata": {
|
||||
"id": "autoset_project_id"
|
||||
},
|
||||
@@ -248,7 +267,6 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "53236aa8",
|
||||
"metadata": {
|
||||
"id": "set_gcloud_project_id"
|
||||
},
|
||||
@@ -259,7 +277,6 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "c009cc18",
|
||||
"metadata": {
|
||||
"id": "region"
|
||||
},
|
||||
@@ -281,61 +298,47 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "071a11c0",
|
||||
"metadata": {
|
||||
"id": "region"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"REGION = \"us-central1\" # @param {type: \"string\"}\n",
|
||||
"\n",
|
||||
"if REGION == \"[your-region]\":\n",
|
||||
" REGION = \"us-central1\""
|
||||
"REGION = \"us-central1\" # @param {type: \"string\"}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "ae48374d",
|
||||
"metadata": {
|
||||
"id": "timestamp"
|
||||
},
|
||||
"source": [
|
||||
"#### UUID\n",
|
||||
"#### Timestamp\n",
|
||||
"\n",
|
||||
"If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
|
||||
"If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "41ba0990",
|
||||
"metadata": {
|
||||
"id": "timestamp"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import random\n",
|
||||
"import string\n",
|
||||
"from datetime import datetime\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Generate a uuid of a specifed length(default=8)\n",
|
||||
"def generate_uuid(length: int = 8) -> str:\n",
|
||||
" return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"UUID = generate_uuid()"
|
||||
"TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "2128e871",
|
||||
"metadata": {
|
||||
"id": "gcp_authenticate"
|
||||
},
|
||||
"source": [
|
||||
"### Authenticate your Google Cloud account\n",
|
||||
"\n",
|
||||
"**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated.\n",
|
||||
"**If you are using Google Cloud Notebooks**, your environment is already authenticated. Skip this step.\n",
|
||||
"\n",
|
||||
"**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
|
||||
"\n",
|
||||
@@ -357,7 +360,6 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "433e860c",
|
||||
"metadata": {
|
||||
"id": "gcp_authenticate"
|
||||
},
|
||||
@@ -371,11 +373,8 @@
|
||||
"import os\n",
|
||||
"import sys\n",
|
||||
"\n",
|
||||
"# If on Vertex AI Workbench, then don't execute this code\n",
|
||||
"IS_COLAB = \"google.colab\" in sys.modules\n",
|
||||
"if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
|
||||
" \"DL_ANACONDA_HOME\"\n",
|
||||
"):\n",
|
||||
"# If on Google Cloud Notebook, then don't execute this code\n",
|
||||
"if not os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
|
||||
" if \"google.colab\" in sys.modules:\n",
|
||||
" from google.colab import auth as google_auth\n",
|
||||
"\n",
|
||||
@@ -390,7 +389,6 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "b57cb5f6",
|
||||
"metadata": {
|
||||
"id": "bucket:mbsdk"
|
||||
},
|
||||
@@ -407,33 +405,28 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "61b082b1",
|
||||
"metadata": {
|
||||
"id": "bucket"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"BUCKET_NAME = \"[your-bucket-name]\" # @param {type:\"string\"}\n",
|
||||
"BUCKET_URI = f\"gs://{BUCKET_NAME}\""
|
||||
"BUCKET_NAME = \"gs://[your-bucket-name]\" # @param {type:\"string\"}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "ff81b3cc",
|
||||
"metadata": {
|
||||
"id": "autoset_bucket"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
|
||||
" BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
|
||||
" BUCKET_URI = \"gs://\" + BUCKET_NAME"
|
||||
"if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
|
||||
" BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "f8c009cd",
|
||||
"metadata": {
|
||||
"id": "create_bucket"
|
||||
},
|
||||
@@ -444,18 +437,16 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "2f881cb5",
|
||||
"metadata": {
|
||||
"id": "create_bucket"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil mb -l $REGION $BUCKET_URI"
|
||||
"! gsutil mb -l $REGION $BUCKET_NAME"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "d746d0f0",
|
||||
"metadata": {
|
||||
"id": "validate_bucket"
|
||||
},
|
||||
@@ -466,18 +457,16 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "8c435668",
|
||||
"metadata": {
|
||||
"id": "validate_bucket"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil ls -al $BUCKET_URI"
|
||||
"! gsutil ls -al $BUCKET_NAME"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "f578b01b",
|
||||
"metadata": {
|
||||
"id": "setup_vars"
|
||||
},
|
||||
@@ -491,7 +480,6 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "f41ecf1e",
|
||||
"metadata": {
|
||||
"id": "import_aip:mbsdk"
|
||||
},
|
||||
@@ -502,31 +490,28 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "292245fd",
|
||||
"metadata": {
|
||||
"id": "init_aip:mbsdk"
|
||||
},
|
||||
"source": [
|
||||
"## Initialize Vertex AI SDK for Python\n",
|
||||
"## Initialize Vertex SDK for Python\n",
|
||||
"\n",
|
||||
"Initialize the Vertex AI SDK for Python for your project and corresponding bucket."
|
||||
"Initialize the Vertex SDK for Python for your project and corresponding bucket."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "56dc88d8",
|
||||
"metadata": {
|
||||
"id": "init_aip:mbsdk"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"aip.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
|
||||
"aip.init(project=PROJECT_ID, staging_bucket=BUCKET_NAME)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "87e20f86",
|
||||
"metadata": {
|
||||
"id": "import_file:u_dataset,csv"
|
||||
},
|
||||
@@ -539,7 +524,6 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "fffdec5d",
|
||||
"metadata": {
|
||||
"id": "import_file:claritin,csv,tst"
|
||||
},
|
||||
@@ -551,7 +535,6 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "9c8d950b",
|
||||
"metadata": {
|
||||
"id": "quick_peek:csv"
|
||||
},
|
||||
@@ -566,7 +549,6 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "da6d0980",
|
||||
"metadata": {
|
||||
"id": "quick_peek:csv"
|
||||
},
|
||||
@@ -586,7 +568,6 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "00e9f81e",
|
||||
"metadata": {
|
||||
"id": "create_a_dataset:migration"
|
||||
},
|
||||
@@ -596,7 +577,6 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "14b72768",
|
||||
"metadata": {
|
||||
"id": "datasets_create:migration,new,mbsdk"
|
||||
},
|
||||
@@ -606,7 +586,6 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "00d777bd",
|
||||
"metadata": {
|
||||
"id": "create_dataset:text,tst"
|
||||
},
|
||||
@@ -625,14 +604,13 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "ca8a6f66",
|
||||
"metadata": {
|
||||
"id": "create_dataset:text,tst"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"dataset = aip.TextDataset.create(\n",
|
||||
" display_name=\"Crowdflower Claritin-Twitter\" + \"_\" + UUID,\n",
|
||||
" display_name=\"Crowdflower Claritin-Twitter\" + \"_\" + TIMESTAMP,\n",
|
||||
" gcs_source=[IMPORT_FILE],\n",
|
||||
" import_schema_uri=aip.schema.dataset.ioformat.text.sentiment,\n",
|
||||
")\n",
|
||||
@@ -642,7 +620,6 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "068df169",
|
||||
"metadata": {
|
||||
"id": "create_dataset:text,tst"
|
||||
},
|
||||
@@ -662,7 +639,6 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "fb50a4ce",
|
||||
"metadata": {
|
||||
"id": "train_a_model:migration"
|
||||
},
|
||||
@@ -672,7 +648,6 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "293160ba",
|
||||
"metadata": {
|
||||
"id": "trainingpipelines_create:migration,new,mbsdk"
|
||||
},
|
||||
@@ -682,7 +657,6 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "84801634",
|
||||
"metadata": {
|
||||
"id": "create_automl_pipeline:text,tst"
|
||||
},
|
||||
@@ -709,14 +683,13 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "69eaae0e",
|
||||
"metadata": {
|
||||
"id": "create_automl_pipeline:text,tst"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"dag = aip.AutoMLTextTrainingJob(\n",
|
||||
" display_name=\"claritin_\" + UUID,\n",
|
||||
" display_name=\"claritin_\" + TIMESTAMP,\n",
|
||||
" prediction_type=\"sentiment\",\n",
|
||||
" sentiment_max=SENTIMENT_MAX,\n",
|
||||
")\n",
|
||||
@@ -726,7 +699,6 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "da9ecb4e",
|
||||
"metadata": {
|
||||
"id": "create_automl_pipeline:text,tst"
|
||||
},
|
||||
@@ -738,7 +710,6 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "55f19997",
|
||||
"metadata": {
|
||||
"id": "run_automl_pipeline:text"
|
||||
},
|
||||
@@ -755,13 +726,12 @@
|
||||
"\n",
|
||||
"The `run` method when completed returns the `Model` resource.\n",
|
||||
"\n",
|
||||
"The execution of the training pipeline take upto 20 minutes."
|
||||
"The execution of the training pipeline will take upto 20 minutes."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "6149074c",
|
||||
"metadata": {
|
||||
"id": "run_automl_pipeline:text"
|
||||
},
|
||||
@@ -769,7 +739,7 @@
|
||||
"source": [
|
||||
"model = dag.run(\n",
|
||||
" dataset=dataset,\n",
|
||||
" model_display_name=\"claritin_\" + UUID,\n",
|
||||
" model_display_name=\"claritin_\" + TIMESTAMP,\n",
|
||||
" training_fraction_split=0.8,\n",
|
||||
" validation_fraction_split=0.1,\n",
|
||||
" test_fraction_split=0.1,\n",
|
||||
@@ -778,7 +748,6 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "6e8fe148",
|
||||
"metadata": {
|
||||
"id": "run_automl_pipeline:text"
|
||||
},
|
||||
@@ -804,7 +773,6 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "c25dee28",
|
||||
"metadata": {
|
||||
"id": "evaluate_the_model:migration"
|
||||
},
|
||||
@@ -814,7 +782,6 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "903e8226",
|
||||
"metadata": {
|
||||
"id": "models_evaluations_list:migration,new"
|
||||
},
|
||||
@@ -824,7 +791,6 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "cb2d95f3",
|
||||
"metadata": {
|
||||
"id": "evaluate_the_model:mbsdk"
|
||||
},
|
||||
@@ -838,14 +804,13 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "9b1ec312",
|
||||
"metadata": {
|
||||
"id": "evaluate_the_model:mbsdk"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Get model resource ID\n",
|
||||
"models = aip.Model.list(filter=\"display_name=claritin_\" + UUID)\n",
|
||||
"models = aip.Model.list(filter=\"display_name=claritin_\" + TIMESTAMP)\n",
|
||||
"\n",
|
||||
"# Get a reference to the Model Service client\n",
|
||||
"client_options = {\"api_endpoint\": f\"{REGION}-aiplatform.googleapis.com\"}\n",
|
||||
@@ -860,7 +825,6 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "9eab460e",
|
||||
"metadata": {
|
||||
"id": "evaluate_the_model:mbsdk"
|
||||
},
|
||||
@@ -901,7 +865,6 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "d4111c50",
|
||||
"metadata": {
|
||||
"id": "make_batch_predictions:migration"
|
||||
},
|
||||
@@ -911,7 +874,6 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "f73fad68",
|
||||
"metadata": {
|
||||
"id": "batchpredictionjobs_create:migration,new,mbsdk"
|
||||
},
|
||||
@@ -921,20 +883,18 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "ba77f1c7",
|
||||
"metadata": {
|
||||
"id": "get_test_items:batch_prediction"
|
||||
},
|
||||
"source": [
|
||||
"### Get test item(s)\n",
|
||||
"\n",
|
||||
"Now do a batch prediction to your Vertex model. You use arbitrary examples out of the dataset as a test items. Don't be concerned that the examples were likely used in training the model -- we just want to demonstrate how to make a prediction."
|
||||
"Now do a batch prediction to your Vertex model. You will use arbitrary examples out of the dataset as a test items. Don't be concerned that the examples were likely used in training the model -- we just want to demonstrate how to make a prediction."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "1c9fc91f",
|
||||
"metadata": {
|
||||
"id": "get_test_items:automl,tst,csv"
|
||||
},
|
||||
@@ -956,14 +916,13 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "2a18c8e2",
|
||||
"metadata": {
|
||||
"id": "make_batch_file:automl,text"
|
||||
},
|
||||
"source": [
|
||||
"### Make the batch input file\n",
|
||||
"\n",
|
||||
"Now make a batch input file, which you store in your local Cloud Storage bucket. The batch input file can only be in JSONL format. For JSONL file, you make one dictionary entry per line for each data item (instance). The dictionary contains the key/value pairs:\n",
|
||||
"Now make a batch input file, which you will store in your local Cloud Storage bucket. The batch input file can only be in JSONL format. For JSONL file, you make one dictionary entry per line for each data item (instance). The dictionary contains the key/value pairs:\n",
|
||||
"\n",
|
||||
"- `content`: The Cloud Storage path to the file with the text item.\n",
|
||||
"- `mime_type`: The content type. In our example, it is a `text` file.\n",
|
||||
@@ -976,7 +935,6 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "70461c41",
|
||||
"metadata": {
|
||||
"id": "make_batch_file:automl,text"
|
||||
},
|
||||
@@ -986,14 +944,14 @@
|
||||
"\n",
|
||||
"import tensorflow as tf\n",
|
||||
"\n",
|
||||
"gcs_test_item_1 = BUCKET_URI + \"/test1.txt\"\n",
|
||||
"gcs_test_item_1 = BUCKET_NAME + \"/test1.txt\"\n",
|
||||
"with tf.io.gfile.GFile(gcs_test_item_1, \"w\") as f:\n",
|
||||
" f.write(test_item_1 + \"\\n\")\n",
|
||||
"gcs_test_item_2 = BUCKET_URI + \"/test2.txt\"\n",
|
||||
"gcs_test_item_2 = BUCKET_NAME + \"/test2.txt\"\n",
|
||||
"with tf.io.gfile.GFile(gcs_test_item_2, \"w\") as f:\n",
|
||||
" f.write(test_item_2 + \"\\n\")\n",
|
||||
"\n",
|
||||
"gcs_input_uri = BUCKET_URI + \"/test.jsonl\"\n",
|
||||
"gcs_input_uri = BUCKET_NAME + \"/test.jsonl\"\n",
|
||||
"with tf.io.gfile.GFile(gcs_input_uri, \"w\") as f:\n",
|
||||
" data = {\"content\": gcs_test_item_1, \"mime_type\": \"text/plain\"}\n",
|
||||
" f.write(json.dumps(data) + \"\\n\")\n",
|
||||
@@ -1006,7 +964,6 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "254cbdbb",
|
||||
"metadata": {
|
||||
"id": "batch_request:mbsdk"
|
||||
},
|
||||
@@ -1018,22 +975,21 @@
|
||||
"- `job_display_name`: The human readable name for the batch prediction job.\n",
|
||||
"- `gcs_source`: A list of one or more batch request input files.\n",
|
||||
"- `gcs_destination_prefix`: The Cloud Storage location for storing the batch prediction resuls.\n",
|
||||
"- `sync`: If set to True, the call block while waiting for the asynchronous batch job to complete."
|
||||
"- `sync`: If set to True, the call will block while waiting for the asynchronous batch job to complete."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "8f3cf8b6",
|
||||
"metadata": {
|
||||
"id": "batch_request:mbsdk"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"batch_predict_job = model.batch_predict(\n",
|
||||
" job_display_name=\"claritin_\" + UUID,\n",
|
||||
" job_display_name=\"claritin_\" + TIMESTAMP,\n",
|
||||
" gcs_source=gcs_input_uri,\n",
|
||||
" gcs_destination_prefix=BUCKET_URI,\n",
|
||||
" gcs_destination_prefix=BUCKET_NAME,\n",
|
||||
" sync=False,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
@@ -1042,7 +998,6 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "530dbf5b",
|
||||
"metadata": {
|
||||
"id": "batch_request:mbsdk"
|
||||
},
|
||||
@@ -1062,7 +1017,6 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "89414481",
|
||||
"metadata": {
|
||||
"id": "batch_request_wait:mbsdk"
|
||||
},
|
||||
@@ -1075,7 +1029,6 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "a579bd4a",
|
||||
"metadata": {
|
||||
"id": "batch_request_wait:mbsdk"
|
||||
},
|
||||
@@ -1086,7 +1039,6 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "2cba4cc6",
|
||||
"metadata": {
|
||||
"id": "batch_request_wait:mbsdk"
|
||||
},
|
||||
@@ -1121,7 +1073,6 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "c46e3e76",
|
||||
"metadata": {
|
||||
"id": "get_batch_prediction:mbsdk,tst"
|
||||
},
|
||||
@@ -1140,7 +1091,6 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "d2af5ea8",
|
||||
"metadata": {
|
||||
"id": "get_batch_prediction:mbsdk,tst"
|
||||
},
|
||||
@@ -1169,7 +1119,6 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "9fc83253",
|
||||
"metadata": {
|
||||
"id": "get_batch_prediction:mbsdk,tst"
|
||||
},
|
||||
@@ -1181,7 +1130,6 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "19466786",
|
||||
"metadata": {
|
||||
"id": "make_online_predictions:migration"
|
||||
},
|
||||
@@ -1191,7 +1139,6 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "e97f1e55",
|
||||
"metadata": {
|
||||
"id": "deploy_model:migration,new,mbsdk"
|
||||
},
|
||||
@@ -1201,7 +1148,6 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "d2745f77",
|
||||
"metadata": {
|
||||
"id": "deploy_model:mbsdk,automatic"
|
||||
},
|
||||
@@ -1214,7 +1160,6 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "6d30aa15",
|
||||
"metadata": {
|
||||
"id": "deploy_model:mbsdk,automatic"
|
||||
},
|
||||
@@ -1225,7 +1170,6 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "c2c876d0",
|
||||
"metadata": {
|
||||
"id": "deploy_model:mbsdk,automatic"
|
||||
},
|
||||
@@ -1244,7 +1188,6 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "9bb982a8",
|
||||
"metadata": {
|
||||
"id": "endpoints_predict:migration,new,mbsdk"
|
||||
},
|
||||
@@ -1254,20 +1197,18 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "246945bb",
|
||||
"metadata": {
|
||||
"id": "get_test_item"
|
||||
},
|
||||
"source": [
|
||||
"### Get test item\n",
|
||||
"\n",
|
||||
"You use an arbitrary example out of the dataset as a test item. Don't be concerned that the example was likely used in training the model -- we just want to demonstrate how to make a prediction."
|
||||
"You will use an arbitrary example out of the dataset as a test item. Don't be concerned that the example was likely used in training the model -- we just want to demonstrate how to make a prediction."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "e21c3f76",
|
||||
"metadata": {
|
||||
"id": "get_test_item:automl,tst,csv"
|
||||
},
|
||||
@@ -1284,7 +1225,6 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "95ffe1ea",
|
||||
"metadata": {
|
||||
"id": "predict_request:mbsdk,tst"
|
||||
},
|
||||
@@ -1313,7 +1253,6 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "16b7ab95",
|
||||
"metadata": {
|
||||
"id": "predict_request:mbsdk,tst"
|
||||
},
|
||||
@@ -1327,7 +1266,6 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "f4c79c7f",
|
||||
"metadata": {
|
||||
"id": "predict_request:mbsdk,tst"
|
||||
},
|
||||
@@ -1339,7 +1277,6 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "52717fb9",
|
||||
"metadata": {
|
||||
"id": "undeploy_model:mbsdk"
|
||||
},
|
||||
@@ -1352,7 +1289,6 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "7c164d13",
|
||||
"metadata": {
|
||||
"id": "undeploy_model:mbsdk"
|
||||
},
|
||||
@@ -1363,7 +1299,6 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "4b844c87",
|
||||
"metadata": {
|
||||
"id": "cleanup:mbsdk"
|
||||
},
|
||||
@@ -1389,7 +1324,6 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "2ea906d0",
|
||||
"metadata": {
|
||||
"id": "cleanup:mbsdk"
|
||||
},
|
||||
@@ -1448,7 +1382,7 @@
|
||||
" print(e)\n",
|
||||
"\n",
|
||||
" if \"BUCKET_NAME\" in globals():\n",
|
||||
" ! gsutil rm -r $BUCKET_URI"
|
||||
" ! gsutil rm -r $BUCKET_NAME"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
+1431
-1398
File diff suppressed because it is too large
Load Diff
+6
-5
@@ -1669,9 +1669,9 @@
|
||||
"\n",
|
||||
" from google_cloud_pipeline_components.aiplatform import ModelBatchPredictOp\n",
|
||||
" from google_cloud_pipeline_components.experimental.evaluation import (\n",
|
||||
" EvaluationDataSamplerOp, GetVertexModelOp,\n",
|
||||
" EvaluationDataSamplerOp, EvaluationDataSplitterOp, GetVertexModelOp,\n",
|
||||
" ModelEvaluationFeatureAttributionOp, ModelEvaluationRegressionOp,\n",
|
||||
" ModelImportEvaluationOp, TargetFieldDataRemoverOp)\n",
|
||||
" ModelImportEvaluationOp)\n",
|
||||
"\n",
|
||||
" # Get the Vertex AI model resource\n",
|
||||
" get_model_task = GetVertexModelOp(model_resource_name=model_name)\n",
|
||||
@@ -1687,13 +1687,13 @@
|
||||
" )\n",
|
||||
"\n",
|
||||
" # Run Data-splitter task\n",
|
||||
" data_splitter_task = TargetFieldDataRemoverOp(\n",
|
||||
" data_splitter_task = EvaluationDataSplitterOp(\n",
|
||||
" project=project,\n",
|
||||
" location=location,\n",
|
||||
" root_dir=root_dir,\n",
|
||||
" gcs_source_uris=data_sampler_task.outputs[\"gcs_output_directory\"],\n",
|
||||
" instances_format=batch_predict_instances_format,\n",
|
||||
" target_field_name=target_column_name,\n",
|
||||
" ground_truth_column=target_column_name,\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" # Run Batch Explanations\n",
|
||||
@@ -1720,9 +1720,10 @@
|
||||
" predictions_gcs_source=batch_explain_task.outputs[\"gcs_output_directory\"],\n",
|
||||
" ground_truth_format=\"jsonl\",\n",
|
||||
" ground_truth_gcs_source=data_sampler_task.outputs[\"gcs_output_directory\"],\n",
|
||||
" key_columns=key_columns,\n",
|
||||
" predictions_format=batch_predict_predictions_format,\n",
|
||||
" prediction_score_column=\"prediction\",\n",
|
||||
" target_field_name=target_column_name,\n",
|
||||
" ground_truth_column=target_column_name,\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" # Get Feature Attributions\n",
|
||||
|
||||
BIN
Binary file not shown.
|
Before Width: | Height: | Size: 109 KiB After Width: | Height: | Size: 55 KiB |
-1980
File diff suppressed because it is too large
Load Diff
+98
-145
@@ -202,29 +202,9 @@
|
||||
"if IS_WORKBENCH_NOTEBOOK:\n",
|
||||
" USER_FLAG = \"--user\"\n",
|
||||
"\n",
|
||||
"! pip3 install google-cloud-aiplatform {USER_FLAG} -q\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "b24902cde81b"
|
||||
},
|
||||
"source": [
|
||||
"### Restart the kernel\n",
|
||||
"! pip3 install google-cloud-aiplatform {USER_FLAG} -q\n",
|
||||
"\n",
|
||||
"Once you've installed the additional packages, you need to restart the notebook kernel so it can find the packages."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "c61d171395d7"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"# Automatically restart kernel after installs\n",
|
||||
"\n",
|
||||
"if not os.getenv(\"IS_TESTING\"):\n",
|
||||
" # Automatically restart kernel after installs\n",
|
||||
@@ -234,13 +214,21 @@
|
||||
" app.kernel.do_shutdown(True)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "r_dA3M6UJELw"
|
||||
},
|
||||
"source": [
|
||||
"## Before you begin"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "1Dunp1YrhPYo"
|
||||
},
|
||||
"source": [
|
||||
"## Before you begin\n",
|
||||
"### Set up your Google Cloud project\n",
|
||||
"\n",
|
||||
"**The following steps are required, regardless of your notebook environment.**\n",
|
||||
@@ -278,17 +266,6 @@
|
||||
"**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "cde8e0876d62"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"PROJECT_ID = \"[your-project-id]\" # @param {type:\"string\"}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
@@ -297,11 +274,15 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
|
||||
" # Get your GCP project id from gcloud\n",
|
||||
" shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
|
||||
"PROJECT_ID = \"\"\n",
|
||||
"\n",
|
||||
"import os\n",
|
||||
"\n",
|
||||
"# Get your Google Cloud project ID from gcloud\n",
|
||||
"if not os.getenv(\"IS_TESTING\"):\n",
|
||||
" shell_output = !gcloud config list --format 'value(core.project)' 2>/dev/null\n",
|
||||
" PROJECT_ID = shell_output[0]\n",
|
||||
" print(\"Project ID:\", PROJECT_ID)"
|
||||
" print(\"Project ID: \", PROJECT_ID)"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -321,7 +302,8 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gcloud config set project $PROJECT_ID"
|
||||
"if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
|
||||
" PROJECT_ID = \"[your-project-id]\" # @param {type:\"string\"}"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -330,9 +312,16 @@
|
||||
"id": "K-KuU54IaVz5"
|
||||
},
|
||||
"source": [
|
||||
"#### UUID\n",
|
||||
"\n",
|
||||
"If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
|
||||
"### Timestamp"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "doEJxrvsaWyt"
|
||||
},
|
||||
"source": [
|
||||
"If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append it onto the name of resources you create in this tutorial."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -343,16 +332,9 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import random\n",
|
||||
"import string\n",
|
||||
"from datetime import datetime\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Generate a uuid of a specifed length(default=8)\n",
|
||||
"def generate_uuid(length: int = 8) -> str:\n",
|
||||
" return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"UUID = generate_uuid()"
|
||||
"TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -361,18 +343,7 @@
|
||||
"id": "Ee3vBgvdhgTb"
|
||||
},
|
||||
"source": [
|
||||
"#### Region\n",
|
||||
"\n",
|
||||
"You can also change the `REGION` variable, which is used for operations\n",
|
||||
"throughout the rest of this notebook. Below are regions supported for Vertex AI. It is recommended that you choose the region closest to you.\n",
|
||||
"\n",
|
||||
"- Americas: `us-central1`\n",
|
||||
"- Europe: `europe-west4`\n",
|
||||
"- Asia Pacific: `asia-east1`\n",
|
||||
"\n",
|
||||
"You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
|
||||
"\n",
|
||||
"Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
|
||||
"### Set your region"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -384,7 +355,6 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"REGION = \"[your-region]\" # @param {type: \"string\"}\n",
|
||||
"\n",
|
||||
"if REGION == \"[your-region]\":\n",
|
||||
" REGION = \"us-central1\""
|
||||
]
|
||||
@@ -395,47 +365,16 @@
|
||||
"id": "KuNRbXkIijp6"
|
||||
},
|
||||
"source": [
|
||||
"### Authenticate your Google Cloud account\n",
|
||||
"\n",
|
||||
"**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
|
||||
"authenticated."
|
||||
"### Login to your Google Cloud account"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "f40aa139740f"
|
||||
},
|
||||
"source": [
|
||||
"**If you are using Colab**, run the cell below and follow the instructions\n",
|
||||
"when prompted to authenticate your account via oAuth.\n",
|
||||
"\n",
|
||||
"**Otherwise**, follow these steps:\n",
|
||||
"\n",
|
||||
"1. In the Cloud Console, go to the [**Create service account key**\n",
|
||||
" page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
|
||||
"\n",
|
||||
"2. Click **Create service account**.\n",
|
||||
"\n",
|
||||
"3. In the **Service account name** field, enter a name, and\n",
|
||||
" click **Create**.\n",
|
||||
"\n",
|
||||
"4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
|
||||
"into the filter box, and select\n",
|
||||
" **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
|
||||
"\n",
|
||||
"5. Click *Create*. A JSON file that contains your key downloads to your\n",
|
||||
"local environment.\n",
|
||||
"\n",
|
||||
"6. Enter the path to your service account key as the\n",
|
||||
"`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "P9vQxUzfirCV"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# The Google Cloud Notebook product has specific requirements\n",
|
||||
"import os\n",
|
||||
@@ -491,7 +430,7 @@
|
||||
"BUCKET_URI = \"gs://[your-bucket-name]\" # @param {type:\"string\"}\n",
|
||||
"\n",
|
||||
"if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
|
||||
" BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + UUID"
|
||||
" BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -578,25 +517,10 @@
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "4eaef8c7be0e"
|
||||
"id": "0j1NWIQEJI5i"
|
||||
},
|
||||
"source": [
|
||||
"### Enable Artifact Registry API\n",
|
||||
"First, you must enable the Artifact Registry API service for your project.\n",
|
||||
"\n",
|
||||
"Learn more about [Enabling service\n",
|
||||
" page](https://cloud.google.com/artifact-registry/docs/enable-service)."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "d03035c8fb6f"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"!gcloud services enable artifactregistry.googleapis.com"
|
||||
"## Create Docker repository"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -605,8 +529,6 @@
|
||||
"id": "hNmHMIyjBzxx"
|
||||
},
|
||||
"source": [
|
||||
"### Create Docker repository\n",
|
||||
"\n",
|
||||
"Create a Docker repository named `DOCKER_REPOSITORY` in your `REGION`.\n",
|
||||
"This docker repository will be deleted in the clearning up section in the end."
|
||||
]
|
||||
@@ -626,7 +548,7 @@
|
||||
" or DOCKER_REPOSITORY is None\n",
|
||||
" or DOCKER_REPOSITORY == \"[your-docker-repository-name]\"\n",
|
||||
"):\n",
|
||||
" DOCKER_REPOSITORY = \"tb-docker-repo-\" + PROJECT_ID + \"-\" + UUID\n",
|
||||
" DOCKER_REPOSITORY = \"tb-docker-repo-\" + PROJECT_ID + \"-\" + TIMESTAMP\n",
|
||||
"\n",
|
||||
"print(\"Docker repository to create:\", DOCKER_REPOSITORY)"
|
||||
]
|
||||
@@ -639,9 +561,18 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gcloud artifacts repositories create $DOCKER_REPOSITORY --project={PROJECT_ID} \\\n",
|
||||
"! gcloud artifacts repositories create $DOCKER_REPOSITORY --project={PROJECT_ID} \\\n",
|
||||
"--repository-format=docker \\\n",
|
||||
"--location={REGION} --description=\"Repository for TensorBoard Custom Training Job\" "
|
||||
"--location={REGION} --description=\"Repository for TensorBoard Custom Training Job\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "V0pBSC0rDlvq"
|
||||
},
|
||||
"source": [
|
||||
"Verify your Docker repository is created successfully."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -683,7 +614,6 @@
|
||||
"id": "jUcVG77dKmPn"
|
||||
},
|
||||
"source": [
|
||||
"### Create a training code\n",
|
||||
"Write your own training code in task.py file. You can use the following code as an example."
|
||||
]
|
||||
},
|
||||
@@ -823,9 +753,7 @@
|
||||
"id": "DK2E1xz8Q7Q-"
|
||||
},
|
||||
"source": [
|
||||
"Build your container image using `gcloud builds` from your training code and `Dockerfile`. \n",
|
||||
"\n",
|
||||
"*Note* that this step may take a few minutes."
|
||||
"Build your container image using `gcloud builds` from your training code and `Dockerfile`. Note that this step may take a few minutes."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -845,14 +773,21 @@
|
||||
"! gcloud builds submit --project {PROJECT_ID} --region={REGION} --tag {IMAGE_URI} --timeout=20m"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "hwXxa4Qgnh4Y"
|
||||
},
|
||||
"source": [
|
||||
"## Setup service account and permissions"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "7qXFUiHLoFRw"
|
||||
},
|
||||
"source": [
|
||||
"## Setup service account and permissions\n",
|
||||
"\n",
|
||||
"A service account will be used to create custom training job. If you do not want to use your project's Compute Engine service account, set SERVICE_ACCOUNT to another service account ID. You can create a service account by following the [instruction](https://cloud.google.com/iam/docs/creating-managing-service-accounts#creating)."
|
||||
]
|
||||
},
|
||||
@@ -864,7 +799,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"SERVICE_ACCOUNT = \"[your-service-account]\""
|
||||
"SERVICE_ACCOUNT = \"[your-service-account]\" # @param {type:\"string\"}"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -875,9 +810,6 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import sys\n",
|
||||
"\n",
|
||||
"IS_COLAB = \"google.colab\" in sys.modules\n",
|
||||
"if (\n",
|
||||
" SERVICE_ACCOUNT == \"\"\n",
|
||||
" or SERVICE_ACCOUNT is None\n",
|
||||
@@ -900,13 +832,37 @@
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "c7798d69970b"
|
||||
"id": "UlDhuciOt5vo"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
|
||||
"\n",
|
||||
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
|
||||
"# Grant Cloud Storage permission.\n",
|
||||
"! gcloud projects add-iam-policy-binding {PROJECT_ID} \\\n",
|
||||
" --member=serviceAccount:{SERVICE_ACCOUNT} \\\n",
|
||||
" --role=roles/storage.admin"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "lTKVB71soRyr"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Grant AI Platform permission.\n",
|
||||
"! gcloud projects add-iam-policy-binding {PROJECT_ID} \\\n",
|
||||
" --member=serviceAccount:{SERVICE_ACCOUNT} \\\n",
|
||||
" --role=roles/aiplatform.user"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "IaQjIPvuKLwW"
|
||||
},
|
||||
"source": [
|
||||
"## Create a custom training job with your container"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -915,7 +871,6 @@
|
||||
"id": "svUGBOow_Obj"
|
||||
},
|
||||
"source": [
|
||||
"## Create a custom training job with your container\n",
|
||||
"Create a TensorBoard instnace to be used by the custom training job."
|
||||
]
|
||||
},
|
||||
@@ -934,7 +889,7 @@
|
||||
" or TENSORBOARD_NAME is None\n",
|
||||
" or TENSORBOARD_NAME == \"[your-tensorboard-name]\"\n",
|
||||
"):\n",
|
||||
" TENSORBOARD_NAME = PROJECT_ID + \"-tb-\" + UUID\n",
|
||||
" TENSORBOARD_NAME = PROJECT_ID + \"-tb-\" + TIMESTAMP\n",
|
||||
"\n",
|
||||
"tensorboard = aiplatform.Tensorboard.create(\n",
|
||||
" display_name=TENSORBOARD_NAME, project=PROJECT_ID, location=REGION\n",
|
||||
@@ -960,7 +915,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"JOB_NAME = \"tensorboard-example-job-{}\".format(UUID)\n",
|
||||
"JOB_NAME = \"tensorboard-example-job-{}\".format(TIMESTAMP)\n",
|
||||
"BASE_OUTPUT_DIR = \"{}/{}\".format(BUCKET_URI, JOB_NAME)\n",
|
||||
"\n",
|
||||
"job = aiplatform.CustomContainerTrainingJob(\n",
|
||||
@@ -1009,6 +964,9 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Delete GCS bucket.\n",
|
||||
"! gsutil -m rm -r {BUCKET_URI}\n",
|
||||
"\n",
|
||||
"# Delete docker repository.\n",
|
||||
"! gcloud artifacts repositories delete $DOCKER_REPOSITORY --project {PROJECT_ID} --location {REGION} --quiet\n",
|
||||
"\n",
|
||||
@@ -1016,12 +974,7 @@
|
||||
"! gcloud ai tensorboards delete {TENSORBOARD_RESOURCE_NAME}\n",
|
||||
"\n",
|
||||
"# Delete custom job.\n",
|
||||
"job.delete()\n",
|
||||
"\n",
|
||||
"# Delete GCS bucket.\n",
|
||||
"delete_bucket = False\n",
|
||||
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
|
||||
" ! gsutil -m rm -r $BUCKET_URI"
|
||||
"job.delete()"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -34,18 +34,18 @@
|
||||
"<table align=\"left\">\n",
|
||||
"\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/training/hyperparameter_tuning_tensorflow.ipynb\">\n",
|
||||
" <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/notebook_template.ipynb\">\n",
|
||||
" <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/training/hyperparameter_tuning_tensorflow.ipynb\">\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/notebook_template.ipynb\">\n",
|
||||
" <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/training/hyperparameter_tuning_tensorflow.ipynb\">\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/notebook_template.ipynb\">\n",
|
||||
" <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
|
||||
" Open in Vertex AI Workbench\n",
|
||||
" </a>\n",
|
||||
|
||||
Reference in New Issue
Block a user