mirror of
https://github.com/GoogleCloudPlatform/vertex-ai-samples.git
synced 2026-09-26 22:51:56 +00:00
Compare commits
5
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Andrew Ferlitsch
|
a0f2c5edbd | ||
|
Andrew Ferlitsch
|
852f1bbddd | ||
|
|
c9aa6df17b | ||
|
|
b4955b28a2 | ||
|
Andrew FerlitschandGitHub
|
f33d9eb6d1 |
@@ -6,3 +6,4 @@
|
||||
/sklearn_text_classification_from_script_using_vertex_sdk @maxhardt
|
||||
/pluto_on_workbench @wkharold
|
||||
/cpr-examples @samthrasher
|
||||
/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines @Ark-kun
|
||||
|
||||
+83
@@ -0,0 +1,83 @@
|
||||
name: Train tabular classification logistic regression model using Scikit learn pipeline
|
||||
metadata:
|
||||
annotations:
|
||||
author: Alexey Volkov <alexey.volkov@ark-kun.com>
|
||||
canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_classification_logistic_regression_model_using_Scikit_learn_and_import_to_Vertex_AI/pipeline.component.yaml
|
||||
sdk: https://cloud-pipelines.net/pipeline-editor/
|
||||
implementation:
|
||||
graph:
|
||||
tasks:
|
||||
Download from GCS:
|
||||
componentRef:
|
||||
digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":40,"width":180,"height":40}'
|
||||
Select columns using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: Data
|
||||
taskId: Download from GCS
|
||||
column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":140,"width":180,"height":54}'
|
||||
Fill all missing values using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Select columns using Pandas on CSV data
|
||||
type: CSV
|
||||
replacement_value: '0'
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":250,"width":180,"height":54}'
|
||||
Binarize column using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: d699afd4d7cae862708717cc160f4394ed0c04e536e9515923ef1e8865f01d44
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Fill all missing values using Pandas on CSV data
|
||||
type: CSV
|
||||
column_name: tips
|
||||
predicate: '> 0'
|
||||
new_column_name: class
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":380,"width":180,"height":54}'
|
||||
Train logistic regression model using scikit learn from CSV:
|
||||
componentRef:
|
||||
digest: a864625a822e4b1c8ef6fe4ae1454fd90f15438f70a6712bb4c30e0dda4d35b7
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/cb44b75c9c062fcc40c2b905b2024b4493dbc62b/components/ML_frameworks/Scikit_learn/Train_logistic_regression_model/from_CSV/component.yaml
|
||||
arguments:
|
||||
dataset:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Binarize column using Pandas on CSV data
|
||||
type: CSV
|
||||
label_column_name: class
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":510,"width":180,"height":70}'
|
||||
Upload Scikit learn pickle model to Google Cloud Vertex AI:
|
||||
componentRef:
|
||||
digest: 81c91c8d7d21ec97e0872f669d68bd89edea87279d703685db54aa94743bebcd
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Train logistic regression model using scikit learn from CSV
|
||||
type: ScikitLearnPickleModel
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":660,"width":180,"height":70}'
|
||||
outputValues: {}
|
||||
+73
@@ -0,0 +1,73 @@
|
||||
# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
|
||||
from kfp import components
|
||||
|
||||
# %% Loading components
|
||||
download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml")
|
||||
fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
|
||||
binarize_column_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml")
|
||||
train_logistic_regression_model_using_scikit_learn_from_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/cb44b75c9c062fcc40c2b905b2024b4493dbc62b/components/ML_frameworks/Scikit_learn/Train_logistic_regression_model/from_CSV/component.yaml")
|
||||
upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
|
||||
# %% Pipeline definition
|
||||
def train_tabular_classification_logistic_regression_model_using_Scikit_learn_pipeline():
|
||||
dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
|
||||
feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"] # Excluded "trip_total"
|
||||
label_column = "tips"
|
||||
# Deploying the model might incur additional costs over time
|
||||
deploy_model = False
|
||||
|
||||
classification_label_column = "class"
|
||||
all_columns = [label_column] + feature_columns
|
||||
|
||||
training_data = download_from_gcs_op(
|
||||
gcs_path=dataset_gcs_uri
|
||||
).outputs["Data"]
|
||||
|
||||
training_data = select_columns_using_Pandas_on_CSV_data_op(
|
||||
table=training_data,
|
||||
column_names=all_columns,
|
||||
).outputs["transformed_table"]
|
||||
|
||||
# Cleaning the NaN values.
|
||||
training_data = fill_all_missing_values_using_Pandas_on_CSV_data_op(
|
||||
table=training_data,
|
||||
replacement_value="0",
|
||||
#replacement_type_name="float",
|
||||
).outputs["transformed_table"]
|
||||
|
||||
classification_training_data = binarize_column_using_Pandas_on_CSV_data_op(
|
||||
table=training_data,
|
||||
column_name=label_column,
|
||||
predicate="> 0",
|
||||
new_column_name=classification_label_column,
|
||||
).outputs["transformed_table"]
|
||||
|
||||
model = train_logistic_regression_model_using_scikit_learn_from_CSV_op(
|
||||
dataset=classification_training_data,
|
||||
label_column_name=classification_label_column,
|
||||
# Optional:
|
||||
#penalty="l2",
|
||||
#solver="lbfgs",
|
||||
#max_iterations=100,
|
||||
#multi_class_mode="auto",
|
||||
#random_seed=0,
|
||||
).outputs["model"]
|
||||
|
||||
vertex_model_name = upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op(
|
||||
model=model,
|
||||
).outputs["model_name"]
|
||||
|
||||
# Deploying the model might incur additional costs over time
|
||||
if deploy_model:
|
||||
sklearn_vertex_endpoint_name = deploy_model_to_endpoint_op(
|
||||
model_name=vertex_model_name,
|
||||
).outputs["endpoint_name"]
|
||||
|
||||
pipeline_func = train_tabular_classification_logistic_regression_model_using_Scikit_learn_pipeline
|
||||
|
||||
# %% Pipeline submission
|
||||
if __name__ == '__main__':
|
||||
from google.cloud import aiplatform
|
||||
aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()
|
||||
+114
@@ -0,0 +1,114 @@
|
||||
name: Train tabular classification model using PyTorch pipeline
|
||||
metadata:
|
||||
annotations:
|
||||
author: Alexey Volkov <alexey.volkov@ark-kun.com>
|
||||
canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_classification_model_using_PyTorch_and_import_to_Vertex_AI/pipeline.component.yaml
|
||||
sdk: https://cloud-pipelines.net/pipeline-editor/
|
||||
implementation:
|
||||
graph:
|
||||
tasks:
|
||||
Download from GCS:
|
||||
componentRef:
|
||||
digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
|
||||
annotations:
|
||||
editor.position: '{"x":240,"y":40,"width":180,"height":40}'
|
||||
Select columns using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: Data
|
||||
taskId: Download from GCS
|
||||
column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
|
||||
annotations:
|
||||
editor.position: '{"x":240,"y":140,"width":180,"height":54}'
|
||||
Fill all missing values using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Select columns using Pandas on CSV data
|
||||
type: CSV
|
||||
replacement_value: '0'
|
||||
annotations:
|
||||
editor.position: '{"x":240,"y":250,"width":180,"height":54}'
|
||||
Create fully connected pytorch network:
|
||||
componentRef:
|
||||
digest: d03d8248fd358a0275ec33568ee7dd7dce576cc112b09dfafe2651e4d97e04a9
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1a2ef3eeb77bc278f33cad0dd29008ea2431e191/components/PyTorch/Create_fully_connected_network/component.yaml
|
||||
arguments:
|
||||
input_size: '7'
|
||||
hidden_layer_sizes: '[10]'
|
||||
activation_name: elu
|
||||
output_activation_name: sigmoid
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":360,"width":180,"height":54}'
|
||||
Binarize column using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: d699afd4d7cae862708717cc160f4394ed0c04e536e9515923ef1e8865f01d44
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Fill all missing values using Pandas on CSV data
|
||||
type: CSV
|
||||
column_name: tips
|
||||
predicate: ' > 0'
|
||||
new_column_name: class
|
||||
annotations:
|
||||
editor.position: '{"x":240,"y":360,"width":180,"height":54}'
|
||||
Train pytorch model from csv:
|
||||
componentRef:
|
||||
digest: 40f3185eb61e9727f41a4e0c05dd3d3b44bd802aa0f378cfc31756560033949a
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml
|
||||
arguments:
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Create fully connected pytorch network
|
||||
type: PyTorchScriptModule
|
||||
training_data:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Binarize column using Pandas on CSV data
|
||||
type: CSV
|
||||
label_column_name: class
|
||||
loss_function_name: binary_cross_entropy
|
||||
annotations:
|
||||
editor.position: '{"x":240,"y":490,"width":180,"height":40}'
|
||||
Create PyTorch Model Archive with base handler:
|
||||
componentRef:
|
||||
digest: 8298b5ee1b0f0879f893add4cf352c8dec7cf9e21bb9db134c91a2d046cdb0ec
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/46d51383e6554b7f3ab4fd8cf614d8c2b422fb22/components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml
|
||||
arguments:
|
||||
Model:
|
||||
taskOutput:
|
||||
outputName: trained_model
|
||||
taskId: Train pytorch model from csv
|
||||
type: PyTorchScriptModule
|
||||
Model name: model
|
||||
Model version: '1.0'
|
||||
annotations:
|
||||
editor.position: '{"x":240,"y":590,"width":180,"height":54}'
|
||||
Upload PyTorch model archive to Google Cloud Vertex AI:
|
||||
componentRef:
|
||||
digest: 4450212fae7b9001482aca7eb78b28413c205506eccf08a04e7754a8dfa99004
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
model_archive:
|
||||
taskOutput:
|
||||
outputName: Model archive
|
||||
taskId: Create PyTorch Model Archive with base handler
|
||||
type: PyTorchModelArchive
|
||||
annotations:
|
||||
editor.position: '{"x":240,"y":720,"width":180,"height":70}'
|
||||
outputValues: {}
|
||||
+95
@@ -0,0 +1,95 @@
|
||||
# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
|
||||
from kfp import components
|
||||
|
||||
# %% Loading components
|
||||
download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml")
|
||||
fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
|
||||
binarize_column_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml")
|
||||
create_fully_connected_pytorch_network_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/1a2ef3eeb77bc278f33cad0dd29008ea2431e191/components/PyTorch/Create_fully_connected_network/component.yaml")
|
||||
train_pytorch_model_from_csv_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml")
|
||||
create_pytorch_model_archive_with_base_handler_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/46d51383e6554b7f3ab4fd8cf614d8c2b422fb22/components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml")
|
||||
upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
|
||||
# %% Pipeline definition
|
||||
def train_tabular_classification_model_using_PyTorch_pipeline():
|
||||
dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
|
||||
feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"] # Excluded "trip_total"
|
||||
label_column = "tips"
|
||||
# Deploying the model might incur additional costs over time
|
||||
deploy_model = False
|
||||
|
||||
classification_label_column = "class"
|
||||
all_columns = [label_column] + feature_columns
|
||||
|
||||
training_data = download_from_gcs_op(
|
||||
gcs_path=dataset_gcs_uri
|
||||
).outputs["Data"]
|
||||
|
||||
training_data = select_columns_using_Pandas_on_CSV_data_op(
|
||||
table=training_data,
|
||||
column_names=all_columns,
|
||||
).outputs["transformed_table"]
|
||||
|
||||
# Cleaning the NaN values.
|
||||
training_data = fill_all_missing_values_using_Pandas_on_CSV_data_op(
|
||||
table=training_data,
|
||||
replacement_value="0",
|
||||
#replacement_type_name="float",
|
||||
).outputs["transformed_table"]
|
||||
|
||||
classification_training_data = binarize_column_using_Pandas_on_CSV_data_op(
|
||||
table=training_data,
|
||||
column_name=label_column,
|
||||
predicate=" > 0",
|
||||
new_column_name=classification_label_column,
|
||||
).outputs["transformed_table"]
|
||||
|
||||
network = create_fully_connected_pytorch_network_op(
|
||||
input_size=len(feature_columns),
|
||||
# Optional:
|
||||
hidden_layer_sizes=[10],
|
||||
activation_name="elu",
|
||||
output_activation_name="sigmoid",
|
||||
# output_size=1,
|
||||
).outputs["model"]
|
||||
|
||||
model = train_pytorch_model_from_csv_op(
|
||||
model=network,
|
||||
training_data=classification_training_data,
|
||||
label_column_name=classification_label_column,
|
||||
loss_function_name="binary_cross_entropy",
|
||||
# Optional:
|
||||
#number_of_epochs=1,
|
||||
#learning_rate=0.1,
|
||||
#optimizer_name="Adadelta",
|
||||
#optimizer_parameters={},
|
||||
#batch_size=32,
|
||||
#batch_log_interval=100,
|
||||
#random_seed=0,
|
||||
).outputs["trained_model"]
|
||||
|
||||
model_archive = create_pytorch_model_archive_with_base_handler_op(
|
||||
model=model,
|
||||
# Optional:
|
||||
# model_name="model",
|
||||
# model_version="1.0",
|
||||
).outputs["Model archive"]
|
||||
|
||||
vertex_model_name = upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op(
|
||||
model_archive=model_archive,
|
||||
).outputs["model_name"]
|
||||
|
||||
# Deploying the model might incur additional costs over time
|
||||
if deploy_model:
|
||||
vertex_endpoint_name = deploy_model_to_endpoint_op(
|
||||
model_name=vertex_model_name,
|
||||
).outputs["endpoint_name"]
|
||||
|
||||
pipeline_func=train_tabular_classification_model_using_PyTorch_pipeline
|
||||
|
||||
# %% Pipeline submission
|
||||
if __name__ == '__main__':
|
||||
from google.cloud import aiplatform
|
||||
aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()
|
||||
+132
@@ -0,0 +1,132 @@
|
||||
name: Train tabular classification model using TensorFlow pipeline
|
||||
metadata:
|
||||
annotations:
|
||||
author: Alexey Volkov <alexey.volkov@ark-kun.com>
|
||||
canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_classification_model_using_TensorFlow_and_import_to_Vertex_AI/pipeline.component.yaml
|
||||
sdk: https://cloud-pipelines.net/pipeline-editor/
|
||||
implementation:
|
||||
graph:
|
||||
tasks:
|
||||
Download from GCS:
|
||||
componentRef:
|
||||
digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":40,"width":180,"height":40}'
|
||||
Select columns using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: Data
|
||||
taskId: Download from GCS
|
||||
column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":140,"width":180,"height":54}'
|
||||
Fill all missing values using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Select columns using Pandas on CSV data
|
||||
type: CSV
|
||||
replacement_value: '0'
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":250,"width":180,"height":54}'
|
||||
Binarize column using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: d699afd4d7cae862708717cc160f4394ed0c04e536e9515923ef1e8865f01d44
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Fill all missing values using Pandas on CSV data
|
||||
type: CSV
|
||||
column_name: tips
|
||||
predicate: ' > 0'
|
||||
new_column_name: class
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":370,"width":180,"height":54}'
|
||||
Split rows into subsets:
|
||||
componentRef:
|
||||
digest: a609c3c9196484290f24a1174955f95b27f07a7b458aa5cb8cde28866cb2cb46
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Binarize column using Pandas on CSV data
|
||||
type: CSV
|
||||
fraction_1: '0.8'
|
||||
annotations:
|
||||
editor.position: '{"x":170,"y":500,"width":180,"height":40}'
|
||||
Create fully connected tensorflow network:
|
||||
componentRef:
|
||||
digest: bfcafbc5ce711b1f69cabf1338212d10d50136a73db9f9f7c984de7b80b4bfb0
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/9ca0f9eecf5f896f65b8538bbd809747052617d1/components/tensorflow/Create_fully_connected_network/component.yaml
|
||||
arguments:
|
||||
input_size: '7'
|
||||
hidden_layer_sizes: '[10]'
|
||||
activation_name: elu
|
||||
output_activation_name: sigmoid
|
||||
annotations:
|
||||
editor.position: '{"x":370,"y":500,"width":180,"height":54}'
|
||||
Train model using Keras on CSV:
|
||||
componentRef:
|
||||
digest: 42ae60c889034dbad74815653e95b4f7d576b5f47f803173e8679c7b54984609
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c504a4010348c50eaaf6d4337586ccc008f4dcef/components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml
|
||||
arguments:
|
||||
training_data:
|
||||
taskOutput:
|
||||
outputName: split_1
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Create fully connected tensorflow network
|
||||
type: TensorflowSavedModel
|
||||
label_column_name: class
|
||||
loss_function_name: binary_crossentropy
|
||||
number_of_epochs: '10'
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":620,"width":180,"height":54}'
|
||||
Upload Tensorflow model to Google Cloud Vertex AI:
|
||||
componentRef:
|
||||
digest: 2e45263ff640b1a688e359b6936e27a81b2407749a84f340af2aa5547e0cb92c
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: trained_model
|
||||
taskId: Train model using Keras on CSV
|
||||
type: TensorflowSavedModel
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":750,"width":180,"height":54}'
|
||||
Predict with TensorFlow model on CSV data:
|
||||
componentRef:
|
||||
digest: 921bb1563e93a78233b8acceab87055b9154ccf5595d056028cf0396ca224cd4
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/59c759ce6f543184e30db6817d2a703879bc0f39/components/tensorflow/Predict/on_CSV/component.yaml
|
||||
arguments:
|
||||
dataset:
|
||||
taskOutput:
|
||||
outputName: split_2
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: trained_model
|
||||
taskId: Train model using Keras on CSV
|
||||
type: TensorflowSavedModel
|
||||
label_column_name: class
|
||||
annotations:
|
||||
editor.position: '{"x":240,"y":750,"width":180,"height":54}'
|
||||
outputValues: {}
|
||||
+106
@@ -0,0 +1,106 @@
|
||||
# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
|
||||
from kfp import components
|
||||
|
||||
# %% Loading components
|
||||
download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml")
|
||||
fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
|
||||
binarize_column_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml")
|
||||
split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
|
||||
create_fully_connected_tensorflow_network_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/9ca0f9eecf5f896f65b8538bbd809747052617d1/components/tensorflow/Create_fully_connected_network/component.yaml")
|
||||
train_model_using_Keras_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c504a4010348c50eaaf6d4337586ccc008f4dcef/components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml")
|
||||
predict_with_TensorFlow_model_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/59c759ce6f543184e30db6817d2a703879bc0f39/components/tensorflow/Predict/on_CSV/component.yaml")
|
||||
upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
|
||||
# %% Pipeline definition
|
||||
def train_tabular_classification_model_using_TensorFlow_pipeline():
|
||||
dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
|
||||
feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"] # Excluded "trip_total"
|
||||
label_column = "tips"
|
||||
training_set_fraction = 0.8
|
||||
# Deploying the model might incur additional costs over time
|
||||
deploy_model = False
|
||||
|
||||
classification_label_column = "class"
|
||||
all_columns = [label_column] + feature_columns
|
||||
|
||||
dataset = download_from_gcs_op(
|
||||
gcs_path=dataset_gcs_uri
|
||||
).outputs["Data"]
|
||||
|
||||
dataset = select_columns_using_Pandas_on_CSV_data_op(
|
||||
table=dataset,
|
||||
column_names=all_columns,
|
||||
).outputs["transformed_table"]
|
||||
|
||||
dataset = fill_all_missing_values_using_Pandas_on_CSV_data_op(
|
||||
table=dataset,
|
||||
replacement_value="0",
|
||||
# # Optional:
|
||||
# column_names=None, # =[...]
|
||||
).outputs["transformed_table"]
|
||||
|
||||
classification_dataset = binarize_column_using_Pandas_on_CSV_data_op(
|
||||
table=dataset,
|
||||
column_name=label_column,
|
||||
predicate=" > 0",
|
||||
new_column_name=classification_label_column,
|
||||
).outputs["transformed_table"]
|
||||
|
||||
split_task = split_rows_into_subsets_op(
|
||||
table=classification_dataset,
|
||||
fraction_1=training_set_fraction,
|
||||
)
|
||||
classification_training_data = split_task.outputs["split_1"]
|
||||
classification_testing_data = split_task.outputs["split_2"]
|
||||
|
||||
network = create_fully_connected_tensorflow_network_op(
|
||||
input_size=len(feature_columns),
|
||||
# Optional:
|
||||
hidden_layer_sizes=[10],
|
||||
activation_name="elu",
|
||||
output_activation_name="sigmoid",
|
||||
# output_size=1,
|
||||
).outputs["model"]
|
||||
|
||||
model = train_model_using_Keras_on_CSV_op(
|
||||
training_data=classification_training_data,
|
||||
model=network,
|
||||
label_column_name=classification_label_column,
|
||||
# Optional:
|
||||
loss_function_name="binary_crossentropy",
|
||||
number_of_epochs=10,
|
||||
#learning_rate=0.1,
|
||||
#optimizer_name="Adadelta",
|
||||
#optimizer_parameters={},
|
||||
#batch_size=32,
|
||||
#metric_names=["mean_absolute_error"],
|
||||
#random_seed=0,
|
||||
).outputs["trained_model"]
|
||||
|
||||
predictions = predict_with_TensorFlow_model_on_CSV_data_op(
|
||||
dataset=classification_testing_data,
|
||||
model=model,
|
||||
# label_column_name needs to be set when doing prediction on a dataset that has labels
|
||||
label_column_name=classification_label_column,
|
||||
# Optional:
|
||||
# batch_size=1000,
|
||||
).outputs["predictions"]
|
||||
|
||||
vertex_model_name = upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op(
|
||||
model=model,
|
||||
).outputs["model_name"]
|
||||
|
||||
# Deploying the model might incur additional costs over time
|
||||
if deploy_model:
|
||||
vertex_endpoint_name = deploy_model_to_endpoint_op(
|
||||
model_name=vertex_model_name,
|
||||
).outputs["endpoint_name"]
|
||||
|
||||
pipeline_func = train_tabular_classification_model_using_TensorFlow_pipeline
|
||||
|
||||
# %% Pipeline submission
|
||||
if __name__ == '__main__':
|
||||
from google.cloud import aiplatform
|
||||
aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()
|
||||
+115
@@ -0,0 +1,115 @@
|
||||
name: Train tabular classification model using XGBoost pipeline
|
||||
metadata:
|
||||
annotations:
|
||||
author: Alexey Volkov <alexey.volkov@ark-kun.com>
|
||||
canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_classification_model_using_XGBoost_and_import_to_Vertex_AI/pipeline.component.yaml
|
||||
sdk: https://cloud-pipelines.net/pipeline-editor/
|
||||
implementation:
|
||||
graph:
|
||||
tasks:
|
||||
Download from GCS:
|
||||
componentRef:
|
||||
digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":40,"width":180,"height":40}'
|
||||
Select columns using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: Data
|
||||
taskId: Download from GCS
|
||||
column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":140,"width":180,"height":54}'
|
||||
Fill all missing values using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Select columns using Pandas on CSV data
|
||||
type: CSV
|
||||
replacement_value: '0'
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":250,"width":180,"height":54}'
|
||||
Binarize column using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: d699afd4d7cae862708717cc160f4394ed0c04e536e9515923ef1e8865f01d44
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Fill all missing values using Pandas on CSV data
|
||||
type: CSV
|
||||
column_name: tips
|
||||
predicate: '> 0'
|
||||
new_column_name: class
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":380,"width":180,"height":54}'
|
||||
Split rows into subsets:
|
||||
componentRef:
|
||||
digest: a609c3c9196484290f24a1174955f95b27f07a7b458aa5cb8cde28866cb2cb46
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Binarize column using Pandas on CSV data
|
||||
type: CSV
|
||||
fraction_1: '0.8'
|
||||
annotations:
|
||||
editor.position: '{"x":170,"y":510,"width":180,"height":40}'
|
||||
Train XGBoost model on CSV:
|
||||
componentRef:
|
||||
digest: 538c5a01eb38deaf532d619f0bbeaff4efc550fe1f0f776fc06791097b68ceac
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/58d3a47f904f32a64af8403330ba7e2134cae46d/components/XGBoost/Train/component.yaml
|
||||
arguments:
|
||||
training_data:
|
||||
taskOutput:
|
||||
outputName: split_1
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
label_column_name: class
|
||||
objective: binary:logistic
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":630,"width":180,"height":40}'
|
||||
Upload XGBoost model to Google Cloud Vertex AI:
|
||||
componentRef:
|
||||
digest: 5a5a273c403670743820986c03a4175b7cb4595a556524fefcce403656286977
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Train XGBoost model on CSV
|
||||
type: XGBoostModel
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":750,"width":180,"height":54}'
|
||||
Xgboost predict on CSV:
|
||||
componentRef:
|
||||
digest: 0876233a0c7306fefec188bd70f059b46d1fb5aa57be231799570e3bbbdd0d95
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/4694ec97baccf59284c2a1db4aa2250c22291eab/components/XGBoost/Predict/component.yaml
|
||||
arguments:
|
||||
data:
|
||||
taskOutput:
|
||||
outputName: split_2
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Train XGBoost model on CSV
|
||||
type: XGBoostModel
|
||||
label_column_name: class
|
||||
annotations:
|
||||
editor.position: '{"x":240,"y":750,"width":180,"height":40}'
|
||||
outputValues: {}
|
||||
+94
@@ -0,0 +1,94 @@
|
||||
# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
|
||||
from kfp import components
|
||||
|
||||
# %% Loading components
|
||||
download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml")
|
||||
fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
|
||||
binarize_column_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml")
|
||||
split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
|
||||
train_XGBoost_model_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/58d3a47f904f32a64af8403330ba7e2134cae46d/components/XGBoost/Train/component.yaml")
|
||||
xgboost_predict_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/4694ec97baccf59284c2a1db4aa2250c22291eab/components/XGBoost/Predict/component.yaml")
|
||||
upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
|
||||
# %% Pipeline definition
|
||||
def train_tabular_classification_model_using_XGBoost_pipeline():
|
||||
dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
|
||||
feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"] # Excluded "trip_total"
|
||||
label_column = "tips"
|
||||
training_set_fraction = 0.8
|
||||
# Deploying the model might incur additional costs over time
|
||||
deploy_model = False
|
||||
|
||||
classification_label_column = "class"
|
||||
all_columns = [label_column] + feature_columns
|
||||
|
||||
dataset = download_from_gcs_op(
|
||||
gcs_path=dataset_gcs_uri
|
||||
).outputs["Data"]
|
||||
|
||||
dataset = select_columns_using_Pandas_on_CSV_data_op(
|
||||
table=dataset,
|
||||
column_names=all_columns,
|
||||
).outputs["transformed_table"]
|
||||
|
||||
dataset = fill_all_missing_values_using_Pandas_on_CSV_data_op(
|
||||
table=dataset,
|
||||
replacement_value="0",
|
||||
# # Optional:
|
||||
# column_names=None, # =[...]
|
||||
).outputs["transformed_table"]
|
||||
|
||||
classification_dataset = binarize_column_using_Pandas_on_CSV_data_op(
|
||||
table=dataset,
|
||||
column_name=label_column,
|
||||
predicate="> 0",
|
||||
new_column_name=classification_label_column,
|
||||
).outputs["transformed_table"]
|
||||
|
||||
split_task = split_rows_into_subsets_op(
|
||||
table=classification_dataset,
|
||||
fraction_1=training_set_fraction,
|
||||
)
|
||||
classification_training_data = split_task.outputs["split_1"]
|
||||
classification_testing_data = split_task.outputs["split_2"]
|
||||
|
||||
model = train_XGBoost_model_on_CSV_op(
|
||||
training_data=classification_training_data,
|
||||
label_column_name=classification_label_column,
|
||||
objective="binary:logistic",
|
||||
# Optional:
|
||||
#starting_model=None,
|
||||
#num_iterations=10,
|
||||
#booster_params={},
|
||||
#booster="gbtree",
|
||||
#learning_rate=0.3,
|
||||
#min_split_loss=0,
|
||||
#max_depth=6,
|
||||
).outputs["model"]
|
||||
|
||||
# Predicting on the testing data
|
||||
predictions = xgboost_predict_on_CSV_op(
|
||||
data=classification_testing_data,
|
||||
model=model,
|
||||
# label_column needs to be set when doing prediction on a dataset that has labels
|
||||
label_column_name=classification_label_column,
|
||||
).outputs["predictions"]
|
||||
|
||||
vertex_model_name = upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op(
|
||||
model=model,
|
||||
).outputs["model_name"]
|
||||
|
||||
# Deploying the model might incur additional costs over time
|
||||
if deploy_model:
|
||||
vertex_endpoint_name = deploy_model_to_endpoint_op(
|
||||
model_name=vertex_model_name,
|
||||
).outputs["endpoint_name"]
|
||||
|
||||
pipeline_func = train_tabular_classification_model_using_XGBoost_pipeline
|
||||
|
||||
# %% Pipeline submission
|
||||
if __name__ == '__main__':
|
||||
from google.cloud import aiplatform
|
||||
aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()
|
||||
+257
@@ -0,0 +1,257 @@
|
||||
name: Train tabular classification model using all frameworks pipeline
|
||||
metadata:
|
||||
annotations:
|
||||
author: Alexey Volkov <alexey.volkov@ark-kun.com>
|
||||
canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_classification_model_using_all_frameworks_and_import_to_Vertex_AI/pipeline.component.yaml
|
||||
sdk: https://cloud-pipelines.net/pipeline-editor/
|
||||
implementation:
|
||||
graph:
|
||||
tasks:
|
||||
Download from GCS:
|
||||
componentRef:
|
||||
digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
|
||||
annotations:
|
||||
editor.position: '{"x":550,"y":40,"width":180,"height":40}'
|
||||
Select columns using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: Data
|
||||
taskId: Download from GCS
|
||||
column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
|
||||
annotations:
|
||||
editor.position: '{"x":550,"y":140,"width":180,"height":54}'
|
||||
Fill all missing values using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Select columns using Pandas on CSV data
|
||||
type: CSV
|
||||
replacement_value: '0'
|
||||
annotations:
|
||||
editor.position: '{"x":550,"y":250,"width":180,"height":54}'
|
||||
Binarize column using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: d699afd4d7cae862708717cc160f4394ed0c04e536e9515923ef1e8865f01d44
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Fill all missing values using Pandas on CSV data
|
||||
type: CSV
|
||||
column_name: tips
|
||||
predicate: ' > 0'
|
||||
new_column_name: class
|
||||
annotations:
|
||||
editor.position: '{"x":550,"y":380,"width":180,"height":54}'
|
||||
Split rows into subsets:
|
||||
componentRef:
|
||||
digest: a609c3c9196484290f24a1174955f95b27f07a7b458aa5cb8cde28866cb2cb46
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Binarize column using Pandas on CSV data
|
||||
type: CSV
|
||||
fraction_1: '0.8'
|
||||
annotations:
|
||||
editor.position: '{"x":550,"y":490,"width":180,"height":40}'
|
||||
Create fully connected pytorch network:
|
||||
componentRef:
|
||||
digest: d03d8248fd358a0275ec33568ee7dd7dce576cc112b09dfafe2651e4d97e04a9
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1a2ef3eeb77bc278f33cad0dd29008ea2431e191/components/PyTorch/Create_fully_connected_network/component.yaml
|
||||
arguments:
|
||||
input_size: '7'
|
||||
hidden_layer_sizes: '[10]'
|
||||
activation_name: elu
|
||||
output_activation_name: sigmoid
|
||||
annotations:
|
||||
editor.position: '{"x":380,"y":620,"width":180,"height":54}'
|
||||
Create fully connected tensorflow network:
|
||||
componentRef:
|
||||
digest: bfcafbc5ce711b1f69cabf1338212d10d50136a73db9f9f7c984de7b80b4bfb0
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/9ca0f9eecf5f896f65b8538bbd809747052617d1/components/tensorflow/Create_fully_connected_network/component.yaml
|
||||
arguments:
|
||||
input_size: '7'
|
||||
hidden_layer_sizes: '[10]'
|
||||
activation_name: elu
|
||||
output_activation_name: sigmoid
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":630,"width":180,"height":54}'
|
||||
Train model using Keras on CSV:
|
||||
componentRef:
|
||||
digest: 42ae60c889034dbad74815653e95b4f7d576b5f47f803173e8679c7b54984609
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c504a4010348c50eaaf6d4337586ccc008f4dcef/components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml
|
||||
arguments:
|
||||
training_data:
|
||||
taskOutput:
|
||||
outputName: split_1
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Create fully connected tensorflow network
|
||||
type: TensorflowSavedModel
|
||||
label_column_name: class
|
||||
loss_function_name: binary_crossentropy
|
||||
number_of_epochs: '10'
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":750,"width":180,"height":54}'
|
||||
Train pytorch model from csv:
|
||||
componentRef:
|
||||
digest: 40f3185eb61e9727f41a4e0c05dd3d3b44bd802aa0f378cfc31756560033949a
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml
|
||||
arguments:
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Create fully connected pytorch network
|
||||
type: PyTorchScriptModule
|
||||
training_data:
|
||||
taskOutput:
|
||||
outputName: split_1
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
label_column_name: class
|
||||
loss_function_name: binary_cross_entropy
|
||||
annotations:
|
||||
editor.position: '{"x":380,"y":750,"width":180,"height":40}'
|
||||
Train XGBoost model on CSV:
|
||||
componentRef:
|
||||
digest: 538c5a01eb38deaf532d619f0bbeaff4efc550fe1f0f776fc06791097b68ceac
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/58d3a47f904f32a64af8403330ba7e2134cae46d/components/XGBoost/Train/component.yaml
|
||||
arguments:
|
||||
training_data:
|
||||
taskOutput:
|
||||
outputName: split_1
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
label_column_name: class
|
||||
objective: binary:logistic
|
||||
annotations:
|
||||
editor.position: '{"x":720,"y":750,"width":180,"height":40}'
|
||||
Train logistic regression model using scikit learn from CSV:
|
||||
componentRef:
|
||||
digest: a864625a822e4b1c8ef6fe4ae1454fd90f15438f70a6712bb4c30e0dda4d35b7
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/cb44b75c9c062fcc40c2b905b2024b4493dbc62b/components/ML_frameworks/Scikit_learn/Train_logistic_regression_model/from_CSV/component.yaml
|
||||
arguments:
|
||||
dataset:
|
||||
taskOutput:
|
||||
outputName: split_1
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
label_column_name: class
|
||||
annotations:
|
||||
editor.position: '{"x":1030,"y":750,"width":180,"height":70}'
|
||||
Predict with TensorFlow model on CSV data:
|
||||
componentRef:
|
||||
digest: 921bb1563e93a78233b8acceab87055b9154ccf5595d056028cf0396ca224cd4
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/59c759ce6f543184e30db6817d2a703879bc0f39/components/tensorflow/Predict/on_CSV/component.yaml
|
||||
arguments:
|
||||
dataset:
|
||||
taskOutput:
|
||||
outputName: split_2
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: trained_model
|
||||
taskId: Train model using Keras on CSV
|
||||
type: TensorflowSavedModel
|
||||
label_column_name: class
|
||||
annotations:
|
||||
editor.position: '{"x":160,"y":880,"width":180,"height":54}'
|
||||
Create PyTorch Model Archive with base handler:
|
||||
componentRef:
|
||||
digest: 8298b5ee1b0f0879f893add4cf352c8dec7cf9e21bb9db134c91a2d046cdb0ec
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/46d51383e6554b7f3ab4fd8cf614d8c2b422fb22/components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml
|
||||
arguments:
|
||||
Model:
|
||||
taskOutput:
|
||||
outputName: trained_model
|
||||
taskId: Train pytorch model from csv
|
||||
type: PyTorchScriptModule
|
||||
Model name: model
|
||||
Model version: '1.0'
|
||||
annotations:
|
||||
editor.position: '{"x":380,"y":880,"width":180,"height":54}'
|
||||
Xgboost predict on CSV:
|
||||
componentRef:
|
||||
digest: 0876233a0c7306fefec188bd70f059b46d1fb5aa57be231799570e3bbbdd0d95
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/4694ec97baccf59284c2a1db4aa2250c22291eab/components/XGBoost/Predict/component.yaml
|
||||
arguments:
|
||||
data:
|
||||
taskOutput:
|
||||
outputName: split_2
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Train XGBoost model on CSV
|
||||
type: XGBoostModel
|
||||
label_column_name: class
|
||||
annotations:
|
||||
editor.position: '{"x":810,"y":880,"width":180,"height":40}'
|
||||
Upload Scikit learn pickle model to Google Cloud Vertex AI:
|
||||
componentRef:
|
||||
digest: 81c91c8d7d21ec97e0872f669d68bd89edea87279d703685db54aa94743bebcd
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Train logistic regression model using scikit learn from CSV
|
||||
type: ScikitLearnPickleModel
|
||||
annotations:
|
||||
editor.position: '{"x":1030,"y":880,"width":180,"height":70}'
|
||||
Upload Tensorflow model to Google Cloud Vertex AI:
|
||||
componentRef:
|
||||
digest: 2e45263ff640b1a688e359b6936e27a81b2407749a84f340af2aa5547e0cb92c
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: trained_model
|
||||
taskId: Train model using Keras on CSV
|
||||
type: TensorflowSavedModel
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":1010,"width":180,"height":54}'
|
||||
Upload PyTorch model archive to Google Cloud Vertex AI:
|
||||
componentRef:
|
||||
digest: 4450212fae7b9001482aca7eb78b28413c205506eccf08a04e7754a8dfa99004
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
model_archive:
|
||||
taskOutput:
|
||||
outputName: Model archive
|
||||
taskId: Create PyTorch Model Archive with base handler
|
||||
type: PyTorchModelArchive
|
||||
annotations:
|
||||
editor.position: '{"x":380,"y":1010,"width":180,"height":70}'
|
||||
Upload XGBoost model to Google Cloud Vertex AI:
|
||||
componentRef:
|
||||
digest: 5a5a273c403670743820986c03a4175b7cb4595a556524fefcce403656286977
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Train XGBoost model on CSV
|
||||
type: XGBoostModel
|
||||
annotations:
|
||||
editor.position: '{"x":720,"y":1010,"width":180,"height":54}'
|
||||
outputValues: {}
|
||||
+224
@@ -0,0 +1,224 @@
|
||||
# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
|
||||
from kfp import components
|
||||
|
||||
# %% Loading components
|
||||
download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml")
|
||||
fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
|
||||
binarize_column_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/1e2558325f4c708aca75827c8acc13d230ee7e9f/components/pandas/Binarize_column/in_CSV_format/component.yaml")
|
||||
split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
|
||||
|
||||
# TensorFlow
|
||||
create_fully_connected_tensorflow_network_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/9ca0f9eecf5f896f65b8538bbd809747052617d1/components/tensorflow/Create_fully_connected_network/component.yaml")
|
||||
train_model_using_Keras_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c504a4010348c50eaaf6d4337586ccc008f4dcef/components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml")
|
||||
predict_with_TensorFlow_model_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/59c759ce6f543184e30db6817d2a703879bc0f39/components/tensorflow/Predict/on_CSV/component.yaml")
|
||||
upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
|
||||
# PyTorch
|
||||
create_fully_connected_pytorch_network_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/1a2ef3eeb77bc278f33cad0dd29008ea2431e191/components/PyTorch/Create_fully_connected_network/component.yaml")
|
||||
train_pytorch_model_from_csv_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml")
|
||||
create_pytorch_model_archive_with_base_handler_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/46d51383e6554b7f3ab4fd8cf614d8c2b422fb22/components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml")
|
||||
upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
|
||||
# XGBoost
|
||||
train_XGBoost_model_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/58d3a47f904f32a64af8403330ba7e2134cae46d/components/XGBoost/Train/component.yaml")
|
||||
xgboost_predict_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/4694ec97baccf59284c2a1db4aa2250c22291eab/components/XGBoost/Predict/component.yaml")
|
||||
upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
|
||||
# Scikit-learn
|
||||
#train_linear_regression_model_using_scikit_learn_from_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/f807e02b54d4886c65a05f40848fd51c72407f40/components/ML_frameworks/Scikit_learn/Train_linear_regression_model/from_CSV/component.yaml")
|
||||
train_logistic_regression_model_using_scikit_learn_from_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/cb44b75c9c062fcc40c2b905b2024b4493dbc62b/components/ML_frameworks/Scikit_learn/Train_logistic_regression_model/from_CSV/component.yaml")
|
||||
upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
|
||||
# Vertex AI
|
||||
deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
|
||||
# %% Pipeline definition
|
||||
def train_tabular_classification_model_using_all_frameworks_pipeline():
|
||||
dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
|
||||
feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"] # Excluded "trip_total"
|
||||
label_column = "tips"
|
||||
training_set_fraction = 0.8
|
||||
# Deploying the model might incur additional costs over time
|
||||
deploy_model = False
|
||||
|
||||
classification_label_column = "class"
|
||||
all_columns = [label_column] + feature_columns
|
||||
|
||||
dataset = download_from_gcs_op(
|
||||
gcs_path=dataset_gcs_uri
|
||||
).outputs["Data"]
|
||||
|
||||
dataset = select_columns_using_Pandas_on_CSV_data_op(
|
||||
table=dataset,
|
||||
column_names=all_columns,
|
||||
).outputs["transformed_table"]
|
||||
|
||||
dataset = fill_all_missing_values_using_Pandas_on_CSV_data_op(
|
||||
table=dataset,
|
||||
replacement_value="0",
|
||||
# # Optional:
|
||||
# column_names=None, # =[...]
|
||||
).outputs["transformed_table"]
|
||||
|
||||
classification_dataset = binarize_column_using_Pandas_on_CSV_data_op(
|
||||
table=dataset,
|
||||
column_name=label_column,
|
||||
predicate=" > 0",
|
||||
new_column_name=classification_label_column,
|
||||
).outputs["transformed_table"]
|
||||
|
||||
split_task = split_rows_into_subsets_op(
|
||||
table=classification_dataset,
|
||||
fraction_1=training_set_fraction,
|
||||
)
|
||||
classification_training_data = split_task.outputs["split_1"]
|
||||
classification_testing_data = split_task.outputs["split_2"]
|
||||
|
||||
# TensorFlow
|
||||
tensorflow_network = create_fully_connected_tensorflow_network_op(
|
||||
input_size=len(feature_columns),
|
||||
# Optional:
|
||||
hidden_layer_sizes=[10],
|
||||
activation_name="elu",
|
||||
output_activation_name="sigmoid",
|
||||
# output_size=1,
|
||||
).outputs["model"]
|
||||
|
||||
tensorflow_model = train_model_using_Keras_on_CSV_op(
|
||||
training_data=classification_training_data,
|
||||
model=tensorflow_network,
|
||||
label_column_name=classification_label_column,
|
||||
# Optional:
|
||||
loss_function_name="binary_crossentropy",
|
||||
number_of_epochs=10,
|
||||
#learning_rate=0.1,
|
||||
#optimizer_name="Adadelta",
|
||||
#optimizer_parameters={},
|
||||
#batch_size=32,
|
||||
#metric_names=["mean_absolute_error"],
|
||||
#random_seed=0,
|
||||
).outputs["trained_model"]
|
||||
|
||||
tensorflow_predictions = predict_with_TensorFlow_model_on_CSV_data_op(
|
||||
dataset=classification_testing_data,
|
||||
model=tensorflow_model,
|
||||
# label_column_name needs to be set when doing prediction on a dataset that has labels
|
||||
label_column_name=classification_label_column,
|
||||
# Optional:
|
||||
# batch_size=1000,
|
||||
).outputs["predictions"]
|
||||
|
||||
tensorflow_vertex_model_name = upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op(
|
||||
model=tensorflow_model,
|
||||
).outputs["model_name"]
|
||||
|
||||
# Deploying the model might incur additional costs over time
|
||||
if deploy_model:
|
||||
tensorflow_vertex_endpoint_name = deploy_model_to_endpoint_op(
|
||||
model_name=tensorflow_vertex_model_name,
|
||||
).outputs["endpoint_name"]
|
||||
|
||||
# PyTorch
|
||||
pytorch_network = create_fully_connected_pytorch_network_op(
|
||||
input_size=len(feature_columns),
|
||||
# Optional:
|
||||
hidden_layer_sizes=[10],
|
||||
activation_name="elu",
|
||||
output_activation_name="sigmoid",
|
||||
# output_size=1,
|
||||
).outputs["model"]
|
||||
|
||||
pytorch_model = train_pytorch_model_from_csv_op(
|
||||
model=pytorch_network,
|
||||
training_data=classification_training_data,
|
||||
label_column_name=classification_label_column,
|
||||
loss_function_name="binary_cross_entropy",
|
||||
# Optional:
|
||||
#number_of_epochs=1,
|
||||
#learning_rate=0.1,
|
||||
#optimizer_name="Adadelta",
|
||||
#optimizer_parameters={},
|
||||
#batch_size=32,
|
||||
#batch_log_interval=100,
|
||||
#random_seed=0,
|
||||
).outputs["trained_model"]
|
||||
|
||||
pytorch_model_archive = create_pytorch_model_archive_with_base_handler_op(
|
||||
model=pytorch_model,
|
||||
# Optional:
|
||||
# model_name="model",
|
||||
# model_version="1.0",
|
||||
).outputs["Model archive"]
|
||||
|
||||
pytorch_vertex_model_name = upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op(
|
||||
model_archive=pytorch_model_archive,
|
||||
).outputs["model_name"]
|
||||
|
||||
# Deploying the model might incur additional costs over time
|
||||
if deploy_model:
|
||||
pytorch_vertex_endpoint_name = deploy_model_to_endpoint_op(
|
||||
model_name=pytorch_vertex_model_name,
|
||||
).outputs["endpoint_name"]
|
||||
|
||||
# XGBoost
|
||||
xgboost_model = train_XGBoost_model_on_CSV_op(
|
||||
training_data=classification_training_data,
|
||||
label_column_name=classification_label_column,
|
||||
objective="binary:logistic",
|
||||
# Optional:
|
||||
#starting_model=None,
|
||||
#num_iterations=10,
|
||||
#booster_params={},
|
||||
#booster="gbtree",
|
||||
#learning_rate=0.3,
|
||||
#min_split_loss=0,
|
||||
#max_depth=6,
|
||||
).outputs["model"]
|
||||
|
||||
# Predicting on the testing data
|
||||
xgboost_predictions = xgboost_predict_on_CSV_op(
|
||||
data=classification_testing_data,
|
||||
model=xgboost_model,
|
||||
# label_column needs to be set when doing prediction on a dataset that has labels
|
||||
label_column_name=classification_label_column,
|
||||
).outputs["predictions"]
|
||||
|
||||
xgboost_vertex_model_name = upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op(
|
||||
model=xgboost_model,
|
||||
).outputs["model_name"]
|
||||
|
||||
# Deploying the model might incur additional costs over time
|
||||
if deploy_model:
|
||||
xgboost_vertex_endpoint_name = deploy_model_to_endpoint_op(
|
||||
model_name=xgboost_vertex_model_name,
|
||||
).outputs["endpoint_name"]
|
||||
|
||||
# Scikit-learn
|
||||
sklearn_model = train_logistic_regression_model_using_scikit_learn_from_CSV_op(
|
||||
dataset=classification_training_data,
|
||||
label_column_name=classification_label_column,
|
||||
# Optional:
|
||||
#penalty="l2",
|
||||
#solver="lbfgs",
|
||||
#max_iterations=100,
|
||||
#multi_class_mode="auto",
|
||||
#random_seed=0,
|
||||
).outputs["model"]
|
||||
|
||||
sklearn_vertex_model_name = upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op(
|
||||
model=sklearn_model,
|
||||
).outputs["model_name"]
|
||||
|
||||
# Deploying the model might incur additional costs over time
|
||||
if deploy_model:
|
||||
sklearn_vertex_endpoint_name = deploy_model_to_endpoint_op(
|
||||
model_name=sklearn_vertex_model_name,
|
||||
).outputs["endpoint_name"]
|
||||
|
||||
pipeline_func=train_tabular_classification_model_using_all_frameworks_pipeline
|
||||
|
||||
# %% Pipeline submission
|
||||
if __name__ == '__main__':
|
||||
from google.cloud import aiplatform
|
||||
aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()
|
||||
+68
@@ -0,0 +1,68 @@
|
||||
name: Train tabular regression linear model using Scikit learn pipeline
|
||||
metadata:
|
||||
annotations:
|
||||
author: Alexey Volkov <alexey.volkov@ark-kun.com>
|
||||
canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_regression_linear_model_using_Scikit_learn_and_import_to_Vertex_AI/pipeline.component.yaml
|
||||
sdk: https://cloud-pipelines.net/pipeline-editor/
|
||||
implementation:
|
||||
graph:
|
||||
tasks:
|
||||
Download from GCS:
|
||||
componentRef:
|
||||
digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":40,"width":180,"height":40}'
|
||||
Select columns using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: Data
|
||||
taskId: Download from GCS
|
||||
column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":140,"width":180,"height":54}'
|
||||
Fill all missing values using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Select columns using Pandas on CSV data
|
||||
type: CSV
|
||||
replacement_value: '0'
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":250,"width":180,"height":54}'
|
||||
Train linear regression model using scikit learn from CSV:
|
||||
componentRef:
|
||||
digest: c7fe7912ab0d1fb45d201d452e9ce6be5544e7d8c6d229db7a4b931ff58560f3
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/f807e02b54d4886c65a05f40848fd51c72407f40/components/ML_frameworks/Scikit_learn/Train_linear_regression_model/from_CSV/component.yaml
|
||||
arguments:
|
||||
dataset:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Fill all missing values using Pandas on CSV data
|
||||
type: CSV
|
||||
label_column_name: tips
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":360,"width":180,"height":54}'
|
||||
Upload Scikit learn pickle model to Google Cloud Vertex AI:
|
||||
componentRef:
|
||||
digest: 81c91c8d7d21ec97e0872f669d68bd89edea87279d703685db54aa94743bebcd
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Train linear regression model using scikit learn from CSV
|
||||
type: ScikitLearnPickleModel
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":490,"width":180,"height":70}'
|
||||
outputValues: {}
|
||||
+57
@@ -0,0 +1,57 @@
|
||||
# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
|
||||
from kfp import components
|
||||
|
||||
# %% Loading components
|
||||
download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml")
|
||||
fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
|
||||
train_linear_regression_model_using_scikit_learn_from_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/f807e02b54d4886c65a05f40848fd51c72407f40/components/ML_frameworks/Scikit_learn/Train_linear_regression_model/from_CSV/component.yaml")
|
||||
upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
|
||||
# %% Pipeline definition
|
||||
def train_tabular_regression_linear_model_using_Scikit_learn_pipeline():
|
||||
dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
|
||||
feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"] # Excluded "trip_total"
|
||||
label_column = "tips"
|
||||
all_columns = [label_column] + feature_columns
|
||||
# Deploying the model might incur additional costs over time
|
||||
deploy_model = False
|
||||
|
||||
training_data = download_from_gcs_op(
|
||||
gcs_path=dataset_gcs_uri
|
||||
).outputs["Data"]
|
||||
|
||||
training_data = select_columns_using_Pandas_on_CSV_data_op(
|
||||
table=training_data,
|
||||
column_names=all_columns,
|
||||
).outputs["transformed_table"]
|
||||
|
||||
# Cleaning the NaN values.
|
||||
training_data = fill_all_missing_values_using_Pandas_on_CSV_data_op(
|
||||
table=training_data,
|
||||
replacement_value="0",
|
||||
#replacement_type_name="float",
|
||||
).outputs["transformed_table"]
|
||||
|
||||
model = train_linear_regression_model_using_scikit_learn_from_CSV_op(
|
||||
dataset=training_data,
|
||||
label_column_name=label_column,
|
||||
).outputs["model"]
|
||||
|
||||
vertex_model_name = upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op(
|
||||
model=model,
|
||||
).outputs["model_name"]
|
||||
|
||||
# Deploying the model might incur additional costs over time
|
||||
if deploy_model:
|
||||
sklearn_vertex_endpoint_name = deploy_model_to_endpoint_op(
|
||||
model_name=vertex_model_name,
|
||||
).outputs["endpoint_name"]
|
||||
|
||||
pipeline_func = train_tabular_regression_linear_model_using_Scikit_learn_pipeline
|
||||
|
||||
# %% Pipeline submission
|
||||
if __name__ == '__main__':
|
||||
from google.cloud import aiplatform
|
||||
aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()
|
||||
+97
@@ -0,0 +1,97 @@
|
||||
name: Train tabular regression model using PyTorch pipeline
|
||||
metadata:
|
||||
annotations:
|
||||
author: Alexey Volkov <alexey.volkov@ark-kun.com>
|
||||
canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_regression_model_using_PyTorch_and_import_to_Vertex_AI/pipeline.component.yaml
|
||||
sdk: https://cloud-pipelines.net/pipeline-editor/
|
||||
implementation:
|
||||
graph:
|
||||
tasks:
|
||||
Download from GCS:
|
||||
componentRef:
|
||||
digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
|
||||
annotations:
|
||||
editor.position: '{"x":240,"y":40,"width":180,"height":40}'
|
||||
Select columns using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: Data
|
||||
taskId: Download from GCS
|
||||
column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
|
||||
annotations:
|
||||
editor.position: '{"x":240,"y":130,"width":180,"height":54}'
|
||||
Create fully connected pytorch network:
|
||||
componentRef:
|
||||
digest: d03d8248fd358a0275ec33568ee7dd7dce576cc112b09dfafe2651e4d97e04a9
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1a2ef3eeb77bc278f33cad0dd29008ea2431e191/components/PyTorch/Create_fully_connected_network/component.yaml
|
||||
arguments:
|
||||
input_size: '7'
|
||||
hidden_layer_sizes: '[10]'
|
||||
activation_name: elu
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":240,"width":180,"height":54}'
|
||||
Fill all missing values using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Select columns using Pandas on CSV data
|
||||
type: CSV
|
||||
replacement_value: '0'
|
||||
annotations:
|
||||
editor.position: '{"x":240,"y":240,"width":180,"height":54}'
|
||||
Train pytorch model from csv:
|
||||
componentRef:
|
||||
digest: 40f3185eb61e9727f41a4e0c05dd3d3b44bd802aa0f378cfc31756560033949a
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml
|
||||
arguments:
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Create fully connected pytorch network
|
||||
type: PyTorchScriptModule
|
||||
training_data:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Fill all missing values using Pandas on CSV data
|
||||
type: CSV
|
||||
label_column_name: tips
|
||||
annotations:
|
||||
editor.position: '{"x":240,"y":380,"width":180,"height":40}'
|
||||
Create PyTorch Model Archive with base handler:
|
||||
componentRef:
|
||||
digest: 8298b5ee1b0f0879f893add4cf352c8dec7cf9e21bb9db134c91a2d046cdb0ec
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/46d51383e6554b7f3ab4fd8cf614d8c2b422fb22/components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml
|
||||
arguments:
|
||||
Model:
|
||||
taskOutput:
|
||||
outputName: trained_model
|
||||
taskId: Train pytorch model from csv
|
||||
type: PyTorchScriptModule
|
||||
Model name: model
|
||||
Model version: '1.0'
|
||||
annotations:
|
||||
editor.position: '{"x":240,"y":500,"width":180,"height":54}'
|
||||
Upload PyTorch model archive to Google Cloud Vertex AI:
|
||||
componentRef:
|
||||
digest: 4450212fae7b9001482aca7eb78b28413c205506eccf08a04e7754a8dfa99004
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
model_archive:
|
||||
taskOutput:
|
||||
outputName: Model archive
|
||||
taskId: Create PyTorch Model Archive with base handler
|
||||
type: PyTorchModelArchive
|
||||
annotations:
|
||||
editor.position: '{"x":240,"y":630,"width":180,"height":70}'
|
||||
outputValues: {}
|
||||
+85
@@ -0,0 +1,85 @@
|
||||
# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
|
||||
from kfp import components
|
||||
|
||||
# %% Loading components
|
||||
download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml")
|
||||
fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
|
||||
create_fully_connected_pytorch_network_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/1a2ef3eeb77bc278f33cad0dd29008ea2431e191/components/PyTorch/Create_fully_connected_network/component.yaml")
|
||||
train_pytorch_model_from_csv_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml")
|
||||
create_pytorch_model_archive_with_base_handler_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/46d51383e6554b7f3ab4fd8cf614d8c2b422fb22/components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml")
|
||||
upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
|
||||
# %% Pipeline definition
|
||||
def train_tabular_regression_model_using_PyTorch_pipeline():
|
||||
dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
|
||||
feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"] # Excluded "trip_total"
|
||||
label_column = "tips"
|
||||
all_columns = [label_column] + feature_columns
|
||||
# Deploying the model might incur additional costs over time
|
||||
deploy_model = False
|
||||
|
||||
training_data = download_from_gcs_op(
|
||||
gcs_path=dataset_gcs_uri
|
||||
).outputs["Data"]
|
||||
|
||||
training_data = select_columns_using_Pandas_on_CSV_data_op(
|
||||
table=training_data,
|
||||
column_names=all_columns,
|
||||
).outputs["transformed_table"]
|
||||
|
||||
# Cleaning the NaN values.
|
||||
training_data = fill_all_missing_values_using_Pandas_on_CSV_data_op(
|
||||
table=training_data,
|
||||
replacement_value="0",
|
||||
#replacement_type_name="float",
|
||||
).outputs["transformed_table"]
|
||||
|
||||
network = create_fully_connected_pytorch_network_op(
|
||||
input_size=len(feature_columns),
|
||||
# Optional:
|
||||
hidden_layer_sizes=[10],
|
||||
activation_name="elu",
|
||||
# output_activation_name=None,
|
||||
# output_size=1,
|
||||
).outputs["model"]
|
||||
|
||||
model = train_pytorch_model_from_csv_op(
|
||||
model=network,
|
||||
training_data=training_data,
|
||||
label_column_name=label_column,
|
||||
# Optional:
|
||||
#loss_function_name="mse_loss",
|
||||
#number_of_epochs=1,
|
||||
#learning_rate=0.1,
|
||||
#optimizer_name="Adadelta",
|
||||
#optimizer_parameters={},
|
||||
#batch_size=32,
|
||||
#batch_log_interval=100,
|
||||
#random_seed=0,
|
||||
).outputs["trained_model"]
|
||||
|
||||
model_archive = create_pytorch_model_archive_with_base_handler_op(
|
||||
model=model,
|
||||
# Optional:
|
||||
# model_name="model",
|
||||
# model_version="1.0",
|
||||
).outputs["Model archive"]
|
||||
|
||||
vertex_model_name = upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op(
|
||||
model_archive=model_archive,
|
||||
).outputs["model_name"]
|
||||
|
||||
# Deploying the model might incur additional costs over time
|
||||
if deploy_model:
|
||||
vertex_endpoint_name = deploy_model_to_endpoint_op(
|
||||
model_name=vertex_model_name,
|
||||
).outputs["endpoint_name"]
|
||||
|
||||
pipeline_func=train_tabular_regression_model_using_PyTorch_pipeline
|
||||
|
||||
# %% Pipeline submission
|
||||
if __name__ == '__main__':
|
||||
from google.cloud import aiplatform
|
||||
aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()
|
||||
+116
@@ -0,0 +1,116 @@
|
||||
name: Train tabular regression model using Tensorflow pipeline
|
||||
metadata:
|
||||
annotations:
|
||||
author: Alexey Volkov <alexey.volkov@ark-kun.com>
|
||||
canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_regression_model_using_TensorFlow_and_import_to_Vertex_AI/pipeline.component.yaml
|
||||
sdk: https://cloud-pipelines.net/pipeline-editor/
|
||||
implementation:
|
||||
graph:
|
||||
tasks:
|
||||
Download from GCS:
|
||||
componentRef:
|
||||
digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":40,"width":180,"height":40}'
|
||||
Select columns using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: Data
|
||||
taskId: Download from GCS
|
||||
column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":140,"width":180,"height":54}'
|
||||
Fill all missing values using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Select columns using Pandas on CSV data
|
||||
type: CSV
|
||||
replacement_value: '0'
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":250,"width":180,"height":54}'
|
||||
Split rows into subsets:
|
||||
componentRef:
|
||||
digest: a609c3c9196484290f24a1174955f95b27f07a7b458aa5cb8cde28866cb2cb46
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Fill all missing values using Pandas on CSV data
|
||||
type: CSV
|
||||
fraction_1: '0.8'
|
||||
annotations:
|
||||
editor.position: '{"x":170,"y":380,"width":180,"height":40}'
|
||||
Create fully connected tensorflow network:
|
||||
componentRef:
|
||||
digest: bfcafbc5ce711b1f69cabf1338212d10d50136a73db9f9f7c984de7b80b4bfb0
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/9ca0f9eecf5f896f65b8538bbd809747052617d1/components/tensorflow/Create_fully_connected_network/component.yaml
|
||||
arguments:
|
||||
input_size: '7'
|
||||
hidden_layer_sizes: '[10]'
|
||||
activation_name: elu
|
||||
annotations:
|
||||
editor.position: '{"x":370,"y":380,"width":180,"height":54}'
|
||||
Train model using Keras on CSV:
|
||||
componentRef:
|
||||
digest: 42ae60c889034dbad74815653e95b4f7d576b5f47f803173e8679c7b54984609
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c504a4010348c50eaaf6d4337586ccc008f4dcef/components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml
|
||||
arguments:
|
||||
training_data:
|
||||
taskOutput:
|
||||
outputName: split_1
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Create fully connected tensorflow network
|
||||
type: TensorflowSavedModel
|
||||
label_column_name: tips
|
||||
number_of_epochs: '10'
|
||||
metric_names: '["mean_absolute_error"]'
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":500,"width":180,"height":54}'
|
||||
Upload Tensorflow model to Google Cloud Vertex AI:
|
||||
componentRef:
|
||||
digest: 2e45263ff640b1a688e359b6936e27a81b2407749a84f340af2aa5547e0cb92c
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: trained_model
|
||||
taskId: Train model using Keras on CSV
|
||||
type: TensorflowSavedModel
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":630,"width":180,"height":54}'
|
||||
Predict with TensorFlow model on CSV data:
|
||||
componentRef:
|
||||
digest: 921bb1563e93a78233b8acceab87055b9154ccf5595d056028cf0396ca224cd4
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/59c759ce6f543184e30db6817d2a703879bc0f39/components/tensorflow/Predict/on_CSV/component.yaml
|
||||
arguments:
|
||||
dataset:
|
||||
taskOutput:
|
||||
outputName: split_2
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: trained_model
|
||||
taskId: Train model using Keras on CSV
|
||||
type: TensorflowSavedModel
|
||||
label_column_name: tips
|
||||
annotations:
|
||||
editor.position: '{"x":240,"y":630,"width":180,"height":54}'
|
||||
outputValues: {}
|
||||
+97
@@ -0,0 +1,97 @@
|
||||
# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
|
||||
from kfp import components
|
||||
|
||||
# %% Loading components
|
||||
download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml")
|
||||
fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
|
||||
split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
|
||||
create_fully_connected_tensorflow_network_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/9ca0f9eecf5f896f65b8538bbd809747052617d1/components/tensorflow/Create_fully_connected_network/component.yaml")
|
||||
train_model_using_Keras_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c504a4010348c50eaaf6d4337586ccc008f4dcef/components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml")
|
||||
predict_with_TensorFlow_model_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/59c759ce6f543184e30db6817d2a703879bc0f39/components/tensorflow/Predict/on_CSV/component.yaml")
|
||||
upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
|
||||
# %% Pipeline definition
|
||||
def train_tabular_regression_model_using_Tensorflow_pipeline():
|
||||
dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
|
||||
feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"] # Excluded "trip_total"
|
||||
label_column = "tips"
|
||||
training_set_fraction = 0.8
|
||||
# Deploying the model might incur additional costs over time
|
||||
deploy_model = False
|
||||
|
||||
all_columns = [label_column] + feature_columns
|
||||
|
||||
dataset = download_from_gcs_op(
|
||||
gcs_path=dataset_gcs_uri
|
||||
).outputs["Data"]
|
||||
|
||||
dataset = select_columns_using_Pandas_on_CSV_data_op(
|
||||
table=dataset,
|
||||
column_names=all_columns,
|
||||
).outputs["transformed_table"]
|
||||
|
||||
dataset = fill_all_missing_values_using_Pandas_on_CSV_data_op(
|
||||
table=dataset,
|
||||
replacement_value="0",
|
||||
# # Optional:
|
||||
# column_names=None, # =[...]
|
||||
).outputs["transformed_table"]
|
||||
|
||||
split_task = split_rows_into_subsets_op(
|
||||
table=dataset,
|
||||
fraction_1=training_set_fraction,
|
||||
)
|
||||
training_data = split_task.outputs["split_1"]
|
||||
testing_data = split_task.outputs["split_2"]
|
||||
|
||||
network = create_fully_connected_tensorflow_network_op(
|
||||
input_size=len(feature_columns),
|
||||
# Optional:
|
||||
hidden_layer_sizes=[10],
|
||||
activation_name="elu",
|
||||
# output_activation_name=None,
|
||||
# output_size=1,
|
||||
).outputs["model"]
|
||||
|
||||
model = train_model_using_Keras_on_CSV_op(
|
||||
training_data=training_data,
|
||||
model=network,
|
||||
label_column_name=label_column,
|
||||
# Optional:
|
||||
#loss_function_name="mean_squared_error",
|
||||
number_of_epochs=10,
|
||||
#learning_rate=0.1,
|
||||
#optimizer_name="Adadelta",
|
||||
#optimizer_parameters={},
|
||||
#batch_size=32,
|
||||
metric_names=["mean_absolute_error"],
|
||||
#random_seed=0,
|
||||
).outputs["trained_model"]
|
||||
|
||||
predictions = predict_with_TensorFlow_model_on_CSV_data_op(
|
||||
dataset=testing_data,
|
||||
model=model,
|
||||
# label_column_name needs to be set when doing prediction on a dataset that has labels
|
||||
label_column_name=label_column,
|
||||
# Optional:
|
||||
# batch_size=1000,
|
||||
).outputs["predictions"]
|
||||
|
||||
vertex_model_name = upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op(
|
||||
model=model,
|
||||
).outputs["model_name"]
|
||||
|
||||
# Deploying the model might incur additional costs over time
|
||||
if deploy_model:
|
||||
vertex_endpoint_name = deploy_model_to_endpoint_op(
|
||||
model_name=vertex_model_name,
|
||||
).outputs["endpoint_name"]
|
||||
|
||||
pipeline_func=train_tabular_regression_model_using_Tensorflow_pipeline
|
||||
|
||||
# %% Pipeline submission
|
||||
if __name__ == '__main__':
|
||||
from google.cloud import aiplatform
|
||||
aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()
|
||||
+99
@@ -0,0 +1,99 @@
|
||||
name: Train tabular regression model using XGBoost pipeline
|
||||
metadata:
|
||||
annotations:
|
||||
author: Alexey Volkov <alexey.volkov@ark-kun.com>
|
||||
canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_regression_model_using_XGBoost_and_import_to_Vertex_AI/pipeline.component.yaml
|
||||
sdk: https://cloud-pipelines.net/pipeline-editor/
|
||||
implementation:
|
||||
graph:
|
||||
tasks:
|
||||
Download from GCS:
|
||||
componentRef:
|
||||
digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":40,"width":180,"height":40}'
|
||||
Select columns using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: Data
|
||||
taskId: Download from GCS
|
||||
column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":140,"width":180,"height":54}'
|
||||
Fill all missing values using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Select columns using Pandas on CSV data
|
||||
type: CSV
|
||||
replacement_value: '0'
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":250,"width":180,"height":54}'
|
||||
Split rows into subsets:
|
||||
componentRef:
|
||||
digest: a609c3c9196484290f24a1174955f95b27f07a7b458aa5cb8cde28866cb2cb46
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Fill all missing values using Pandas on CSV data
|
||||
type: CSV
|
||||
fraction_1: '0.8'
|
||||
annotations:
|
||||
editor.position: '{"x":170,"y":360,"width":180,"height":40}'
|
||||
Train XGBoost model on CSV:
|
||||
componentRef:
|
||||
digest: 538c5a01eb38deaf532d619f0bbeaff4efc550fe1f0f776fc06791097b68ceac
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/58d3a47f904f32a64af8403330ba7e2134cae46d/components/XGBoost/Train/component.yaml
|
||||
arguments:
|
||||
training_data:
|
||||
taskOutput:
|
||||
outputName: split_1
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
label_column_name: tips
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":480,"width":180,"height":40}'
|
||||
Upload XGBoost model to Google Cloud Vertex AI:
|
||||
componentRef:
|
||||
digest: 5a5a273c403670743820986c03a4175b7cb4595a556524fefcce403656286977
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Train XGBoost model on CSV
|
||||
type: XGBoostModel
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":600,"width":180,"height":54}'
|
||||
Xgboost predict on CSV:
|
||||
componentRef:
|
||||
digest: 0876233a0c7306fefec188bd70f059b46d1fb5aa57be231799570e3bbbdd0d95
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/4694ec97baccf59284c2a1db4aa2250c22291eab/components/XGBoost/Predict/component.yaml
|
||||
arguments:
|
||||
data:
|
||||
taskOutput:
|
||||
outputName: split_2
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Train XGBoost model on CSV
|
||||
type: XGBoostModel
|
||||
label_column_name: tips
|
||||
annotations:
|
||||
editor.position: '{"x":240,"y":600,"width":180,"height":40}'
|
||||
outputValues: {}
|
||||
+85
@@ -0,0 +1,85 @@
|
||||
# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
|
||||
from kfp import components
|
||||
|
||||
# %% Loading components
|
||||
download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml")
|
||||
fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
|
||||
split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
|
||||
train_XGBoost_model_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/58d3a47f904f32a64af8403330ba7e2134cae46d/components/XGBoost/Train/component.yaml")
|
||||
xgboost_predict_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/4694ec97baccf59284c2a1db4aa2250c22291eab/components/XGBoost/Predict/component.yaml")
|
||||
upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
|
||||
# %% Pipeline definition
|
||||
def train_tabular_regression_model_using_XGBoost_pipeline():
|
||||
dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
|
||||
feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"] # Excluded "trip_total"
|
||||
label_column = "tips"
|
||||
training_set_fraction = 0.8
|
||||
# Deploying the model might incur additional costs over time
|
||||
deploy_model = False
|
||||
|
||||
all_columns = [label_column] + feature_columns
|
||||
|
||||
dataset = download_from_gcs_op(
|
||||
gcs_path=dataset_gcs_uri
|
||||
).outputs["Data"]
|
||||
|
||||
dataset = select_columns_using_Pandas_on_CSV_data_op(
|
||||
table=dataset,
|
||||
column_names=all_columns,
|
||||
).outputs["transformed_table"]
|
||||
|
||||
dataset = fill_all_missing_values_using_Pandas_on_CSV_data_op(
|
||||
table=dataset,
|
||||
replacement_value="0",
|
||||
# # Optional:
|
||||
# column_names=None, # =[...]
|
||||
).outputs["transformed_table"]
|
||||
|
||||
split_task = split_rows_into_subsets_op(
|
||||
table=dataset,
|
||||
fraction_1=training_set_fraction,
|
||||
)
|
||||
training_data = split_task.outputs["split_1"]
|
||||
testing_data = split_task.outputs["split_2"]
|
||||
|
||||
model = train_XGBoost_model_on_CSV_op(
|
||||
training_data=training_data,
|
||||
label_column_name=label_column,
|
||||
# Optional:
|
||||
#starting_model=None,
|
||||
#num_iterations=10,
|
||||
#booster_params={},
|
||||
#objective="reg:squarederror",
|
||||
#booster="gbtree",
|
||||
#learning_rate=0.3,
|
||||
#min_split_loss=0,
|
||||
#max_depth=6,
|
||||
).outputs["model"]
|
||||
|
||||
# Predicting on the testing data
|
||||
predictions = xgboost_predict_on_CSV_op(
|
||||
data=testing_data,
|
||||
model=model,
|
||||
# label_column needs to be set when doing prediction on a dataset that has labels
|
||||
label_column_name=label_column,
|
||||
).outputs["predictions"]
|
||||
|
||||
vertex_model_name = upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op(
|
||||
model=model,
|
||||
).outputs["model_name"]
|
||||
|
||||
# Deploying the model might incur additional costs over time
|
||||
if deploy_model:
|
||||
vertex_endpoint_name = deploy_model_to_endpoint_op(
|
||||
model_name=vertex_model_name,
|
||||
).outputs["endpoint_name"]
|
||||
|
||||
pipeline_func = train_tabular_regression_model_using_XGBoost_pipeline
|
||||
|
||||
# %% Pipeline submission
|
||||
if __name__ == '__main__':
|
||||
from google.cloud import aiplatform
|
||||
aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()
|
||||
+238
@@ -0,0 +1,238 @@
|
||||
name: Train tabular regression model using all frameworks pipeline
|
||||
metadata:
|
||||
annotations:
|
||||
author: Alexey Volkov <alexey.volkov@ark-kun.com>
|
||||
canonical_location: https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/samples/Google_Cloud_Vertex_AI/Train_tabular_regression_model_using_all_frameworks_and_import_to_Vertex_AI/pipeline.component.yaml
|
||||
sdk: https://cloud-pipelines.net/pipeline-editor/
|
||||
implementation:
|
||||
graph:
|
||||
tasks:
|
||||
Download from GCS:
|
||||
componentRef:
|
||||
digest: 4175c9ff143cb8cc75d05451c0a0ebdf5a0d6d020816e29f5e9cefbb7d56f241
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
GCS path: gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv
|
||||
annotations:
|
||||
editor.position: '{"x":550,"y":40,"width":180,"height":40}'
|
||||
Select columns using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: 9b9500f461c1d04f1e48992de9138db14a6800f23649d73048673d5ea6dc56ad
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: Data
|
||||
taskId: Download from GCS
|
||||
column_names: '["tips", "trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]'
|
||||
annotations:
|
||||
editor.position: '{"x":550,"y":140,"width":180,"height":54}'
|
||||
Fill all missing values using Pandas on CSV data:
|
||||
componentRef:
|
||||
digest: a1b0c29a4615f2e3652aa5d31b9255fa15700e146627c755f8fc172f82e71af7
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Select columns using Pandas on CSV data
|
||||
type: CSV
|
||||
replacement_value: '0'
|
||||
annotations:
|
||||
editor.position: '{"x":550,"y":250,"width":180,"height":54}'
|
||||
Split rows into subsets:
|
||||
componentRef:
|
||||
digest: a609c3c9196484290f24a1174955f95b27f07a7b458aa5cb8cde28866cb2cb46
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml
|
||||
arguments:
|
||||
table:
|
||||
taskOutput:
|
||||
outputName: transformed_table
|
||||
taskId: Fill all missing values using Pandas on CSV data
|
||||
type: CSV
|
||||
fraction_1: '0.8'
|
||||
annotations:
|
||||
editor.position: '{"x":550,"y":360,"width":180,"height":40}'
|
||||
Create fully connected pytorch network:
|
||||
componentRef:
|
||||
digest: d03d8248fd358a0275ec33568ee7dd7dce576cc112b09dfafe2651e4d97e04a9
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/1a2ef3eeb77bc278f33cad0dd29008ea2431e191/components/PyTorch/Create_fully_connected_network/component.yaml
|
||||
arguments:
|
||||
input_size: '7'
|
||||
hidden_layer_sizes: '[10]'
|
||||
activation_name: elu
|
||||
annotations:
|
||||
editor.position: '{"x":380,"y":490,"width":180,"height":54}'
|
||||
Create fully connected tensorflow network:
|
||||
componentRef:
|
||||
digest: bfcafbc5ce711b1f69cabf1338212d10d50136a73db9f9f7c984de7b80b4bfb0
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/9ca0f9eecf5f896f65b8538bbd809747052617d1/components/tensorflow/Create_fully_connected_network/component.yaml
|
||||
arguments:
|
||||
input_size: '7'
|
||||
hidden_layer_sizes: '[10]'
|
||||
activation_name: elu
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":500,"width":180,"height":54}'
|
||||
Train model using Keras on CSV:
|
||||
componentRef:
|
||||
digest: 42ae60c889034dbad74815653e95b4f7d576b5f47f803173e8679c7b54984609
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c504a4010348c50eaaf6d4337586ccc008f4dcef/components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml
|
||||
arguments:
|
||||
training_data:
|
||||
taskOutput:
|
||||
outputName: split_1
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Create fully connected tensorflow network
|
||||
type: TensorflowSavedModel
|
||||
label_column_name: tips
|
||||
number_of_epochs: '10'
|
||||
metric_names: '["mean_absolute_error"]'
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":620,"width":180,"height":54}'
|
||||
Train pytorch model from csv:
|
||||
componentRef:
|
||||
digest: 40f3185eb61e9727f41a4e0c05dd3d3b44bd802aa0f378cfc31756560033949a
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml
|
||||
arguments:
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Create fully connected pytorch network
|
||||
type: PyTorchScriptModule
|
||||
training_data:
|
||||
taskOutput:
|
||||
outputName: split_1
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
label_column_name: tips
|
||||
annotations:
|
||||
editor.position: '{"x":380,"y":620,"width":180,"height":40}'
|
||||
Train XGBoost model on CSV:
|
||||
componentRef:
|
||||
digest: 538c5a01eb38deaf532d619f0bbeaff4efc550fe1f0f776fc06791097b68ceac
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/58d3a47f904f32a64af8403330ba7e2134cae46d/components/XGBoost/Train/component.yaml
|
||||
arguments:
|
||||
training_data:
|
||||
taskOutput:
|
||||
outputName: split_1
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
label_column_name: tips
|
||||
annotations:
|
||||
editor.position: '{"x":720,"y":620,"width":180,"height":40}'
|
||||
Train linear regression model using scikit learn from CSV:
|
||||
componentRef:
|
||||
digest: c7fe7912ab0d1fb45d201d452e9ce6be5544e7d8c6d229db7a4b931ff58560f3
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/f807e02b54d4886c65a05f40848fd51c72407f40/components/ML_frameworks/Scikit_learn/Train_linear_regression_model/from_CSV/component.yaml
|
||||
arguments:
|
||||
dataset:
|
||||
taskOutput:
|
||||
outputName: split_1
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
label_column_name: tips
|
||||
annotations:
|
||||
editor.position: '{"x":1030,"y":620,"width":180,"height":54}'
|
||||
Predict with TensorFlow model on CSV data:
|
||||
componentRef:
|
||||
digest: 921bb1563e93a78233b8acceab87055b9154ccf5595d056028cf0396ca224cd4
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/59c759ce6f543184e30db6817d2a703879bc0f39/components/tensorflow/Predict/on_CSV/component.yaml
|
||||
arguments:
|
||||
dataset:
|
||||
taskOutput:
|
||||
outputName: split_2
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: trained_model
|
||||
taskId: Train model using Keras on CSV
|
||||
type: TensorflowSavedModel
|
||||
label_column_name: tips
|
||||
annotations:
|
||||
editor.position: '{"x":160,"y":750,"width":180,"height":54}'
|
||||
Create PyTorch Model Archive with base handler:
|
||||
componentRef:
|
||||
digest: 8298b5ee1b0f0879f893add4cf352c8dec7cf9e21bb9db134c91a2d046cdb0ec
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/46d51383e6554b7f3ab4fd8cf614d8c2b422fb22/components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml
|
||||
arguments:
|
||||
Model:
|
||||
taskOutput:
|
||||
outputName: trained_model
|
||||
taskId: Train pytorch model from csv
|
||||
type: PyTorchScriptModule
|
||||
Model name: model
|
||||
Model version: '1.0'
|
||||
annotations:
|
||||
editor.position: '{"x":380,"y":750,"width":180,"height":54}'
|
||||
Xgboost predict on CSV:
|
||||
componentRef:
|
||||
digest: 0876233a0c7306fefec188bd70f059b46d1fb5aa57be231799570e3bbbdd0d95
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/4694ec97baccf59284c2a1db4aa2250c22291eab/components/XGBoost/Predict/component.yaml
|
||||
arguments:
|
||||
data:
|
||||
taskOutput:
|
||||
outputName: split_2
|
||||
taskId: Split rows into subsets
|
||||
type: CSV
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Train XGBoost model on CSV
|
||||
type: XGBoostModel
|
||||
label_column_name: tips
|
||||
annotations:
|
||||
editor.position: '{"x":810,"y":750,"width":180,"height":40}'
|
||||
Upload Scikit learn pickle model to Google Cloud Vertex AI:
|
||||
componentRef:
|
||||
digest: 81c91c8d7d21ec97e0872f669d68bd89edea87279d703685db54aa94743bebcd
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Train linear regression model using scikit learn from CSV
|
||||
type: ScikitLearnPickleModel
|
||||
annotations:
|
||||
editor.position: '{"x":1030,"y":750,"width":180,"height":70}'
|
||||
Upload Tensorflow model to Google Cloud Vertex AI:
|
||||
componentRef:
|
||||
digest: 2e45263ff640b1a688e359b6936e27a81b2407749a84f340af2aa5547e0cb92c
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: trained_model
|
||||
taskId: Train model using Keras on CSV
|
||||
type: TensorflowSavedModel
|
||||
annotations:
|
||||
editor.position: '{"x":40,"y":880,"width":180,"height":54}'
|
||||
Upload PyTorch model archive to Google Cloud Vertex AI:
|
||||
componentRef:
|
||||
digest: 4450212fae7b9001482aca7eb78b28413c205506eccf08a04e7754a8dfa99004
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
model_archive:
|
||||
taskOutput:
|
||||
outputName: Model archive
|
||||
taskId: Create PyTorch Model Archive with base handler
|
||||
type: PyTorchModelArchive
|
||||
annotations:
|
||||
editor.position: '{"x":380,"y":880,"width":180,"height":70}'
|
||||
Upload XGBoost model to Google Cloud Vertex AI:
|
||||
componentRef:
|
||||
digest: 5a5a273c403670743820986c03a4175b7cb4595a556524fefcce403656286977
|
||||
url: https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/workaround_for_buggy_KFPv2_compiler/component.yaml
|
||||
arguments:
|
||||
model:
|
||||
taskOutput:
|
||||
outputName: model
|
||||
taskId: Train XGBoost model on CSV
|
||||
type: XGBoostModel
|
||||
annotations:
|
||||
editor.position: '{"x":720,"y":880,"width":180,"height":54}'
|
||||
outputValues: {}
|
||||
+208
@@ -0,0 +1,208 @@
|
||||
# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
|
||||
from kfp import components
|
||||
|
||||
# %% Loading components
|
||||
download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/storage/download/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/8c78aae096806cff3bc331a40566f42f5c3e9d4b/components/pandas/Select_columns/in_CSV_format/component.yaml")
|
||||
fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/23405971f5f16a41b16c343129b893c52e4d1d48/components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
|
||||
split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/daae5a4abaa35e44501818b1534ed7827d7da073/components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
|
||||
|
||||
# TensorFlow
|
||||
create_fully_connected_tensorflow_network_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/9ca0f9eecf5f896f65b8538bbd809747052617d1/components/tensorflow/Create_fully_connected_network/component.yaml")
|
||||
train_model_using_Keras_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c504a4010348c50eaaf6d4337586ccc008f4dcef/components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml")
|
||||
predict_with_TensorFlow_model_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/59c759ce6f543184e30db6817d2a703879bc0f39/components/tensorflow/Predict/on_CSV/component.yaml")
|
||||
upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
|
||||
# PyTorch
|
||||
create_fully_connected_pytorch_network_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/1a2ef3eeb77bc278f33cad0dd29008ea2431e191/components/PyTorch/Create_fully_connected_network/component.yaml")
|
||||
train_pytorch_model_from_csv_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/d8c4cf5e6403bc65bcf8d606e6baf87e2528a3dc/components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml")
|
||||
create_pytorch_model_archive_with_base_handler_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/46d51383e6554b7f3ab4fd8cf614d8c2b422fb22/components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml")
|
||||
upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
|
||||
# XGBoost
|
||||
train_XGBoost_model_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/58d3a47f904f32a64af8403330ba7e2134cae46d/components/XGBoost/Train/component.yaml")
|
||||
xgboost_predict_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/4694ec97baccf59284c2a1db4aa2250c22291eab/components/XGBoost/Predict/component.yaml")
|
||||
upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
|
||||
# Scikit-learn
|
||||
train_linear_regression_model_using_scikit_learn_from_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/f807e02b54d4886c65a05f40848fd51c72407f40/components/ML_frameworks/Scikit_learn/Train_linear_regression_model/from_CSV/component.yaml")
|
||||
upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/c6a8b67d1ada2cc17665c99ff6b410df588bee28/components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
|
||||
# Vertex AI
|
||||
deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/Ark-kun/pipeline_components/27a5ea25e849c9e8c0cb6ed65518bc3ece259aaf/components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/workaround_for_buggy_KFPv2_compiler/component.yaml")
|
||||
|
||||
# %% Pipeline definition
|
||||
def train_tabular_regression_model_using_all_frameworks_pipeline():
|
||||
dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
|
||||
feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"] # Excluded "trip_total"
|
||||
label_column = "tips"
|
||||
training_set_fraction = 0.8
|
||||
# Deploying the model might incur additional costs over time
|
||||
deploy_model = False
|
||||
|
||||
all_columns = [label_column] + feature_columns
|
||||
|
||||
dataset = download_from_gcs_op(
|
||||
gcs_path=dataset_gcs_uri
|
||||
).outputs["Data"]
|
||||
|
||||
dataset = select_columns_using_Pandas_on_CSV_data_op(
|
||||
table=dataset,
|
||||
column_names=all_columns,
|
||||
).outputs["transformed_table"]
|
||||
|
||||
dataset = fill_all_missing_values_using_Pandas_on_CSV_data_op(
|
||||
table=dataset,
|
||||
replacement_value="0",
|
||||
# # Optional:
|
||||
# column_names=None, # =[...]
|
||||
).outputs["transformed_table"]
|
||||
|
||||
split_task = split_rows_into_subsets_op(
|
||||
table=dataset,
|
||||
fraction_1=training_set_fraction,
|
||||
)
|
||||
training_data = split_task.outputs["split_1"]
|
||||
testing_data = split_task.outputs["split_2"]
|
||||
|
||||
# TensorFlow
|
||||
tensorflow_network = create_fully_connected_tensorflow_network_op(
|
||||
input_size=len(feature_columns),
|
||||
# Optional:
|
||||
hidden_layer_sizes=[10],
|
||||
activation_name="elu",
|
||||
# output_activation_name=None,
|
||||
# output_size=1,
|
||||
).outputs["model"]
|
||||
|
||||
tensorflow_model = train_model_using_Keras_on_CSV_op(
|
||||
training_data=training_data,
|
||||
model=tensorflow_network,
|
||||
label_column_name=label_column,
|
||||
# Optional:
|
||||
#loss_function_name="mean_squared_error",
|
||||
number_of_epochs=10,
|
||||
#learning_rate=0.1,
|
||||
#optimizer_name="Adadelta",
|
||||
#optimizer_parameters={},
|
||||
#batch_size=32,
|
||||
metric_names=["mean_absolute_error"],
|
||||
#random_seed=0,
|
||||
).outputs["trained_model"]
|
||||
|
||||
tensorflow_predictions = predict_with_TensorFlow_model_on_CSV_data_op(
|
||||
dataset=testing_data,
|
||||
model=tensorflow_model,
|
||||
# label_column_name needs to be set when doing prediction on a dataset that has labels
|
||||
label_column_name=label_column,
|
||||
# Optional:
|
||||
# batch_size=1000,
|
||||
).outputs["predictions"]
|
||||
|
||||
tensorflow_vertex_model_name = upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op(
|
||||
model=tensorflow_model,
|
||||
).outputs["model_name"]
|
||||
|
||||
# Deploying the model might incur additional costs over time
|
||||
if deploy_model:
|
||||
tensorflow_vertex_endpoint_name = deploy_model_to_endpoint_op(
|
||||
model_name=tensorflow_vertex_model_name,
|
||||
).outputs["endpoint_name"]
|
||||
|
||||
# PyTorch
|
||||
pytorch_network = create_fully_connected_pytorch_network_op(
|
||||
input_size=len(feature_columns),
|
||||
# Optional:
|
||||
hidden_layer_sizes=[10],
|
||||
activation_name="elu",
|
||||
# output_activation_name=None,
|
||||
# output_size=1,
|
||||
).outputs["model"]
|
||||
|
||||
pytorch_model = train_pytorch_model_from_csv_op(
|
||||
model=pytorch_network,
|
||||
training_data=training_data,
|
||||
label_column_name=label_column,
|
||||
# Optional:
|
||||
#loss_function_name="mse_loss",
|
||||
#number_of_epochs=1,
|
||||
#learning_rate=0.1,
|
||||
#optimizer_name="Adadelta",
|
||||
#optimizer_parameters={},
|
||||
#batch_size=32,
|
||||
#batch_log_interval=100,
|
||||
#random_seed=0,
|
||||
).outputs["trained_model"]
|
||||
|
||||
pytorch_model_archive = create_pytorch_model_archive_with_base_handler_op(
|
||||
model=pytorch_model,
|
||||
# Optional:
|
||||
# model_name="model",
|
||||
# model_version="1.0",
|
||||
).outputs["Model archive"]
|
||||
|
||||
pytorch_vertex_model_name = upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op(
|
||||
model_archive=pytorch_model_archive,
|
||||
).outputs["model_name"]
|
||||
|
||||
# Deploying the model might incur additional costs over time
|
||||
if deploy_model:
|
||||
pytorch_vertex_endpoint_name = deploy_model_to_endpoint_op(
|
||||
model_name=pytorch_vertex_model_name,
|
||||
).outputs["endpoint_name"]
|
||||
|
||||
# XGBoost
|
||||
xgboost_model = train_XGBoost_model_on_CSV_op(
|
||||
training_data=training_data,
|
||||
label_column_name=label_column,
|
||||
# Optional:
|
||||
#starting_model=None,
|
||||
#num_iterations=10,
|
||||
#booster_params={},
|
||||
#objective="reg:squarederror",
|
||||
#booster="gbtree",
|
||||
#learning_rate=0.3,
|
||||
#min_split_loss=0,
|
||||
#max_depth=6,
|
||||
).outputs["model"]
|
||||
|
||||
# Predicting on the testing data
|
||||
xgboost_predictions = xgboost_predict_on_CSV_op(
|
||||
data=testing_data,
|
||||
model=xgboost_model,
|
||||
# label_column needs to be set when doing prediction on a dataset that has labels
|
||||
label_column_name=label_column,
|
||||
).outputs["predictions"]
|
||||
|
||||
xgboost_vertex_model_name = upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op(
|
||||
model=xgboost_model,
|
||||
).outputs["model_name"]
|
||||
|
||||
# Deploying the model might incur additional costs over time
|
||||
if deploy_model:
|
||||
xgboost_vertex_endpoint_name = deploy_model_to_endpoint_op(
|
||||
model_name=xgboost_vertex_model_name,
|
||||
).outputs["endpoint_name"]
|
||||
|
||||
# Scikit-learn
|
||||
sklearn_model = train_linear_regression_model_using_scikit_learn_from_CSV_op(
|
||||
dataset=training_data,
|
||||
label_column_name=label_column,
|
||||
).outputs["model"]
|
||||
|
||||
sklearn_vertex_model_name = upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op(
|
||||
model=sklearn_model,
|
||||
).outputs["model_name"]
|
||||
|
||||
# Deploying the model might incur additional costs over time
|
||||
if deploy_model:
|
||||
sklearn_vertex_endpoint_name = deploy_model_to_endpoint_op(
|
||||
model_name=sklearn_vertex_model_name,
|
||||
).outputs["endpoint_name"]
|
||||
|
||||
pipeline_func=train_tabular_regression_model_using_all_frameworks_pipeline
|
||||
|
||||
# %% Pipeline submission
|
||||
if __name__ == '__main__':
|
||||
from google.cloud import aiplatform
|
||||
aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -670,6 +670,56 @@
|
||||
"print(\"Train machine type\", DEPLOY_COMPUTE)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "8ac30ffa72b5"
|
||||
},
|
||||
"source": [
|
||||
"### Create the instance schema\n",
|
||||
"\n",
|
||||
"The instance.yaml file is used to validate the format of the input request before it goes to the model server. If you are sending correctly you don't need instance.yaml. If you get bad requests, the instance.yaml will give a more meaningful error msg than what you probably get from the model server.\n",
|
||||
"\n",
|
||||
"*The instance.yaml can also be reused as the input schema for Model Monitoring custom models.*\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"Learn more about [Predict schemas](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/PredictSchemata)."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "59becb56ad34"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"yaml = \"\"\"type: array\n",
|
||||
"properties:\n",
|
||||
" sepal_length:\n",
|
||||
" type: numeric\n",
|
||||
" sepal_width:\n",
|
||||
" type: numeric\n",
|
||||
" petal_length:\n",
|
||||
" type: numeric\n",
|
||||
" petal_width:\n",
|
||||
" type: numeric\n",
|
||||
"required:\n",
|
||||
" - sepal_length\n",
|
||||
" - sepal_width\n",
|
||||
" - petal_length\n",
|
||||
" - petal_width\n",
|
||||
"\"\"\"\n",
|
||||
"\n",
|
||||
"print(yaml)\n",
|
||||
"\n",
|
||||
"with open(\"instance.yaml\", \"w\") as f:\n",
|
||||
" f.write(yaml)\n",
|
||||
"\n",
|
||||
"! gsutil cp instance.yaml {BUCKET_URI}/instance.yaml"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
@@ -709,6 +759,7 @@
|
||||
" display_name=f\"xgboost_iris_{UUID}\",\n",
|
||||
" serving_container_image_uri=DEPLOY_IMAGE,\n",
|
||||
" artifact_uri=MODEL_ARTIFACTS,\n",
|
||||
" instance_schema_uri=f\"{BUCKET_URI}/instance.yaml\",\n",
|
||||
" is_default_version=True,\n",
|
||||
" version_aliases=[\"v1\", \"version1\"],\n",
|
||||
" version_description=\"This is the first version of the model\",\n",
|
||||
@@ -749,7 +800,7 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"endpoint = aiplatform.Endpoint.create(\n",
|
||||
" display_name=\"flowers_\" + UUID,\n",
|
||||
" display_name=\"xgboost_iris_\" + UUID,\n",
|
||||
" project=PROJECT_ID,\n",
|
||||
" location=REGION,\n",
|
||||
" labels={\"your_key\": \"your_value\"},\n",
|
||||
|
||||
@@ -34,18 +34,18 @@
|
||||
"<table align=\"left\">\n",
|
||||
"\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/notebook_template.ipynb\">\n",
|
||||
" <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/training/hyperparameter_tuning_tensorflow.ipynb\">\n",
|
||||
" <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/notebook_template.ipynb\">\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/training/hyperparameter_tuning_tensorflow.ipynb\">\n",
|
||||
" <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/notebook_template.ipynb\">\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/training/hyperparameter_tuning_tensorflow.ipynb\">\n",
|
||||
" <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
|
||||
" Open in Vertex AI Workbench\n",
|
||||
" </a>\n",
|
||||
|
||||
Reference in New Issue
Block a user