mirror of
https://github.com/GoogleCloudPlatform/vertex-ai-samples.git
synced 2026-09-26 14:42:04 +00:00
Compare commits
7
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Andrew Ferlitsch
|
3e48068223 | ||
|
Andrew Ferlitsch
|
a505e1ae88 | ||
|
Andrew Ferlitsch
|
3318ef3c6a | ||
|
Andrew Ferlitsch
|
3b07050e59 | ||
|
Andrew FerlitschandGitHub
|
defec0e5f6 | ||
|
Andrew Ferlitsch
|
e10df6fab1 | ||
|
Andrew Ferlitsch
|
f63bf39797 |
@@ -33,21 +33,22 @@
|
||||
"\n",
|
||||
"<table align=\"left\">\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/official/pipelines/lightweight_functions_component_io_kfp.ipynb\">\n",
|
||||
" <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/lightweight_functions_component_io_kfp.ipynb\">\n",
|
||||
" <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/official/pipelines/lightweight_functions_component_io_kfp.ipynb\">\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/lightweight_functions_component_io_kfp.ipynb\">\n",
|
||||
" <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/official/pipelines/lightweight_functions_component_io_kfp.ipynb\">\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/lightweight_functions_component_io_kfp.ipynb\">\n",
|
||||
" <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
|
||||
" Open in Vertex AI Workbench\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
"</table>\n",
|
||||
"<br/><br/><br/>"
|
||||
]
|
||||
@@ -71,13 +72,20 @@
|
||||
"source": [
|
||||
"### Objective\n",
|
||||
"\n",
|
||||
"In this tutorial, you use the KFP SDK to build lightweight Python function-based components.\n",
|
||||
"In this tutorial, you learn to use the KFP SDK to build lightweight Python function-based components, and then you learn to use `Vertex AI Pipelines` to execute the pipeline.\n",
|
||||
"\n",
|
||||
"This tutorial uses the following Google Cloud ML services:\n",
|
||||
"\n",
|
||||
"- `Vertex AI Pipelines`\n",
|
||||
"\n",
|
||||
"The steps performed include:\n",
|
||||
"\n",
|
||||
"- Build Python function-based components.\n",
|
||||
"- Build Python function-based KFP components.\n",
|
||||
"- Construct a KFP pipeline.\n",
|
||||
"- Pass *Artifacts* and *parameters* between components, both by path reference and by value.\n",
|
||||
"- Use the `kfp.dsl.importer` method."
|
||||
"- Use the `kfp.dsl.importer` method.\n",
|
||||
"- Compile the KFP pipeline.\n",
|
||||
"- Execute the KFP pipeline using `Vertex AI Pipelines`"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -167,7 +175,7 @@
|
||||
"source": [
|
||||
"## Installation\n",
|
||||
"\n",
|
||||
"Install the latest version of Vertex AI SDK for Python."
|
||||
"Install the packages required for executing this notebook."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -180,53 +188,20 @@
|
||||
"source": [
|
||||
"import os\n",
|
||||
"\n",
|
||||
"# Google Cloud Notebook\n",
|
||||
"if os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
|
||||
" USER_FLAG = \"--user\"\n",
|
||||
"else:\n",
|
||||
" USER_FLAG = \"\"\n",
|
||||
"# The Vertex AI Workbench Notebook product has specific requirements\n",
|
||||
"IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
|
||||
"IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
|
||||
" \"/opt/deeplearning/metadata/env_version\"\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "install_storage"
|
||||
},
|
||||
"source": [
|
||||
"Install the latest GA version of *google-cloud-storage* library as well."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "install_storage"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! pip3 install -U google-cloud-storage $USER_FLAG"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "install_gcpc"
|
||||
},
|
||||
"source": [
|
||||
"Install the latest GA version of *google-cloud-pipeline-components* library as well."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "install_gcpc"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! pip3 install $USER kfp google-cloud-pipeline-components --upgrade"
|
||||
"# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
|
||||
"USER_FLAG = \"\"\n",
|
||||
"if IS_WORKBENCH_NOTEBOOK:\n",
|
||||
" USER_FLAG = \"--user\"\n",
|
||||
"\n",
|
||||
"! pip3 install --upgrade google-cloud-aiplatform {USER_FLAG} -q\n",
|
||||
"! pip3 install -U google-cloud-storage {USER_FLAG} -q\n",
|
||||
"! pip3 install {USER_FLAG} kfp google-cloud-pipeline-components --upgrade -q"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -409,23 +384,30 @@
|
||||
"source": [
|
||||
"### Authenticate your Google Cloud account\n",
|
||||
"\n",
|
||||
"**If you are using Google Cloud Notebook**, your environment is already authenticated. Skip this step.\n",
|
||||
"**If you are using Vertex AI Workbench Notebook**, your environment is already authenticated. Skip this step.\n",
|
||||
"\n",
|
||||
"**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
|
||||
"**If you are using Colab**, run the cell below and follow the instructions\n",
|
||||
"when prompted to authenticate your account via oAuth.\n",
|
||||
"\n",
|
||||
"**Otherwise**, follow these steps:\n",
|
||||
"\n",
|
||||
"In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
|
||||
"1. In the Cloud Console, go to the [**Create service account key**\n",
|
||||
" page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
|
||||
"\n",
|
||||
"**Click Create service account**.\n",
|
||||
"2. Click **Create service account**.\n",
|
||||
"\n",
|
||||
"In the **Service account name** field, enter a name, and click **Create**.\n",
|
||||
"3. In the **Service account name** field, enter a name, and\n",
|
||||
" click **Create**.\n",
|
||||
"\n",
|
||||
"In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex\" into the filter box, and select **Vertex Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
|
||||
"4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
|
||||
"into the filter box, and select\n",
|
||||
" **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
|
||||
"\n",
|
||||
"Click Create. A JSON file that contains your key downloads to your local environment.\n",
|
||||
"5. Click *Create*. A JSON file that contains your key downloads to your\n",
|
||||
"local environment.\n",
|
||||
"\n",
|
||||
"Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
|
||||
"6. Enter the path to your service account key as the\n",
|
||||
"`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -444,8 +426,11 @@
|
||||
"import os\n",
|
||||
"import sys\n",
|
||||
"\n",
|
||||
"# If on Google Cloud Notebook, then don't execute this code\n",
|
||||
"if not os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
|
||||
"# If on Vertex AI Workbench, then don't execute this code\n",
|
||||
"IS_COLAB = \"google.colab\" in sys.modules\n",
|
||||
"if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
|
||||
" \"DL_ANACONDA_HOME\"\n",
|
||||
"):\n",
|
||||
" if \"google.colab\" in sys.modules:\n",
|
||||
" from google.colab import auth as google_auth\n",
|
||||
"\n",
|
||||
@@ -481,7 +466,8 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"BUCKET_NAME = \"gs://[your-bucket-name]\" # @param {type:\"string\"}"
|
||||
"BUCKET_NAME = \"[your-bucket-name]\" # @param {type:\"string\"}\n",
|
||||
"BUCKET_URI = f\"gs://{BUCKET_NAME}\""
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -492,8 +478,9 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
|
||||
" BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
|
||||
"if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
|
||||
" BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
|
||||
" BUCKET_URI = \"gs://\" + BUCKET_NAME"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -513,7 +500,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil mb -l $REGION $BUCKET_NAME"
|
||||
"! gsutil mb -l $REGION $BUCKET_URI"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -533,7 +520,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil ls -al $BUCKET_NAME"
|
||||
"! gsutil ls -al $BUCKET_URI"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -571,9 +558,16 @@
|
||||
" or SERVICE_ACCOUNT is None\n",
|
||||
" or SERVICE_ACCOUNT == \"[your-service-account]\"\n",
|
||||
"):\n",
|
||||
" # Get your GCP project id from gcloud\n",
|
||||
" shell_output = !gcloud auth list 2>/dev/null\n",
|
||||
" SERVICE_ACCOUNT = shell_output[2].strip()\n",
|
||||
" # Get your service account from gcloud\n",
|
||||
" if not IS_COLAB:\n",
|
||||
" shell_output = !gcloud auth list 2>/dev/null\n",
|
||||
" SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
|
||||
"\n",
|
||||
" if IS_COLAB:\n",
|
||||
" shell_output = ! gcloud projects describe $PROJECT_ID\n",
|
||||
" project_number = shell_output[-1].split(\":\")[1].strip().replace(\"'\", \"\")\n",
|
||||
" SERVICE_ACCOUNT = f\"{project_number}-compute@developer.gserviceaccount.com\"\n",
|
||||
"\n",
|
||||
" print(\"Service Account:\", SERVICE_ACCOUNT)"
|
||||
]
|
||||
},
|
||||
@@ -596,9 +590,9 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_NAME\n",
|
||||
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
|
||||
"\n",
|
||||
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_NAME"
|
||||
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -643,7 +637,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"PIPELINE_ROOT = \"{}/pipeline_root/shakespeare\".format(BUCKET_NAME)"
|
||||
"PIPELINE_ROOT = \"{}/pipeline_root/shakespeare\".format(BUCKET_URI)"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -690,7 +684,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"aip.init(project=PROJECT_ID, staging_bucket=BUCKET_NAME)"
|
||||
"aip.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -942,7 +936,7 @@
|
||||
"from kfp.v2 import compiler # noqa: F811\n",
|
||||
"\n",
|
||||
"compiler.Compiler().compile(\n",
|
||||
" pipeline_func=pipeline, package_path=\"lightweight_pipeline.json\".replace(\" \", \"_\")\n",
|
||||
" pipeline_func=pipeline, package_path=\"lightweight_pipeline.json\"\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
@@ -969,7 +963,7 @@
|
||||
"\n",
|
||||
"job = aip.PipelineJob(\n",
|
||||
" display_name=DISPLAY_NAME,\n",
|
||||
" template_path=\"lightweight_pipeline.json\".replace(\" \", \"_\"),\n",
|
||||
" template_path=\"lightweight_pipeline.json\",\n",
|
||||
" pipeline_root=PIPELINE_ROOT,\n",
|
||||
" parameter_values={\"message\": \"Hello, World\"},\n",
|
||||
")\n",
|
||||
@@ -1007,16 +1001,7 @@
|
||||
"To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
|
||||
"project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
|
||||
"\n",
|
||||
"Otherwise, you can delete the individual resources you created in this tutorial -- *Note:* this is auto-generated and not all resources may be applicable for this tutorial:\n",
|
||||
"\n",
|
||||
"- Dataset\n",
|
||||
"- Pipeline\n",
|
||||
"- Model\n",
|
||||
"- Endpoint\n",
|
||||
"- Batch Job\n",
|
||||
"- Custom Job\n",
|
||||
"- Hyperparameter Tuning Job\n",
|
||||
"- Cloud Storage Bucket"
|
||||
"Otherwise, you can delete the individual resources you created in this tutorial."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -1027,96 +1012,10 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"delete_dataset = True\n",
|
||||
"delete_pipeline = True\n",
|
||||
"delete_model = True\n",
|
||||
"delete_endpoint = True\n",
|
||||
"delete_batchjob = True\n",
|
||||
"delete_customjob = True\n",
|
||||
"delete_hptjob = True\n",
|
||||
"delete_bucket = True\n",
|
||||
"delete_bucket = False\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" if delete_model and \"DISPLAY_NAME\" in globals():\n",
|
||||
" models = aip.Model.list(\n",
|
||||
" filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
|
||||
" )\n",
|
||||
" model = models[0]\n",
|
||||
" aip.Model.delete(model)\n",
|
||||
" print(\"Deleted model:\", model)\n",
|
||||
"except Exception as e:\n",
|
||||
" print(e)\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" if delete_endpoint and \"DISPLAY_NAME\" in globals():\n",
|
||||
" endpoints = aip.Endpoint.list(\n",
|
||||
" filter=f\"display_name={DISPLAY_NAME}_endpoint\", order_by=\"create_time\"\n",
|
||||
" )\n",
|
||||
" endpoint = endpoints[0]\n",
|
||||
" endpoint.undeploy_all()\n",
|
||||
" aip.Endpoint.delete(endpoint.resource_name)\n",
|
||||
" print(\"Deleted endpoint:\", endpoint)\n",
|
||||
"except Exception as e:\n",
|
||||
" print(e)\n",
|
||||
"\n",
|
||||
"if delete_dataset and \"DISPLAY_NAME\" in globals():\n",
|
||||
" if \"text\" == \"tabular\":\n",
|
||||
" try:\n",
|
||||
" datasets = aip.TabularDataset.list(\n",
|
||||
" filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
|
||||
" )\n",
|
||||
" dataset = datasets[0]\n",
|
||||
" aip.TabularDataset.delete(dataset.resource_name)\n",
|
||||
" print(\"Deleted dataset:\", dataset)\n",
|
||||
" except Exception as e:\n",
|
||||
" print(e)\n",
|
||||
"\n",
|
||||
" if \"text\" == \"image\":\n",
|
||||
" try:\n",
|
||||
" datasets = aip.ImageDataset.list(\n",
|
||||
" filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
|
||||
" )\n",
|
||||
" dataset = datasets[0]\n",
|
||||
" aip.ImageDataset.delete(dataset.resource_name)\n",
|
||||
" print(\"Deleted dataset:\", dataset)\n",
|
||||
" except Exception as e:\n",
|
||||
" print(e)\n",
|
||||
"\n",
|
||||
" if \"text\" == \"text\":\n",
|
||||
" try:\n",
|
||||
" datasets = aip.TextDataset.list(\n",
|
||||
" filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
|
||||
" )\n",
|
||||
" dataset = datasets[0]\n",
|
||||
" aip.TextDataset.delete(dataset.resource_name)\n",
|
||||
" print(\"Deleted dataset:\", dataset)\n",
|
||||
" except Exception as e:\n",
|
||||
" print(e)\n",
|
||||
"\n",
|
||||
" if \"text\" == \"video\":\n",
|
||||
" try:\n",
|
||||
" datasets = aip.VideoDataset.list(\n",
|
||||
" filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
|
||||
" )\n",
|
||||
" dataset = datasets[0]\n",
|
||||
" aip.VideoDataset.delete(dataset.resource_name)\n",
|
||||
" print(\"Deleted dataset:\", dataset)\n",
|
||||
" except Exception as e:\n",
|
||||
" print(e)\n",
|
||||
"\n",
|
||||
"try:\n",
|
||||
" if delete_pipeline and \"DISPLAY_NAME\" in globals():\n",
|
||||
" pipelines = aip.PipelineJob.list(\n",
|
||||
" filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
|
||||
" )\n",
|
||||
" pipeline = pipelines[0]\n",
|
||||
" aip.PipelineJob.delete(pipeline.resource_name)\n",
|
||||
" print(\"Deleted pipeline:\", pipeline)\n",
|
||||
"except Exception as e:\n",
|
||||
" print(e)\n",
|
||||
"\n",
|
||||
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
|
||||
" ! gsutil rm -r $BUCKET_NAME"
|
||||
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
|
||||
" ! gsutil rm -r $BUCKET_URI"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
||||
Reference in New Issue
Block a user