mirror of
https://github.com/GoogleCloudPlatform/vertex-ai-samples.git
synced 2026-09-26 14:42:04 +00:00
refactor, chore(egen): Refactored code according to template guidelines (#3136)
* refactor, chore(egen): Refactored code according to template guidelines, performed linter test * refactor, chore(egen): Removed vertexai SDK initiation in the beginning, performed linter test * refactor, chore(egen): Made some grammatical changes in markdown script, performed linter test * refactor(egen): modified code to delete locally created file, code to delete custom job * comment change --------- Co-authored-by: sumanvita-springml <sumanvita.kandregula@egen.ai>
This commit is contained in:
Rohith Alla
co-authored by
sumanvita-springml
sumanvita-springml
parent
3e33b7e0c6
commit
00729da920
@@ -8,7 +8,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Copyright 2022 Google LLC\n",
|
||||
"# Copyright 2024 Google LLC\n",
|
||||
"#\n",
|
||||
"# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
|
||||
"# you may not use this file except in compliance with the License.\n",
|
||||
@@ -32,22 +32,24 @@
|
||||
"# Profile model training performance using Vertex AI TensorBoard Profiler\n",
|
||||
"\n",
|
||||
"<table align=\"left\">\n",
|
||||
"\n",
|
||||
" <td>\n",
|
||||
" <td style=\"text-align: center\">\n",
|
||||
" <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/tensorboard/tensorboard_profiler_custom_training.ipynb\">\n",
|
||||
" <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
|
||||
" <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Google Colaboratory logo\"><br> Open in Colab\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/tensorboard/tensorboard_profiler_custom_training.ipynb\">\n",
|
||||
" <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" <td style=\"text-align: center\">\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/colab/import/https:%2F%2Fraw.githubusercontent.com%2FGoogleCloudPlatform%2Fvertex-ai-samples%2Fmain%2Fnotebooks%2Fofficial%2Ftensorboard%2Ftensorboard_profiler_custom_training.ipynb\">\n",
|
||||
" <img width=\"32px\" src=\"https://cloud.google.com/ml-engine/images/colab-enterprise-logo-32px.png\" alt=\"Google Cloud Colab Enterprise logo\"><br> Open in Colab Enterprise\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" </td> \n",
|
||||
" <td style=\"text-align: center\">\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/tensorboard/tensorboard_profiler_custom_training.ipynb\">\n",
|
||||
" <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
|
||||
" Open in Vertex AI Workbench\n",
|
||||
" <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\"><br> Open in Workbench\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td style=\"text-align: center\">\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/tensorboard/tensorboard_profiler_custom_training.ipynb\">\n",
|
||||
" <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\"><br> View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
"</table>"
|
||||
@@ -123,12 +125,19 @@
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "ze4-nDLfK4pw"
|
||||
"id": "61RBz8LLbxCR"
|
||||
},
|
||||
"source": [
|
||||
"## Installation\n",
|
||||
"\n",
|
||||
"Install the following packages required to execute this notebook. "
|
||||
"## Get started"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "No17Cw5hgx12"
|
||||
},
|
||||
"source": [
|
||||
"### Install Vertex AI SDK for Python and other required packages\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -145,158 +154,92 @@
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "aUw6ibN-n5Za"
|
||||
"id": "R5Xep4W9lq-Z"
|
||||
},
|
||||
"source": [
|
||||
"### Colab only: Uncomment the following cell to restart the kernel."
|
||||
"### Restart runtime (Colab only)\n",
|
||||
"\n",
|
||||
"To use the newly installed packages, you must restart the runtime on Google Colab."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "FM12wbWhn7w0"
|
||||
"id": "XRvKdaPDTznN"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Automatically restart kernel after installs so that your environment can access the new packages\n",
|
||||
"# import IPython\n",
|
||||
"import sys\n",
|
||||
"\n",
|
||||
"# app = IPython.Application.instance()\n",
|
||||
"# app.kernel.do_shutdown(True)"
|
||||
"if \"google.colab\" in sys.modules:\n",
|
||||
"\n",
|
||||
" import IPython\n",
|
||||
"\n",
|
||||
" app = IPython.Application.instance()\n",
|
||||
" app.kernel.do_shutdown(True)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "LgFWLeJfoGQu"
|
||||
"id": "SbmM4z7FOBpM"
|
||||
},
|
||||
"source": [
|
||||
"## Before you begin\n",
|
||||
"\n",
|
||||
"### Set up your Google Cloud project\n",
|
||||
"\n",
|
||||
"**The following steps are required, regardless of your notebook environment.**\n",
|
||||
"\n",
|
||||
"1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
|
||||
"\n",
|
||||
"2. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
|
||||
"\n",
|
||||
"3. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com). \n",
|
||||
"\n",
|
||||
"4. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk)."
|
||||
"<div class=\"alert alert-block alert-warning\">\n",
|
||||
"<b>⚠️ The kernel is going to restart. Wait until it's finished before continuing to the next step. ⚠️</b>\n",
|
||||
"</div>\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "8ckyxpX_oSzD"
|
||||
"id": "dmWOrTJ3gx13"
|
||||
},
|
||||
"source": [
|
||||
"#### Set your project ID\n",
|
||||
"### Authenticate your notebook environment (Colab only)\n",
|
||||
"\n",
|
||||
"**If you don't know your project ID**, try the following:\n",
|
||||
"* Run `gcloud config list`.\n",
|
||||
"* Run `gcloud projects list`.\n",
|
||||
"* See the support page: [Locate the project ID](https://support.google.com/googleapi/answer/7014113)"
|
||||
"Authenticate your environment on Google Colab.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "zY8DKBoVoVy3"
|
||||
"id": "NyKGtVQjgx13"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import sys\n",
|
||||
"\n",
|
||||
"if \"google.colab\" in sys.modules:\n",
|
||||
"\n",
|
||||
" from google.colab import auth\n",
|
||||
"\n",
|
||||
" auth.authenticate_user()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "DF4l8DTdWgPY"
|
||||
},
|
||||
"source": [
|
||||
"### Set Google Cloud project information and initialize Vertex AI SDK for Python\n",
|
||||
"\n",
|
||||
"To get started using Vertex AI, you must have an existing Google Cloud project and [enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com). Learn more about [setting up a project and a development environment](https://cloud.google.com/vertex-ai/docs/start/cloud-environment)."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "Nqwi-5ufWp_B"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"PROJECT_ID = \"[your-project-id]\" # @param {type:\"string\"}\n",
|
||||
"\n",
|
||||
"# Set the project id\n",
|
||||
"! gcloud config set project {PROJECT_ID}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "mSQjVQmMosMl"
|
||||
},
|
||||
"source": [
|
||||
"#### Region\n",
|
||||
"\n",
|
||||
"You can also change the `REGION` variable used by Vertex AI. Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "Se9FWWhLotvB"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"REGION = \"us-central1\" # @param {type:\"string\"}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "IfJRIMBpo5Pg"
|
||||
},
|
||||
"source": [
|
||||
"### Authenticate your Google Cloud account\n",
|
||||
"\n",
|
||||
"Depending on your Jupyter environment, you may have to manually authenticate. Follow the relevant instructions below."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "acFN0s3So9-Y"
|
||||
},
|
||||
"source": [
|
||||
"**1. Vertex AI Workbench**\n",
|
||||
"* Do nothing as you are already authenticated."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "dQ_mNwuapE5T"
|
||||
},
|
||||
"source": [
|
||||
"**2. Local JupyterLab instance, uncomment and run:**"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "cR_MzpknpGgM"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# ! gcloud auth login"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "h-MuVI_ypJfw"
|
||||
},
|
||||
"source": [
|
||||
"**3. Colab, uncomment and run:**"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "BeaQlCwMpQUT"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# from google.colab import auth\n",
|
||||
"# auth.authenticate_user()"
|
||||
"LOCATION = \"us-central1\" # @param {type:\"string\"}"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -305,9 +248,9 @@
|
||||
"id": "3ivZkPUjpaFz"
|
||||
},
|
||||
"source": [
|
||||
"**4. Setup service account and permissions**\n",
|
||||
"### Setup service account and permissions\n",
|
||||
"\n",
|
||||
"A service account is used to create custom training jobs. If you do not want to use your project's Compute Engine service account, set SERVICE_ACCOUNT to another service account ID. You can create a service account by following the [instructions](https://cloud.google.com/iam/docs/creating-managing-service-accounts#creating)."
|
||||
"A service account is used to create custom training jobs. If you don't want to use your project's Compute Engine service account, set SERVICE_ACCOUNT to another service account ID. You can create a service account by following the [instructions](https://cloud.google.com/iam/docs/creating-managing-service-accounts#creating)."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -404,7 +347,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI"
|
||||
"! gsutil mb -l $LOCATION -p $PROJECT_ID $BUCKET_URI"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -425,6 +368,7 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"import sys\n",
|
||||
"\n",
|
||||
"from google.cloud import aiplatform"
|
||||
]
|
||||
@@ -448,7 +392,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=BUCKET_URI)"
|
||||
"aiplatform.init(project=PROJECT_ID, location=LOCATION, staging_bucket=BUCKET_URI)"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -530,7 +474,7 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"tensorboard = aiplatform.Tensorboard.create(\n",
|
||||
" display_name=TENSORBOARD_NAME, project=PROJECT_ID, location=REGION\n",
|
||||
" display_name=TENSORBOARD_NAME, project=PROJECT_ID, location=LOCATION\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"TENSORBOARD_INSTANCE_NAME = tensorboard.resource_name\n",
|
||||
@@ -544,7 +488,7 @@
|
||||
"id": "yoR29gW2S24w"
|
||||
},
|
||||
"source": [
|
||||
"## Train a model\n",
|
||||
"### Train a model\n",
|
||||
"\n",
|
||||
"To train a model using your custom training code, choose one of the following options:\n",
|
||||
"\n",
|
||||
@@ -578,7 +522,7 @@
|
||||
"\n",
|
||||
"! gcloud artifacts repositories create {DOCKER_REPOSITORY} \\\n",
|
||||
" --repository-format=docker \\\n",
|
||||
" --location={REGION} \\\n",
|
||||
" --location={LOCATION} \\\n",
|
||||
" --description=\"Repository for TensorBoard Custom Training Job\" \\\n",
|
||||
" --quiet\n",
|
||||
"\n",
|
||||
@@ -604,10 +548,8 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import sys\n",
|
||||
"\n",
|
||||
"if not IS_COLAB:\n",
|
||||
" ! gcloud auth configure-docker {REGION}-docker.pkg.dev --quiet"
|
||||
" ! gcloud auth configure-docker {LOCATION}-docker.pkg.dev --quiet"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -817,9 +759,9 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"IMAGE_NAME = \"tensorboard-custom-container\"\n",
|
||||
"IMAGE_URI = f\"{REGION}-docker.pkg.dev/{PROJECT_ID}/{DOCKER_REPOSITORY}/{IMAGE_NAME}\"\n",
|
||||
"IMAGE_URI = f\"{LOCATION}-docker.pkg.dev/{PROJECT_ID}/{DOCKER_REPOSITORY}/{IMAGE_NAME}\"\n",
|
||||
"\n",
|
||||
"! gcloud builds submit --project {PROJECT_ID} --region={REGION} --tag {IMAGE_URI} --timeout=3600s --quiet"
|
||||
"! gcloud builds submit --project {PROJECT_ID} --region={LOCATION} --tag {IMAGE_URI} --timeout=3600s --quiet"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -932,21 +874,31 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"delete_tensorboard = True\n",
|
||||
"delete_bucket = False\n",
|
||||
"\n",
|
||||
"# Delete docker repository.\n",
|
||||
"! gcloud artifacts repositories delete $DOCKER_REPOSITORY --project {PROJECT_ID} --location {REGION} --quiet\n",
|
||||
"! gcloud artifacts repositories delete $DOCKER_REPOSITORY --project {PROJECT_ID} --location {LOCATION} --quiet\n",
|
||||
"\n",
|
||||
"# delete training job\n",
|
||||
"try:\n",
|
||||
" job.delete()\n",
|
||||
"except Exception as e:\n",
|
||||
" print(e)\n",
|
||||
"\n",
|
||||
"# delete tensorboard instance\n",
|
||||
"delete_tensorboard = True\n",
|
||||
"if delete_tensorboard:\n",
|
||||
" tensorboard.delete()\n",
|
||||
"\n",
|
||||
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
|
||||
"# delete custom job\n",
|
||||
"custom_job = f\"{JOB_NAME}-custom-job\"\n",
|
||||
"custom_job_to_delete = aiplatform.CustomJob.list(filter=f\"display_name={custom_job}\")[0]\n",
|
||||
"custom_job_to_delete.delete()\n",
|
||||
"\n",
|
||||
"# delete locally generated files and folders\n",
|
||||
"! rm -rf {PYTHON_PACKAGE_APPLICATION_DIR} Dockerfile\n",
|
||||
"\n",
|
||||
"# delete cloud storage bucket\n",
|
||||
"delete_bucket = False # set True for deletion\n",
|
||||
"if delete_bucket:\n",
|
||||
" ! gsutil -m rm -r $BUCKET_URI"
|
||||
]
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user