mirror of
https://github.com/GoogleCloudPlatform/vertex-ai-samples.git
synced 2026-09-26 14:42:04 +00:00
upgrade: current notebook standards (#686)
* upgrade: notebook standard * upgrade: notebook standard
This commit is contained in:
Andrew Ferlitsch
+70
-66
@@ -32,18 +32,18 @@
|
||||
"<table align=\"left\">\n",
|
||||
"\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-custom-jobs.ipynb\">\n",
|
||||
" <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-custom-jobs.ipynb\">\n",
|
||||
" <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-custom-jobs.ipynb\">\n",
|
||||
" <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-custom-jobs.ipynb\">\n",
|
||||
" <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
|
||||
" View on GitHub\n",
|
||||
" </a>\n",
|
||||
" </td>\n",
|
||||
" <td>\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-custom-jobs.ipynb\">\n",
|
||||
" <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-custom-jobs.ipynb\">\n",
|
||||
" <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
|
||||
" Open in Vertex AI Workbench\n",
|
||||
" </a>\n",
|
||||
@@ -68,17 +68,24 @@
|
||||
"source": [
|
||||
"## Overview\n",
|
||||
"\n",
|
||||
"This notebook demonstrates how to track metrics and parameters for Vertex AI custom training jobs, and how to perform detailed analysis using this data.\n",
|
||||
"This notebook demonstrates how to track metrics and parameters for `Vertex AI` custom training jobs, and how to perform detailed analysis using this data.\n",
|
||||
"\n",
|
||||
"### Dataset\n",
|
||||
"\n",
|
||||
"This example uses the Abalone Dataset. For more information about this dataset please visit: https://archive.ics.uci.edu/ml/datasets/abalone\n",
|
||||
"### Objective\n",
|
||||
"\n",
|
||||
"In this notebook, you will learn how to use Vertex AI SDK for Python to:\n",
|
||||
"In this notebook, you learn how to use `Vertex ML Metadata` to track training parameters and evaluation metrics.\n",
|
||||
"\n",
|
||||
" * Track training parameters and prediction metrics for a custom training job.\n",
|
||||
" * Extract and perform analysis for all parameters and metrics within an Experiment.\n",
|
||||
"This tutorial uses the following Google Cloud ML services:\n",
|
||||
"\n",
|
||||
"- `Vertex ML Metadata`\n",
|
||||
"- `Vertex AI Experiments`\n",
|
||||
"\n",
|
||||
"The steps performed include:\n",
|
||||
"\n",
|
||||
"- Track parameters and metrics for a `Vertex AI` custom trained model.\n",
|
||||
"- Extract and perform analysis for all parameters and metrics within an Experiment.\n",
|
||||
"\n",
|
||||
"### Costs \n",
|
||||
"\n",
|
||||
@@ -165,26 +172,20 @@
|
||||
"source": [
|
||||
"import os\n",
|
||||
"\n",
|
||||
"# The Google Cloud Notebook product has specific requirements\n",
|
||||
"IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
|
||||
"# The Vertex AI Workbench Notebook product has specific requirements\n",
|
||||
"IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
|
||||
"IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
|
||||
" \"/opt/deeplearning/metadata/env_version\"\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# Google Cloud Notebook requires dependencies to be installed with '--user'\n",
|
||||
"# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
|
||||
"USER_FLAG = \"\"\n",
|
||||
"if IS_GOOGLE_CLOUD_NOTEBOOK:\n",
|
||||
" USER_FLAG = \"--user\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "qblyW_dcyOQA"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"! pip3 install -U tensorflow $USER_FLAG\n",
|
||||
"! python3 -m pip install {USER_FLAG} google-cloud-aiplatform --upgrade\n",
|
||||
"! pip3 install scikit-learn {USER_FLAG}\n"
|
||||
"if IS_WORKBENCH_NOTEBOOK:\n",
|
||||
" USER_FLAG = \"--user\"\n",
|
||||
"\n",
|
||||
"! pip3 install --upgrade google-cloud-aiplatform {USER_FLAG} -q\n",
|
||||
"! pip3 install -U tensorflow $USER_FLAG -q\n",
|
||||
"! pip3 install scikit-learn {USER_FLAG} -q"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -273,24 +274,7 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"\n",
|
||||
"PROJECT_ID = \"\"\n",
|
||||
"\n",
|
||||
"# Get your Google Cloud project ID from gcloud\n",
|
||||
"if not os.getenv(\"IS_TESTING\"):\n",
|
||||
" shell_output = !gcloud config list --format 'value(core.project)' 2>/dev/null\n",
|
||||
" PROJECT_ID = shell_output[0]\n",
|
||||
" print(\"Project ID: \", PROJECT_ID)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "qJYoRfYng0XZ"
|
||||
},
|
||||
"source": [
|
||||
"Otherwise, set your project ID here."
|
||||
"PROJECT_ID = \"[your-project-id]\" # @param {type:\"string\"}"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -302,16 +286,11 @@
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
|
||||
" PROJECT_ID = \"[your-project-id]\" # @param {type:\"string\"}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "XsnuGoJM9mUw"
|
||||
},
|
||||
"source": [
|
||||
"Set gcloud config to your project ID."
|
||||
" PROJECT_ID = \"[your-project-id]\" # @param {type:\"string\"}\n",
|
||||
" # Get your GCP project id from gcloud\n",
|
||||
" shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
|
||||
" PROJECT_ID = shell_output[0]\n",
|
||||
" print(\"Project ID:\", PROJECT_ID)"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -325,6 +304,37 @@
|
||||
"!gcloud config set project $PROJECT_ID"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "region"
|
||||
},
|
||||
"source": [
|
||||
"#### Region\n",
|
||||
"\n",
|
||||
"You can also change the `REGION` variable, which is used for operations\n",
|
||||
"throughout the rest of this notebook. Below are regions supported for Vertex AI. We recommend that you choose the region closest to you.\n",
|
||||
"\n",
|
||||
"- Americas: `us-central1`\n",
|
||||
"- Europe: `europe-west4`\n",
|
||||
"- Asia Pacific: `asia-east1`\n",
|
||||
"\n",
|
||||
"You may not use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
|
||||
"\n",
|
||||
"Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"id": "region"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"REGION = \"us-central1\" # @param {type: \"string\"}"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
@@ -408,6 +418,7 @@
|
||||
"# requests.\n",
|
||||
"\n",
|
||||
"# If on Google Cloud Notebooks, then don't execute this code\n",
|
||||
"IS_COLAB = \"google.colab\" in sys.modules\n",
|
||||
"if not os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
|
||||
" if \"google.colab\" in sys.modules:\n",
|
||||
" from google.colab import auth as google_auth\n",
|
||||
@@ -440,12 +451,7 @@
|
||||
"online predictions.\n",
|
||||
"\n",
|
||||
"Set the name of your Cloud Storage bucket below. It must be unique across all\n",
|
||||
"Cloud Storage buckets.\n",
|
||||
"\n",
|
||||
"You may also change the `REGION` variable, which is used for operations\n",
|
||||
"throughout the rest of this notebook. Make sure to [choose a region where Vertex AI services are\n",
|
||||
"available](https://cloud.google.com/vertex-ai/docs/general/locations#available_regions). You may\n",
|
||||
"not use a Multi-Regional Storage bucket for training with Vertex AI."
|
||||
"Cloud Storage buckets."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -456,8 +462,8 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"BUCKET_URI = \"gs://[your-bucket-name]\" # @param {type:\"string\"}\n",
|
||||
"REGION = \"[your-region]\" # @param {type:\"string\"}"
|
||||
"BUCKET_NAME = \"[your-bucket-name]\" # @param {type:\"string\"}\n",
|
||||
"BUCKET_URI = f\"gs://{BUCKET_NAME}\""
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -468,11 +474,9 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
|
||||
" BUCKET_URI = \"gs://\" + PROJECT_ID + \"-aip-\" + TIMESTAMP\n",
|
||||
"\n",
|
||||
"if REGION == \"[your-region]\":\n",
|
||||
" REGION = \"us-central1\""
|
||||
"if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
|
||||
" BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
|
||||
" BUCKET_URI = \"gs://\" + BUCKET_NAME"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user