From 741c6732f13de89047ce7d3acc3c0ac1ebbbb34d Mon Sep 17 00:00:00 2001 From: sudarshan-SpringML <82567512+sudarshan-SpringML@users.noreply.github.com> Date: Fri, 22 Apr 2022 12:35:07 +0530 Subject: [PATCH] Made minor changes to sdk-metric-parameter-tracking-for-custom-jobs file (#479) * modified file * modified file * ran linter test * deleted file in community folder * ran linter test * changed folder name in links * ran linter test * resolved comments * ran linter test * modified file * ran linter test --- ...c-parameter-tracking-for-custom-jobs.ipynb | 1038 ----------------- ...c-parameter-tracking-for-custom-jobs.ipynb | 117 +- 2 files changed, 50 insertions(+), 1105 deletions(-) delete mode 100644 notebooks/community/ml_metadata/sdk-metric-parameter-tracking-for-custom-jobs.ipynb diff --git a/notebooks/community/ml_metadata/sdk-metric-parameter-tracking-for-custom-jobs.ipynb b/notebooks/community/ml_metadata/sdk-metric-parameter-tracking-for-custom-jobs.ipynb deleted file mode 100644 index a469afb14..000000000 --- a/notebooks/community/ml_metadata/sdk-metric-parameter-tracking-for-custom-jobs.ipynb +++ /dev/null @@ -1,1038 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "ur8xi4C7S06n" - }, - "outputs": [], - "source": [ - "# Copyright 2021 Google LLC\n", - "#\n", - "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "JAPoU8Sm5E6e" - }, - "source": [ - "\n", - "\n", - " \n", - " \n", - "
\n", - " \n", - " \"Colab Run in Colab\n", - " \n", - " \n", - " \n", - " \"GitHub\n", - " View on GitHub\n", - " \n", - "
" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "j9gUDU_3vV9d" - }, - "source": [ - "#Vertex AI: Track parameters and metrics for custom training jobs" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "tvgnzT1CKxrO" - }, - "source": [ - "## Overview\n", - "\n", - "This notebook demonstrates how to track metrics and parameters for Vertex AI custom training jobs, and how to perform detailed analysis using this data.\n", - "\n", - "### Dataset\n", - "\n", - "This example uses the Abalone Dataset. For more information about this dataset please visit: https://archive.ics.uci.edu/ml/datasets/abalone\n", - "### Objective\n", - "\n", - "In this notebook, you will learn how to use Vertex SDK for Python to:\n", - "\n", - " * Track training parameters and prediction metrics for a custom training job.\n", - " * Extract and perform analysis for all parameters and metrics within an Experiment.\n", - "\n", - "### Costs \n", - "\n", - "\n", - "This tutorial uses billable components of Google Cloud:\n", - "\n", - "* Vertex AI\n", - "* Cloud Storage\n", - "\n", - "Learn about [Vertex AI\n", - "pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage\n", - "pricing](https://cloud.google.com/storage/pricing), and use the [Pricing\n", - "Calculator](https://cloud.google.com/products/calculator/)\n", - "to generate a cost estimate based on your projected usage." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "ze4-nDLfK4pw" - }, - "source": [ - "### Set up your local development environment\n", - "\n", - "**If you are using Colab or Google Cloud Notebooks**, your environment already meets\n", - "all the requirements to run this notebook. You can skip this step." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "gCuSR8GkAgzl" - }, - "source": [ - "**Otherwise**, make sure your environment meets this notebook's requirements.\n", - "You need the following:\n", - "\n", - "* The Google Cloud SDK\n", - "* Git\n", - "* Python 3\n", - "* virtualenv\n", - "* Jupyter notebook running in a virtual environment with Python 3\n", - "\n", - "The Google Cloud guide to [Setting up a Python development\n", - "environment](https://cloud.google.com/python/setup) and the [Jupyter\n", - "installation guide](https://jupyter.org/install) provide detailed instructions\n", - "for meeting these requirements. The following steps provide a condensed set of\n", - "instructions:\n", - "\n", - "1. [Install and initialize the Cloud SDK.](https://cloud.google.com/sdk/docs/)\n", - "\n", - "1. [Install Python 3.](https://cloud.google.com/python/setup#installing_python)\n", - "\n", - "1. [Install\n", - " virtualenv](https://cloud.google.com/python/setup#installing_and_using_virtualenv)\n", - " and create a virtual environment that uses Python 3. Activate the virtual environment.\n", - "\n", - "1. To install Jupyter, run `pip install jupyter` on the\n", - "command-line in a terminal shell.\n", - "\n", - "1. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n", - "\n", - "1. Open this notebook in the Jupyter Notebook Dashboard." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "i7EUnXsZhAGF" - }, - "source": [ - "### Install additional packages\n", - "\n", - "Run the following commands to install the Vertex SDK for Python." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "IaYsrh0Tc17L" - }, - "outputs": [], - "source": [ - "import sys\n", - "\n", - "if \"google.colab\" in sys.modules:\n", - " USER_FLAG = \"\"\n", - "else:\n", - " USER_FLAG = \"--user\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "qblyW_dcyOQA" - }, - "outputs": [], - "source": [ - "!python3 -m pip install {USER_FLAG} google-cloud-aiplatform --upgrade" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "hhq5zEbGg0XX" - }, - "source": [ - "### Restart the kernel\n", - "\n", - "After you install the additional packages, you need to restart the notebook kernel so it can find the packages." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "EzrelQZ22IZj" - }, - "outputs": [], - "source": [ - "# Automatically restart kernel after installs\n", - "import os\n", - "\n", - "if not os.getenv(\"IS_TESTING\"):\n", - " # Automatically restart kernel after installs\n", - " import IPython\n", - "\n", - " app = IPython.Application.instance()\n", - " app.kernel.do_shutdown(True)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "lWEdiXsJg0XY" - }, - "source": [ - "## Before you begin\n", - "\n", - "### Select a GPU runtime\n", - "\n", - "**Make sure you're running this notebook in a GPU runtime if you have that option. In Colab, select \"Runtime --> Change runtime type > GPU\"**" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "BF1j6f9HApxa" - }, - "source": [ - "### Set up your Google Cloud project\n", - "\n", - "**The following steps are required, regardless of your notebook environment.**\n", - "\n", - "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n", - "\n", - "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n", - "\n", - "1. [Enable the Vertex AI API and Compute Engine API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component).\n", - "\n", - "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n", - "\n", - "1. Enter your project ID in the cell below. Then run the cell to make sure the\n", - "Cloud SDK uses the right project for all the commands in this notebook.\n", - "\n", - "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "WReHDGG5g0XY" - }, - "source": [ - "#### Set your project ID\n", - "\n", - "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "oM1iC_MfAts1" - }, - "outputs": [], - "source": [ - "import os\n", - "\n", - "PROJECT_ID = \"\"\n", - "\n", - "# Get your Google Cloud project ID from gcloud\n", - "if not os.getenv(\"IS_TESTING\"):\n", - " shell_output=!gcloud config list --format 'value(core.project)' 2>/dev/null\n", - " PROJECT_ID = shell_output[0]\n", - " print(\"Project ID: \", PROJECT_ID)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "qJYoRfYng0XZ" - }, - "source": [ - "Otherwise, set your project ID here." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "riG_qUokg0XZ" - }, - "outputs": [], - "source": [ - "if PROJECT_ID == \"\" or PROJECT_ID is None:\n", - " PROJECT_ID = \"[your-project-id]\" # @param {type:\"string\"}" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "XsnuGoJM9mUw" - }, - "source": [ - "Set gcloud config to your project ID." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "TL9QIaVd9hvm" - }, - "outputs": [], - "source": [ - "!gcloud config set project $PROJECT_ID" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "06571eb4063b" - }, - "source": [ - "#### Timestamp\n", - "\n", - "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append it onto the name of resources you create in this tutorial." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "697568e92bd6" - }, - "outputs": [], - "source": [ - "from datetime import datetime\n", - "\n", - "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "dr--iN2kAylZ" - }, - "source": [ - "### Authenticate your Google Cloud account\n", - "\n", - "**If you are using Google Cloud Notebooks**, your environment is already\n", - "authenticated. Skip this step." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "sBCra4QMA2wR" - }, - "source": [ - "**If you are using Colab**, run the cell below and follow the instructions\n", - "when prompted to authenticate your account via oAuth.\n", - "\n", - "**Otherwise**, follow these steps:\n", - "\n", - "1. In the Cloud Console, go to the [**Create service account key**\n", - " page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n", - "\n", - "2. Click **Create service account**.\n", - "\n", - "3. In the **Service account name** field, enter a name, and\n", - " click **Create**.\n", - "\n", - "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n", - "into the filter box, and select\n", - " **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n", - "\n", - "5. Click *Create*. A JSON file that contains your key downloads to your\n", - "local environment.\n", - "\n", - "6. Enter the path to your service account key as the\n", - "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "PyQmSRbKA8r-" - }, - "outputs": [], - "source": [ - "import os\n", - "import sys\n", - "\n", - "# If you are running this notebook in Colab, run this cell and follow the\n", - "# instructions to authenticate your GCP account. This provides access to your\n", - "# Cloud Storage bucket and lets you submit training jobs and prediction\n", - "# requests.\n", - "\n", - "# If on Google Cloud Notebooks, then don't execute this code\n", - "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n", - " if \"google.colab\" in sys.modules:\n", - " from google.colab import auth as google_auth\n", - "\n", - " google_auth.authenticate_user()\n", - "\n", - " # If you are running this notebook locally, replace the string below with the\n", - " # path to your service account key and run this cell to authenticate your GCP\n", - " # account.\n", - " elif not os.getenv(\"IS_TESTING\"):\n", - " %env GOOGLE_APPLICATION_CREDENTIALS ''" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "zgPO1eR3CYjk" - }, - "source": [ - "### Create a Cloud Storage bucket\n", - "\n", - "**The following steps are required, regardless of your notebook environment.**\n", - "\n", - "\n", - "When you submit a training job using the Cloud SDK, you upload a Python package\n", - "containing your training code to a Cloud Storage bucket. Vertex AI runs\n", - "the code from this package. In this tutorial, Vertex AI also saves the\n", - "trained model that results from your job in the same bucket. Using this model artifact, you can then\n", - "create Vertex AI model and endpoint resources in order to serve\n", - "online predictions.\n", - "\n", - "Set the name of your Cloud Storage bucket below. It must be unique across all\n", - "Cloud Storage buckets.\n", - "\n", - "You may also change the `REGION` variable, which is used for operations\n", - "throughout the rest of this notebook. Make sure to [choose a region where Vertex AI services are\n", - "available](https://cloud.google.com/vertex-ai/docs/general/locations#available_regions). You may\n", - "not use a Multi-Regional Storage bucket for training with Vertex AI." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "MzGDU7TWdts_" - }, - "outputs": [], - "source": [ - "BUCKET_NAME = \"gs://[your-bucket-name]\" # @param {type:\"string\"}\n", - "REGION = \"[your-region]\" # @param {type:\"string\"}" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "cf221059d072" - }, - "outputs": [], - "source": [ - "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n", - " BUCKET_NAME = \"gs://\" + PROJECT_ID + \"-aip-\" + TIMESTAMP" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "-EcIXiGsCePi" - }, - "source": [ - "**Only if your bucket doesn't already exist**: Run the following cell to create your Cloud Storage bucket." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "NIq7R4HZCfIc" - }, - "outputs": [], - "source": [ - "! gsutil mb -l $REGION $BUCKET_NAME" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "ucvCsknMCims" - }, - "source": [ - "Finally, validate access to your Cloud Storage bucket by examining its contents:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "vhOb7YnwClBb" - }, - "outputs": [], - "source": [ - "! gsutil ls -al $BUCKET_NAME" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "XoEqT2Y4DJmf" - }, - "source": [ - "### Import libraries and define constants" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Y9Uo3tifg1kx" - }, - "source": [ - "Import required libraries.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "pRUOFELefqf1" - }, - "outputs": [], - "source": [ - "import pandas as pd\n", - "from google.cloud import aiplatform\n", - "from sklearn.metrics import mean_absolute_error, mean_squared_error\n", - "from tensorflow.python.keras.utils import data_utils" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "O8XJZB3gR8eL" - }, - "source": [ - "## Initialize Vertex AI and set an _experiment_\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "xtXZWmYqJ1bh" - }, - "source": [ - "Define experiment name." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "JIOrI-hoJ46P" - }, - "outputs": [], - "source": [ - "EXPERIMENT_NAME = \"\" # @param {type:\"string\"}" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "jWQLXXNVN4Lv" - }, - "source": [ - "If EXEPERIMENT_NAME is not set, set a default one below:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "Q1QInYWOKsmo" - }, - "outputs": [], - "source": [ - "if EXPERIMENT_NAME == \"\" or EXPERIMENT_NAME is None:\n", - " EXPERIMENT_NAME = \"my-experiment-\" + TIMESTAMP" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "DKIsYVjj56_X" - }, - "source": [ - "Initialize the *client* for Vertex AI." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "Wrlk2B2nJ7-X" - }, - "outputs": [], - "source": [ - "aiplatform.init(\n", - " project=PROJECT_ID,\n", - " location=REGION,\n", - " staging_bucket=BUCKET_NAME,\n", - " experiment=EXPERIMENT_NAME,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "6PlilQPFeS_h" - }, - "source": [ - "## Tracking parameters and metrics in Vertex AI custom training jobs" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "9nokDKBAxwV8" - }, - "source": [ - "This example uses the Abalone Dataset. For more information about this dataset please visit: https://archive.ics.uci.edu/ml/datasets/abalone" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "V_T10yTTqcS_" - }, - "outputs": [], - "source": [ - "!wget https://storage.googleapis.com/download.tensorflow.org/data/abalone_train.csv\n", - "!gsutil cp abalone_train.csv {BUCKET_NAME}/data/\n", - "\n", - "gcs_csv_path = f\"{BUCKET_NAME}/data/abalone_train.csv\"" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "35QVNhACqcTJ" - }, - "source": [ - "### Create a managed tabular dataset from a CSV\n", - "\n", - "A Managed dataset can be used to create an AutoML model or a custom model. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "4OfCqaYRqcTJ" - }, - "outputs": [], - "source": [ - "ds = aiplatform.TabularDataset.create(display_name=\"abalone\", gcs_source=[gcs_csv_path])\n", - "\n", - "ds.resource_name" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "VcEOYYolqcTN" - }, - "source": [ - "### Write the training script\n", - "\n", - "Run the following cell to create the training script that is used in the sample custom training job." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "OauJqJmJqcTO" - }, - "outputs": [], - "source": [ - "%%writefile training_script.py\n", - "\n", - "import pandas as pd\n", - "import argparse\n", - "import os\n", - "import tensorflow as tf\n", - "from tensorflow import keras\n", - "from tensorflow.keras import layers\n", - "\n", - "parser = argparse.ArgumentParser()\n", - "parser.add_argument('--epochs', dest='epochs',\n", - " default=10, type=int,\n", - " help='Number of epochs.')\n", - "parser.add_argument('--num_units', dest='num_units',\n", - " default=64, type=int,\n", - " help='Number of unit for first layer.')\n", - "args = parser.parse_args()\n", - "# uncomment and bump up replica_count for distributed training\n", - "# strategy = tf.distribute.experimental.MultiWorkerMirroredStrategy()\n", - "# tf.distribute.experimental_set_strategy(strategy)\n", - "\n", - "col_names = [\"Length\", \"Diameter\", \"Height\", \"Whole weight\", \"Shucked weight\", \"Viscera weight\", \"Shell weight\", \"Age\"]\n", - "target = \"Age\"\n", - "\n", - "def aip_data_to_dataframe(wild_card_path):\n", - " return pd.concat([pd.read_csv(fp.numpy().decode(), names=col_names)\n", - " for fp in tf.data.Dataset.list_files([wild_card_path])])\n", - "\n", - "def get_features_and_labels(df):\n", - " return df.drop(target, axis=1).values, df[target].values\n", - "\n", - "def data_prep(wild_card_path):\n", - " return get_features_and_labels(aip_data_to_dataframe(wild_card_path))\n", - "\n", - "\n", - "model = tf.keras.Sequential([layers.Dense(args.num_units), layers.Dense(1)])\n", - "model.compile(loss='mse', optimizer='adam')\n", - "\n", - "model.fit(*data_prep(os.environ[\"AIP_TRAINING_DATA_URI\"]),\n", - " epochs=args.epochs ,\n", - " validation_data=data_prep(os.environ[\"AIP_VALIDATION_DATA_URI\"]))\n", - "print(model.evaluate(*data_prep(os.environ[\"AIP_TEST_DATA_URI\"])))\n", - "\n", - "# save as Vertex AI Managed model\n", - "tf.saved_model.save(model, os.environ[\"AIP_MODEL_DIR\"])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Yp2clkOJSDhR" - }, - "source": [ - "### Launch a custom training job and track its trainig parameters on Vertex AI ML Metadata" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "btb6d48lqcTT" - }, - "outputs": [], - "source": [ - "job = aiplatform.CustomTrainingJob(\n", - " display_name=\"train-abalone-dist-1-replica\",\n", - " script_path=\"training_script.py\",\n", - " container_uri=\"gcr.io/cloud-aiplatform/training/tf-cpu.2-2:latest\",\n", - " requirements=[\"gcsfs==0.7.1\"],\n", - " model_serving_container_image_uri=\"gcr.io/cloud-aiplatform/prediction/tf2-cpu.2-2:latest\",\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "k_QorXXztzPH" - }, - "source": [ - "Start a new experiment run to track training parameters and start the training job. Note that this operation will take around 10 mins." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "oVTORjQpJ7-Y" - }, - "outputs": [], - "source": [ - "aiplatform.start_run(\"custom-training-run-1\") # Change this to your desired run name\n", - "parameters = {\"epochs\": 10, \"num_units\": 64}\n", - "aiplatform.log_params(parameters)\n", - "\n", - "model = job.run(\n", - " ds,\n", - " replica_count=1,\n", - " model_display_name=\"abalone-model\",\n", - " args=[f\"--epochs={parameters['epochs']}\", f\"--num_units={parameters['num_units']}\"],\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "5vhDsMJNqcTW" - }, - "source": [ - "### Deploy Model and calculate prediction metrics" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "O-uCOL3Naap4" - }, - "source": [ - "Deploy model to Google Cloud. This operation will take 10-20 mins." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "Y9GH72wWqcTX" - }, - "outputs": [], - "source": [ - "endpoint = model.deploy(machine_type=\"n1-standard-4\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "JY-5skFhasWs" - }, - "source": [ - "Once model is deployed, perform online prediction using the `abalone_test` dataset and calculate prediction metrics." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "saw50bqwa-dR" - }, - "source": [ - "Prepare the prediction dataset." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "ABZQmqsWISQv" - }, - "outputs": [], - "source": [ - "def read_data(uri):\n", - " dataset_path = data_utils.get_file(\"abalone_test.data\", uri)\n", - " col_names = [\n", - " \"Length\",\n", - " \"Diameter\",\n", - " \"Height\",\n", - " \"Whole weight\",\n", - " \"Shucked weight\",\n", - " \"Viscera weight\",\n", - " \"Shell weight\",\n", - " \"Age\",\n", - " ]\n", - " dataset = pd.read_csv(\n", - " dataset_path,\n", - " names=col_names,\n", - " na_values=\"?\",\n", - " comment=\"\\t\",\n", - " sep=\",\",\n", - " skipinitialspace=True,\n", - " )\n", - " return dataset\n", - "\n", - "\n", - "def get_features_and_labels(df):\n", - " target = \"Age\"\n", - " return df.drop(target, axis=1).values, df[target].values\n", - "\n", - "\n", - "test_dataset, test_labels = get_features_and_labels(\n", - " read_data(\n", - " \"https://storage.googleapis.com/download.tensorflow.org/data/abalone_test.csv\"\n", - " )\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "_HphZ38obJeB" - }, - "source": [ - "Perform online prediction." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "eXD-OvsrKmCt" - }, - "outputs": [], - "source": [ - "prediction = endpoint.predict(test_dataset.tolist())\n", - "prediction" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "TDKiv_O7bNwE" - }, - "source": [ - "Calculate and track prediction evaluation metrics." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "cj0fHucbKopn" - }, - "outputs": [], - "source": [ - "mse = mean_squared_error(test_labels, prediction.predictions)\n", - "mae = mean_absolute_error(test_labels, prediction.predictions)\n", - "\n", - "aiplatform.log_metrics({\"mse\": mse, \"mae\": mae})" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "CCGmesdIbbHf" - }, - "source": [ - "### Extract all parameters and metrics created during this experiment." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "KlcEBou-Pl4Z" - }, - "outputs": [], - "source": [ - "aiplatform.get_experiment_df()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "WTHvPMweMlP1" - }, - "source": [ - "### View data in the Cloud Console" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "F19_5lw0MqXv" - }, - "source": [ - "Parameters and metrics can also be viewed in the Cloud Console. \n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "GmN9vE9pqqzt" - }, - "outputs": [], - "source": [ - "print(\"Vertex AI Experiments:\")\n", - "print(\n", - " f\"https://console.cloud.google.com/ai/platform/experiments/experiments?folder=&organizationId=&project={PROJECT_ID}\"\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "TpV-iwP9qw9c" - }, - "source": [ - "## Cleaning up\n", - "\n", - "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n", - "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n", - "\n", - "Otherwise, you can delete the individual resources you created in this tutorial:\n", - "Training Job\n", - "Model\n", - "Cloud Storage Bucket\n", - "\n", - "* Training Job\n", - "* Model\n", - "* Endpoint\n", - "* Cloud Storage Bucket\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "rwPZoZISHhaY" - }, - "outputs": [], - "source": [ - "delete_training_job = True\n", - "delete_model = True\n", - "delete_endpoint = True\n", - "\n", - "# Warning: Setting this to true will delete everything in your bucket\n", - "delete_bucket = False\n", - "\n", - "# Delete the training job\n", - "job.delete()\n", - "\n", - "# Delete the model\n", - "model.delete()\n", - "\n", - "# Delete the endpoint\n", - "endpoint.delete()\n", - "\n", - "if delete_bucket and \"BUCKET_NAME\" in globals():\n", - " ! gsutil -m rm -r $BUCKET_NAME" - ] - } - ], - "metadata": { - "colab": { - "collapsed_sections": [], - "name": "sdk-metric-parameter-tracking-for-custom-jobs.ipynb", - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-custom-jobs.ipynb b/notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-custom-jobs.ipynb index 2f2dbe188..1f9a12ad8 100644 --- a/notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-custom-jobs.ipynb +++ b/notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-custom-jobs.ipynb @@ -8,7 +8,7 @@ }, "outputs": [], "source": [ - "# Copyright 2021 Google LLC\n", + "# Copyright 2022 Google LLC\n", "#\n", "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", "# you may not use this file except in compliance with the License.\n", @@ -42,6 +42,12 @@ " View on GitHub\n", " \n", " \n", + " \n", + " \n", + " \"Vertex\n", + " Open in Vertex AI Workbench\n", + " \n", + " \n", "" ] }, @@ -51,7 +57,7 @@ "id": "j9gUDU_3vV9d" }, "source": [ - "#Vertex AI: Track parameters and metrics for custom training jobs" + "# Vertex AI: Track parameters and metrics for custom training jobs" ] }, { @@ -69,7 +75,7 @@ "This example uses the Abalone Dataset. For more information about this dataset please visit: https://archive.ics.uci.edu/ml/datasets/abalone\n", "### Objective\n", "\n", - "In this notebook, you will learn how to use Vertex AI SDK to:\n", + "In this notebook, you will learn how to use Vertex AI SDK for Python to:\n", "\n", " * Track training parameters and prediction metrics for a custom training job.\n", " * Extract and perform analysis for all parameters and metrics within an Experiment.\n", @@ -97,7 +103,7 @@ "source": [ "### Set up your local development environment\n", "\n", - "**If you are using Colab or Vertex AI Workbench notebooks**, your environment already meets\n", + "**If you are using Colab or Vertex AI Workbench**, your environment already meets\n", "all the requirements to run this notebook. You can skip this step." ] }, @@ -146,7 +152,7 @@ "source": [ "### Install additional packages\n", "\n", - "Run the following commands to install the Vertex AI SDK and other packages used in this notebook." + "Install additional package dependencies not installed in your notebook environment." ] }, { @@ -168,15 +174,6 @@ " USER_FLAG = \"--user\"" ] }, - { - "cell_type": "markdown", - "metadata": { - "id": "W-eC8bOL4PcJ" - }, - "source": [ - "Install Vertex AI SDK." - ] - }, { "cell_type": "code", "execution_count": null, @@ -185,27 +182,9 @@ }, "outputs": [], "source": [ - "! pip install {USER_FLAG} --upgrade google-cloud-aiplatform" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "BN3NFbw64SzI" - }, - "source": [ - "Install tensorflow and sklearn for training and evaluation models." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "Nbio3QBp3_E-" - }, - "outputs": [], - "source": [ - "! pip install {USER_FLAG} --upgrade tensorflow sklearn" + "! pip3 install -U tensorflow $USER_FLAG\n", + "! python3 -m pip install {USER_FLAG} google-cloud-aiplatform --upgrade\n", + "! pip3 install scikit-learn {USER_FLAG}\n" ] }, { @@ -300,7 +279,7 @@ "\n", "# Get your Google Cloud project ID from gcloud\n", "if not os.getenv(\"IS_TESTING\"):\n", - " shell_output=!gcloud config list --format 'value(core.project)' 2>/dev/null\n", + " shell_output = !gcloud config list --format 'value(core.project)' 2>/dev/null\n", " PROJECT_ID = shell_output[0]\n", " print(\"Project ID: \", PROJECT_ID)" ] @@ -378,7 +357,7 @@ "source": [ "### Authenticate your Google Cloud account\n", "\n", - "**If you are using Vertex AI Workbench notebooks**, your environment is already\n", + "**If you are using Vertex AI Workbench**, your environment is already\n", "authenticated. Skip this step." ] }, @@ -401,9 +380,9 @@ "3. In the **Service account name** field, enter a name, and\n", " click **Create**.\n", "\n", - "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"AI Platform\"\n", + "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n", "into the filter box, and select\n", - " **AI Platform Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n", + " **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n", "\n", "5. Click *Create*. A JSON file that contains your key downloads to your\n", "local environment.\n", @@ -429,9 +408,7 @@ "# requests.\n", "\n", "# If on Google Cloud Notebooks, then don't execute this code\n", - "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n", - "\n", - "if not IS_GOOGLE_CLOUD_NOTEBOOK:\n", + "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n", " if \"google.colab\" in sys.modules:\n", " from google.colab import auth as google_auth\n", "\n", @@ -456,10 +433,10 @@ "\n", "\n", "When you submit a training job using the Cloud SDK, you upload a Python package\n", - "containing your training code to a Cloud Storage bucket. AI Platform runs\n", - "the code from this package. In this tutorial, AI Platform also saves the\n", + "containing your training code to a Cloud Storage bucket. Vertex AI runs\n", + "the code from this package. In this tutorial, Vertex AI also saves the\n", "trained model that results from your job in the same bucket. Using this model artifact, you can then\n", - "create AI Platform model and endpoint resources in order to serve\n", + "create Vertex AI model and endpoint resources in order to serve\n", "online predictions.\n", "\n", "Set the name of your Cloud Storage bucket below. It must be unique across all\n", @@ -468,7 +445,7 @@ "You may also change the `REGION` variable, which is used for operations\n", "throughout the rest of this notebook. Make sure to [choose a region where Vertex AI services are\n", "available](https://cloud.google.com/vertex-ai/docs/general/locations#available_regions). You may\n", - "not use a Multi-Regional Storage bucket for training with AI Platform." + "not use a Multi-Regional Storage bucket for training with Vertex AI." ] }, { @@ -479,7 +456,7 @@ }, "outputs": [], "source": [ - "BUCKET_NAME = \"gs://[your-bucket-name]\" # @param {type:\"string\"}\n", + "BUCKET_URI = \"gs://[your-bucket-name]\" # @param {type:\"string\"}\n", "REGION = \"[your-region]\" # @param {type:\"string\"}" ] }, @@ -491,8 +468,11 @@ }, "outputs": [], "source": [ - "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n", - " BUCKET_NAME = \"gs://\" + PROJECT_ID + \"-aip-\" + TIMESTAMP" + "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n", + " BUCKET_URI = \"gs://\" + PROJECT_ID + \"-aip-\" + TIMESTAMP\n", + "\n", + "if REGION == \"[your-region]\":\n", + " REGION = \"us-central1\"" ] }, { @@ -512,7 +492,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION $BUCKET_NAME" + "! gsutil mb -l $REGION $BUCKET_URI" ] }, { @@ -532,7 +512,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_NAME" + "! gsutil ls -al $BUCKET_URI" ] }, { @@ -637,7 +617,7 @@ "aiplatform.init(\n", " project=PROJECT_ID,\n", " location=REGION,\n", - " staging_bucket=BUCKET_NAME,\n", + " staging_bucket=BUCKET_URI,\n", " experiment=EXPERIMENT_NAME,\n", ")" ] @@ -669,9 +649,9 @@ "outputs": [], "source": [ "!wget https://storage.googleapis.com/download.tensorflow.org/data/abalone_train.csv\n", - "!gsutil cp abalone_train.csv {BUCKET_NAME}/data/\n", + "!gsutil cp abalone_train.csv {BUCKET_URI}/data/\n", "\n", - "gcs_csv_path = f\"{BUCKET_NAME}/data/abalone_train.csv\"" + "gcs_csv_path = f\"{BUCKET_URI}/data/abalone_train.csv\"" ] }, { @@ -760,7 +740,7 @@ " validation_data=data_prep(os.environ[\"AIP_VALIDATION_DATA_URI\"]))\n", "print(model.evaluate(*data_prep(os.environ[\"AIP_TEST_DATA_URI\"])))\n", "\n", - "# save as AI Platform Managed model\n", + "# save as Vertex AI Managed model\n", "tf.saved_model.save(model, os.environ[\"AIP_MODEL_DIR\"])" ] }, @@ -784,9 +764,9 @@ "job = aiplatform.CustomTrainingJob(\n", " display_name=\"train-abalone-dist-1-replica\",\n", " script_path=\"training_script.py\",\n", - " container_uri=\"gcr.io/cloud-aiplatform/training/tf-cpu.2-2:latest\",\n", + " container_uri=\"us-docker.pkg.dev/vertex-ai/training/tf-cpu.2-8:latest\",\n", " requirements=[\"gcsfs==0.7.1\"],\n", - " model_serving_container_image_uri=\"gcr.io/cloud-aiplatform/prediction/tf2-cpu.2-2:latest\",\n", + " model_serving_container_image_uri=\"us-docker.pkg.dev/vertex-ai/prediction/tf2-cpu.2-8:latest\",\n", ")" ] }, @@ -875,7 +855,7 @@ "outputs": [], "source": [ "def read_data(uri):\n", - " dataset_path = data_utils.get_file(\"auto-mpg.data\", uri)\n", + " dataset_path = data_utils.get_file(\"abalone_test.data\", uri)\n", " col_names = [\n", " \"Length\",\n", " \"Diameter\",\n", @@ -1021,8 +1001,9 @@ "Model\n", "Cloud Storage Bucket\n", "\n", - "* Training Job\n", - "* Model\n", + "* Vertex AI Dataset\n", + "* Training Job\n", + "* Model\n", "* Endpoint\n", "* Cloud Storage Bucket\n" ] @@ -1035,25 +1016,27 @@ }, "outputs": [], "source": [ - "delete_training_job = True\n", - "delete_model = True\n", - "delete_endpoint = True\n", - "\n", "# Warning: Setting this to true will delete everything in your bucket\n", "delete_bucket = False\n", "\n", + "# Delete dataset\n", + "ds.delete()\n", + "\n", "# Delete the training job\n", "job.delete()\n", "\n", - "# Undeploy and delete the endpoint\n", + "# Undeploy model from endpoint\n", "endpoint.undeploy_all()\n", + "\n", + "# Delete the endpoint\n", "endpoint.delete()\n", "\n", "# Delete the model\n", "model.delete()\n", "\n", - "if delete_bucket and \"BUCKET_NAME\" in globals():\n", - " ! gsutil -m rm -r $BUCKET_NAME" + "\n", + "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", + " ! gsutil -m rm -r $BUCKET_URI" ] } ],