Merge branch 'main' into model_monitoring_setup

* Changed prediction_output format from csv to jsonl to support generate_explanations

* ran linter test

* Made the changes as mentioned in the review

* Ran Linter Test

Co-authored-by: Andrew Ferlitsch <aferlitsch@google.com>

notebooks/community/ml_ops/stage6/get_started_with_tf_serving.ipynb

@@ -1214,6 +1214,20 @@
         "     [1.0,3.0,\"cat1\"],\n",
         "     [2.0,4.0,\"cat2\"]\n",
         "    ]}\n",
+        "    \n",
+        "**BigQuery**\n",
+        "\n",
+        "Each row is converted to a JSON array. For example:\n",
+        "\n",
+        "    [1.0,3.0,\"cat1\"]\n",
+        "    [2.0,4.0,\"cat2\"]\n",
+        "    \n",
+        "The batch server generates the pivot data with the same format. The generated pivot data is then wrapped into a payload request:\n",
+        "\n",
+        "    {\"instances\": [\n",
+        "     [1.0,3.0,\"cat1\"],\n",
+        "     [2.0,4.0,\"cat2\"]\n",
+        "    ]}\n",
         "\n",
         "**TFRecords**\n",
         "\n",

notebooks/community/prediction/custom_prediction_routines/SDK_Custom_Preprocess.ipynb

@@ -673,7 +673,7 @@
         "Before you can deploy your model for serving, Vertex AI needs access to the following files in Cloud Storage:\n",
         "\n",
         "* `model.joblib` (model artifact)\n",
-        "* `preprocessor.pkl` (model artifact)\n",
+        "* `preprocessor.pkl` (preprocessor code)\n",
         "\n",
         "Run the following commands to upload your files:"
       ]

notebooks/official/CODEOWNERS

@@ -10,7 +10,7 @@
 /tabnet/tabnet_vertex_tutorial.ipynb @longtle
 
 /migration @andrewferlitsch
-/explainabl_ai   
+/explainabl_ai
 /pipelines @andrewferlitsch
 /ml_metadata @andrewferlitsch
 /model_monitoring @andrewferlitsch
@@ -27,5 +27,6 @@
 /pipelines/google_cloud_pipelines_dataproc_tabular @inardini
 /automl/automl_forecasting_bqml_arima_plus_comparison.ipynb @TheMichaelHu
 /automl/automl_tabular_on_vertex_pipelines.ipynb @helinwang
-/custom/custom_training_tensorboard_profiler.ipynb @gericdong
-/workbench/spark/spark_sample_notebook.ipynb @bradmiro
+/custom/custom_training_tensorboard_profiler.ipynb @itseric
+/workbench/spark/spark_sample_notebook.ipynb @bmiro
+/model-registry/bqml-vertexai-model-registry.ipynb @soheilazangeneh

notebooks/official/experiments/comparing_pipeline_runs.ipynb

@@ -55,7 +55,7 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/experiments/comparing_pipeline_runs.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/experiments/comparing_pipeline_runs.ipynb\">\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
@@ -70,6 +70,8 @@
         "id": "tvgnzT1CKxrO"
       },
       "source": [
+        "# Compare pipeline runs with Vertex AI Experiments\n",
+        "\n",
         "## Overview\n",
         "\n",
         "Depending on the model life cycle of your data science team, you would like to experiment and track training Pipeline runs and its associated parameters. Then, you would to compare runs of these Pipelines to each others in order to figure out which is the best configuration generates the model you will register in the Vertex AI Model Registry."
@@ -207,10 +209,8 @@
         "if IS_WORKBENCH_NOTEBOOK:\n",
         "    USER_FLAG = \"--user\"\n",
         "\n",
-        "! pip install {USER_FLAG} --upgrade google-cloud-aiplatform \\\n",
-        "                                    google-cloud-storage \\\n",
-        "                                    google-auth \\\n",
-        "                                    kfp -q"
+        "!pip3 install {USER_FLAG} --force-reinstall 'google-cloud-aiplatform>=1.15' -q --no-warn-conflicts\n",
+        "!pip3 install {USER_FLAG} kfp -q --no-warn-conflicts"
       ]
     },
     {
@@ -270,7 +270,7 @@
         "\n",
         "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
         "\n",
-        "1. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com).\n",
+        "1. [Enable APIs](https://console.cloud.google.com/flows/enableapi?apiid=cloudresourcemanager.googleapis.com,aiplatform.googleapis.com).\n",
         "\n",
         "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
         "\n",
@@ -451,9 +451,14 @@
         "3. In the **Service account name** field, enter a name, and\n",
         "   click **Create**.\n",
         "\n",
-        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
-        "into the filter box, and select\n",
-        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type and select\n",
+        "the following role into the filter box:\n",
+        "\n",
+        "    *   Storage Admin\n",
+        "    *   Storage Object Admin\n",
+        "    *   Service Account User\n",
+        "    *   Vertex AI Administrator\n",
+        "\n",
         "\n",
         "5. Click *Create*. A JSON file that contains your key downloads to your\n",
         "local environment.\n",
@@ -720,6 +725,7 @@
         "import kfp.v2.dsl as dsl\n",
         "# Vertex AI\n",
         "from google.cloud import aiplatform as vertex_ai\n",
+        "from google.cloud.aiplatform_v1.types.pipeline_state import PipelineState\n",
         "from kfp.v2.dsl import Metrics, Model, Output, component"
       ]
     },
@@ -738,6 +744,7 @@
         "EXPERIMENT_NAME = f\"{PROJECT_ID}-{TASK}-{MODEL_TYPE}-{UUID}\"\n",
         "\n",
         "# Pipeline\n",
+        "PIPELINE_TEMPLATE_FILE = \"pipeline.json\"\n",
         "PIPELINE_URI = f\"{BUCKET_URI}/pipelines\"\n",
         "TRAIN_URI = f\"{BUCKET_URI}/iris/iris_data.csv\"\n",
         "LABEL_URI = f\"{BUCKET_URI}/iris/iris_target.csv\"\n",
@@ -819,9 +826,7 @@
       "source": [
         "Before you start running your pipeline experiments, you have to formalize your training as pipeline component.\n",
         "\n",
-        "To do that, you will use the `kfp.v2.dsl.component` decorator to convert your training task into a pipeline component.\n",
-        "\n",
-        "Training code will import required libraries to train,evaluate and save a model with mentioned features. "
+        "To do that, you build the pipeline by using the `kfp.v2.dsl.component` decorator to convert your training task into a pipeline component. "
       ]
     },
     {
@@ -1075,7 +1080,7 @@
         "\n",
         "    job = vertex_ai.PipelineJob(\n",
         "        display_name=f\"{EXPERIMENT_NAME}-pipeline-run-{i}\",\n",
-        "        template_path=\"pipeline.json\",\n",
+        "        template_path=PIPELINE_TEMPLATE_FILE,\n",
         "        pipeline_root=PIPELINE_URI,\n",
         "        parameter_values={\n",
         "            \"train_uri\": TRAIN_URI,\n",
@@ -1160,10 +1165,9 @@
       "source": [
         "# Get the PipelineJob resource using the experiment run name\n",
         "pipeline_experiments_df = vertex_ai.get_experiment_df(EXPERIMENT_NAME)\n",
-        "for i in range(5):\n",
-        "    job = vertex_ai.PipelineJob.get(pipeline_experiments_df.run_name[i])\n",
-        "    print(job.resource_name)\n",
-        "    print(job._dashboard_uri())"
+        "job = vertex_ai.PipelineJob.get(pipeline_experiments_df.run_name[0])\n",
+        "print(\"Pipeline job name: \", job.resource_name)\n",
+        "print(\"Pipeline Run UI link: \", job._dashboard_uri())"
       ]
     },
     {
@@ -1191,13 +1195,16 @@
       "outputs": [],
       "source": [
         "# Delete the pipeline\n",
-        "# Get the PipelineJob resource using the experiment run name\n",
-        "pipeline_experiments_df = vertex_ai.get_experiment_df(EXPERIMENT_NAME)\n",
-        "for i in range(5):\n",
-        "    job = vertex_ai.PipelineJob.get(pipeline_experiments_df.run_name[i])\n",
-        "    print(job.resource_name)\n",
-        "    print(job._dashboard_uri())\n",
-        "    job.delete()\n",
+        "while True:\n",
+        "    for i in range(0, len(runs)):\n",
+        "        pipeline_job = vertex_ai.PipelineJob.get(pipeline_experiments_df.run_name[i])\n",
+        "        if pipeline_job.state != PipelineState.PIPELINE_STATE_SUCCEEDED:\n",
+        "            print(\"Pipeline job is still running...\")\n",
+        "            time.sleep(60)\n",
+        "        else:\n",
+        "            print(\"Pipeline job is complete.\")\n",
+        "            pipeline_job.delete()\n",
+        "    break\n",
         "\n",
         "# Delete experiment\n",
         "exp = vertex_ai.Experiment(EXPERIMENT_NAME)\n",
@@ -1206,7 +1213,11 @@
         "# Delete bucket\n",
         "delete_bucket = False\n",
         "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
-        "    ! gsutil rm -rf {BUCKET_URI}"
+        "    ! gsutil rm -rf {BUCKET_URI}\n",
+        "\n",
+        "# Remove local files\n",
+        "\n",
+        "!rm {PIPELINE_TEMPLATE_FILE}"
       ]
     }
   ],

notebooks/official/explainable_ai/sdk_automl_tabular_binary_classification_batch_explain.ipynb

@@ -29,7 +29,7 @@
         "id": "title"
       },
       "source": [
-        "# Vertex SDK: AutoML training tabular binary classification model for batch explanation\n",
+        "# Vertex AI SDK: AutoML training tabular binary classification model for batch explanation\n",
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
@@ -65,17 +65,6 @@
         "This tutorial demonstrates how to use the Vertex AI SDK to create tabular binary classification models and do batch prediction with explanation using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) model."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "dataset:bank,lbn"
-      },
-      "source": [
-        "### Dataset\n",
-        "\n",
-        "The dataset used for this tutorial is the [Bank Marketing](https://pantheon.corp.google.com/storage/browser/_details/cloud-ml-tables-data/bank-marketing.csv) . This dataset does not require any feature engineering. The version of the dataset you will use in this tutorial is stored in a public Cloud Storage bucket."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -108,6 +97,17 @@
         "* Batch Prediction Service: Does a queued (batch) prediction for the entire set of instances in the background and stores the results in a Cloud Storage bucket when ready."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "7a4881cf39a4"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The dataset used for this tutorial is the [Bank Marketing](https://pantheon.corp.google.com/storage/browser/_details/cloud-ml-tables-data/bank-marketing.csv) . This dataset does not require any feature engineering. The version of the dataset you use in this tutorial is stored in a public Cloud Storage bucket."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -136,7 +136,7 @@
       "source": [
         "### Set up your local development environment\n",
         "\n",
-        "If you are using Colab or Vertex Workbench Notebook, your environment already meets all the requirements to run this notebook. You can skip this step.\n",
+        "If you are using Colab or Vertex AI Workbench Notebook, your environment already meets all the requirements to run this notebook. You can skip this step.\n",
         "\n",
         "Otherwise, make sure your environment meets this notebook's requirements. You need the following:\n",
         "\n",
@@ -322,7 +322,10 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
       ]
     },
     {
@@ -331,9 +334,8 @@
         "id": "timestamp"
       },
       "source": [
-        "#### Timestamp\n",
-        "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
+        "#### UUID\n",
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append the uuid onto the name of resources you create in this tutorial."
       ]
     },
     {
@@ -344,9 +346,16 @@
       },
       "outputs": [],
       "source": [
-        "from datetime import datetime\n",
+        "import random\n",
+        "import string\n",
         "\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
+        "\n",
+        "# Generate a uuid of length 8\n",
+        "def generate_uuid():\n",
+        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=8))\n",
+        "\n",
+        "\n",
+        "UUID = generate_uuid()"
       ]
     },
     {
@@ -357,7 +366,7 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Workbench AI Notebooks**, your environment is already\n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
         "authenticated. Skip this step.\n",
         "\n",
         "**If you are using Colab**, run the cell below and follow the instructions\n",
@@ -440,7 +449,7 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
+        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
         "BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
@@ -453,7 +462,7 @@
       "outputs": [],
       "source": [
         "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
         "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
       ]
     },
@@ -583,7 +592,7 @@
       "source": [
         "#### Quick peek at your data\n",
         "\n",
-        "You will use a version of the Bank Marketing dataset that is stored in a public Cloud Storage bucket, using a CSV index file.\n",
+        "You use a version of the Bank Marketing dataset that is stored in a public Cloud Storage bucket, using a CSV index file.\n",
         "\n",
         "Start by doing a quick peek at the data. You count the number of examples by counting the number of rows in the CSV index file  (`wc -l`) and then peek at the first few rows.\n",
         "\n",
@@ -637,7 +646,7 @@
       "outputs": [],
       "source": [
         "dataset = aip.TabularDataset.create(\n",
-        "    display_name=\"Bank Marketing\" + \"_\" + TIMESTAMP, gcs_source=[IMPORT_FILE]\n",
+        "    display_name=\"Bank Marketing\" + \"_\" + UUID, gcs_source=[IMPORT_FILE]\n",
         ")\n",
         "\n",
         "print(dataset.resource_name)"
@@ -688,7 +697,7 @@
       "outputs": [],
       "source": [
         "dag = aip.AutoMLTabularTrainingJob(\n",
-        "    display_name=\"bank_\" + TIMESTAMP,\n",
+        "    display_name=\"bank_\" + UUID,\n",
         "    optimization_prediction_type=\"classification\",\n",
         "    optimization_objective=\"minimize-log-loss\",\n",
         ")\n",
@@ -730,7 +739,7 @@
       "source": [
         "model = dag.run(\n",
         "    dataset=dataset,\n",
-        "    model_display_name=\"bank_\" + TIMESTAMP,\n",
+        "    model_display_name=\"bank_\" + UUID,\n",
         "    training_fraction_split=0.6,\n",
         "    validation_fraction_split=0.2,\n",
         "    test_fraction_split=0.2,\n",
@@ -784,7 +793,7 @@
       "source": [
         "### Make test items\n",
         "\n",
-        "You will use synthetic data as a test data items. Don't be concerned that we are using synthetic data -- we just want to demonstrate how to make a prediction."
+        "You use synthetic data as a test data items. Don't be concerned that we are using synthetic data."
       ]
     },
     {
@@ -852,11 +861,11 @@
       "outputs": [],
       "source": [
         "batch_predict_job = model.batch_predict(\n",
-        "    job_display_name=\"bank_\" + TIMESTAMP,\n",
+        "    job_display_name=\"bank_\" + UUID,\n",
         "    gcs_source=gcs_input_uri,\n",
         "    gcs_destination_prefix=BUCKET_URI,\n",
         "    instances_format=\"csv\",\n",
-        "    predictions_format=\"csv\",\n",
+        "    predictions_format=\"jsonl\",\n",
         "    generate_explanation=True,\n",
         "    sync=False,\n",
         ")\n",
@@ -952,6 +961,7 @@
         "# Set this to true only if you'd like to delete your bucket\n",
         "delete_bucket = False\n",
         "\n",
+        "dataset.delete()\n",
         "model.delete()\n",
         "batch_predict_job.delete()\n",
         "\n",

notebooks/official/model-registry/bqml-vertexai-model-registry.ipynb

@@ -0,0 +1,921 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ur8xi4C7S06n"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2022 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "JAPoU8Sm5E6e"
+      },
+      "source": [
+        "<table align=\"left\">\n",
+        "\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/bigquery-ml/bqml-vertexai-model-registry.ipynb\" target=\"_blank\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/bigquery-ml/bqml-vertexai-model-registry.ipynb\" target=\"_blank\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/bigquery-ml/bqml-vertexai-model-registry.ipynb\" target=\"_blank\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "      Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "  </td>                                                                                               \n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tvgnzT1CKxrO"
+      },
+      "source": [
+        "# Deploy BiqQuery ML Model on Vertex AI Model Registry and Make Predictions\n",
+        "\n",
+        "## Overview\n",
+        "\n",
+        "This tutorial demonstrates how to train a model with BigQuery ML and upload it on Vertex AI model registry, then make batch predictions.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "132a9ee68ba6"
+      },
+      "source": [
+        "### Objective\n",
+        "\n",
+        "In this tutorial, you learn how to use `Vertex AI Model Registry` with `BigQuery ML` and make batch predictions:\n",
+        "\n",
+        "This tutorial uses the following Google Cloud ML services and resources:\n",
+        "\n",
+        "- `Vertex AI Model Registry`\n",
+        "- `Vertex AI Model` resources \n",
+        "- `Vertex AI Endpoint` resources\n",
+        "- `Vertex AI Prediction`\n",
+        "- `BigQuery ML`\n",
+        "\n",
+        "\n",
+        "The steps performed include:\n",
+        "\n",
+        "- Train a model with `BQML`\n",
+        "- Upload the model to `Vertex AI Model Registry` \n",
+        "- Create a `Vertex AI Endpoint` resource\n",
+        "- Deploy the `Model` resource to the `Endpoint` resource\n",
+        "- Make `prediction` requests to the model endpoint\n",
+        "- Run `batch prediction` job on the `Model` resource \n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2de0477b10ce"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The dataset used for this tutorial is the Penguins dataset from <a href=\"https://cloud.google.com/bigquery/public-data\" target=\"_blank\">BigQuery public datasets</a>. This version of the dataset is used to predict the species of penguins from the available features like culmen-length, flipper-depth etc."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "76330e07673b"
+      },
+      "source": [
+        "### Costs \n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "* BigQuery ML\n",
+        "\n",
+        "Learn about <a href=\"https://cloud.google.com/vertex-ai/pricing\" target=\"_blank\">Vertex AI\n",
+        "pricing</a> and <a href=\"https://cloud.google.com/bigquery/pricing\" target=\"_blank\">BigQuery pricing</a>, and use the <a href=\"https://cloud.google.com/products/calculator/\" target=\"_blank\">Pricing\n",
+        "Calculator</a>\n",
+        "to generate a cost estimate based on your projected usage."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ze4-nDLfK4pw"
+      },
+      "source": [
+        "### Set up your local development environment\n",
+        "\n",
+        "**If you are using Colab or Vertex AI Workbench Notebooks**, your environment already meets\n",
+        "all the requirements to run this notebook. You can skip this step."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "gCuSR8GkAgzl"
+      },
+      "source": [
+        "**Otherwise**, make sure your environment meets this notebook's requirements.\n",
+        "You need the following:\n",
+        "\n",
+        "* The Google Cloud SDK\n",
+        "* Git\n",
+        "* Python 3\n",
+        "* virtualenv\n",
+        "* Jupyter notebook running in a virtual environment with Python 3\n",
+        "\n",
+        "The Google Cloud guide to <a href=\"https://cloud.google.com/python/setup\" target=\"_blank\">Setting up a Python development\n",
+        "environment</a> and the <a href=\"https://jupyter.org/install\" target=\"_blank\">Jupyter\n",
+        "installation guide</a> provide detailed instructions\n",
+        "for meeting these requirements. The following steps provide a condensed set of\n",
+        "instructions:\n",
+        "\n",
+        "1. <a href=\"https://cloud.google.com/sdk/docs/\" target=\"_blank\">Install and initialize the Cloud SDK.</a>\n",
+        "\n",
+        "1. <a href=\"https://cloud.google.com/python/setup#installing_python\" target=\"_blank\">Install Python 3.</a>\n",
+        "\n",
+        "1. <a href=\"https://cloud.google.com/python/setup#installing_and_using_virtualenv\" target=\"_blank\">Install\n",
+        "   virtualenv</a>\n",
+        "   and create a virtual environment that uses Python 3. Activate the virtual environment.\n",
+        "\n",
+        "1. To install Jupyter, run `pip3 install jupyter` on the\n",
+        "command-line in a terminal shell.\n",
+        "\n",
+        "1. To launch Jupyter, run `jupyter notebook` on the command-line in a terminal shell.\n",
+        "\n",
+        "1. Open this notebook in the Jupyter Notebook Dashboard."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "i7EUnXsZhAGF"
+      },
+      "source": [
+        "### Install additional packages\n",
+        "\n",
+        "Install the following packages required to execute this notebook. "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2b4ef9b72d43"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "# The Vertex AI Workbench Notebook product has specific requirements\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\")\n",
+        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
+        "    \"/opt/deeplearning/metadata/env_version\"\n",
+        ")\n",
+        "\n",
+        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "USER_FLAG = \"\"\n",
+        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\"\n",
+        "\n",
+        "! pip3 install --upgrade google-cloud-aiplatform {USER_FLAG} -q\n",
+        "! pip3 install --upgrade google-cloud-bigquery {USER_FLAG} -q"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "hhq5zEbGg0XX"
+      },
+      "source": [
+        "### Restart the kernel\n",
+        "\n",
+        "After you install the additional packages, you need to restart the notebook kernel so it can find the packages."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "EzrelQZ22IZj"
+      },
+      "outputs": [],
+      "source": [
+        "# Automatically restart kernel after installs\n",
+        "import os\n",
+        "\n",
+        "if not os.getenv(\"IS_TESTING\"):\n",
+        "    # Automatically restart kernel after installs\n",
+        "    import IPython\n",
+        "\n",
+        "    app = IPython.Application.instance()\n",
+        "    app.kernel.do_shutdown(True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "lWEdiXsJg0XY"
+      },
+      "source": [
+        "## Before you begin"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "BF1j6f9HApxa"
+      },
+      "source": [
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. <a href=\"https://console.cloud.google.com/cloud-resource-manager\" target=\"_blank\">Select or create a Google Cloud project</a>. When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "1. <a href=\"https://cloud.google.com/billing/docs/how-to/modify-project\" target=\"_blank\">Make sure that billing is enabled for your project</a>.\n",
+        "\n",
+        "1. <a href=\"https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,bigquery\" target=\"_blank\">Enable the Vertex AI and BigQuery APIs</a>. \n",
+        "\n",
+        "1. If you are running this notebook locally, you will need to install the <a href=\"https://cloud.google.com/sdk\" target=\"_blank\">Cloud SDK</a>.\n",
+        "\n",
+        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
+        "Cloud SDK uses the right project for all the commands in this notebook.\n",
+        "\n",
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "WReHDGG5g0XY"
+      },
+      "source": [
+        "#### Set your project ID\n",
+        "\n",
+        "**If you don't know your project ID**, you can get your project ID using `gcloud`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "o1AuQDpf_hS-"
+      },
+      "outputs": [],
+      "source": [
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "RYbBU1jXAETD"
+      },
+      "outputs": [],
+      "source": [
+        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
+        "    # Get your GCP project id from gcloud\n",
+        "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
+        "    PROJECT_ID = shell_output[0]\n",
+        "    print(\"Project ID:\", PROJECT_ID)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "set_gcloud_project_id"
+      },
+      "outputs": [],
+      "source": [
+        "! gcloud config set project $PROJECT_ID"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "region"
+      },
+      "source": [
+        "#### Region\n",
+        "\n",
+        "You can also change the `REGION` variable, which is used for operations\n",
+        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. We recommend that you choose the region closest to you.\n",
+        "\n",
+        "- Americas: `us-central1`\n",
+        "- Europe: `europe-west4`\n",
+        "- Asia Pacific: `asia-east1`\n",
+        "\n",
+        "You might not be able to use a multi-regional bucket for training with Vertex AI. Not all regions provide support for all Vertex AI services.\n",
+        "\n",
+        "Learn more about <a href=\"https://cloud.google.com/vertex-ai/docs/general/locations\" target=\"_blank\">Vertex AI regions</a>."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "vO3W8YdN2LuA"
+      },
+      "outputs": [],
+      "source": [
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "29e912d1b106"
+      },
+      "source": [
+        "#### UUID\n",
+        "\n",
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 2,
+      "metadata": {
+        "id": "c704897922c0"
+      },
+      "outputs": [],
+      "source": [
+        "import random\n",
+        "import string\n",
+        "\n",
+        "\n",
+        "# Generate a uuid of a specifed length(default=8)\n",
+        "def generate_uuid(length: int = 8) -> str:\n",
+        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
+        "\n",
+        "\n",
+        "UUID = generate_uuid()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dr--iN2kAylZ"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
+        "authenticated. Skip this step."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "sBCra4QMA2wR"
+      },
+      "source": [
+        "**If you are using Colab**, run the cell below and follow the instructions\n",
+        "when prompted to authenticate your account via oAuth.\n",
+        "\n",
+        "**Otherwise**, follow these steps:\n",
+        "\n",
+        "1. In the Cloud Console, go to the <a href=\"https://console.cloud.google.com/apis/credentials/serviceaccountkey\" target=\"_blank\">Create service account key page</a>.\n",
+        "\n",
+        "2. Click **Create service account**.\n",
+        "\n",
+        "3. In the **Service account name** field, enter a name, and\n",
+        "   click **Create**.\n",
+        "\n",
+        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
+        "into the filter box, and select\n",
+        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "\n",
+        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
+        "local environment.\n",
+        "\n",
+        "6. Enter the path to your service account key as the\n",
+        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "PyQmSRbKA8r-"
+      },
+      "outputs": [],
+      "source": [
+        "# If you are running this notebook in Colab, run this cell and follow the\n",
+        "# instructions to authenticate your GCP account. This provides access to your\n",
+        "# Cloud Storage bucket and lets you submit training jobs and prediction\n",
+        "# requests.\n",
+        "\n",
+        "import os\n",
+        "import sys\n",
+        "\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "IS_COLAB = \"google.colab\" in sys.modules\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
+        "    if \"google.colab\" in sys.modules:\n",
+        "        from google.colab import auth as google_auth\n",
+        "\n",
+        "        google_auth.authenticate_user()\n",
+        "\n",
+        "    # If you are running this notebook locally, replace the string below with the\n",
+        "    # path to your service account key and run this cell to authenticate your GCP\n",
+        "    # account.\n",
+        "    elif not os.getenv(\"IS_TESTING\"):\n",
+        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "XoEqT2Y4DJmf"
+      },
+      "source": [
+        "### Import libraries"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "pRUOFELefqf1"
+      },
+      "outputs": [],
+      "source": [
+        "import google.cloud.aiplatform as aiplatform\n",
+        "from google.cloud import bigquery"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "init_aip:mbsdk,all"
+      },
+      "source": [
+        "### Initialize Vertex AI and BigQuery SDKs for Python\n",
+        "\n",
+        "Initialize the Vertex AI and Big Query SDKs for Python for your project and corresponding bucket."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "BgaYKz2-2LuC"
+      },
+      "outputs": [],
+      "source": [
+        "aiplatform.init(project=PROJECT_ID, location=REGION)\n",
+        "bqclient = bigquery.Client(project=PROJECT_ID)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "lUEtpimzL17Z"
+      },
+      "source": [
+        "## BigQuery ML introduction\n",
+        "\n",
+        "BigQuery ML (BQML) provides the capability to train ML tabular models, such as classification and regression, in BigQuery using SQL syntax.\n",
+        "\n",
+        "Learn more about <a href=\"https://cloud.google.com/bigquery-ml/docs\" target=\"_blank\">BigQuery ML documentation</a>."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "30adf1b74bf9"
+      },
+      "source": [
+        "### BigQuery table used for training"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "3O7qlOGWNEU4"
+      },
+      "outputs": [],
+      "source": [
+        "# Define BigQuery table to be used for training\n",
+        "\n",
+        "BQ_TABLE = \"bigquery-public-data.ml_datasets.penguins\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "PKQD2e0eMg3M"
+      },
+      "source": [
+        "### Create BigQuery dataset resource"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "BnXOpvs2MmzF"
+      },
+      "source": [
+        "First, you create an empty dataset resource in your project."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "luqb-DBiMn-0"
+      },
+      "outputs": [],
+      "source": [
+        "BQ_DATASET_NAME = \"penguins\" + UUID\n",
+        "DATASET_QUERY = f\"\"\"CREATE SCHEMA {BQ_DATASET_NAME}\"\"\"\n",
+        "\n",
+        "job = bqclient.query(DATASET_QUERY)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "59bf85366baf"
+      },
+      "outputs": [],
+      "source": [
+        "job.result()\n",
+        "print(job.state)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "b-_8rZO8NIEb"
+      },
+      "source": [
+        "## Train BigQuery ML model and upload it to Vertex AI Model Registry\n",
+        "Next, you create and train a BQML tabular regression model from the public dataset penguins and store the model in your project `Vertex AI Model Registry` using the `CREATE MODEL` statement. The model configuration is specified in the `OPTIONS` statement as follows:\n",
+        "\n",
+        "- `model_type`: The type and archictecture of tabular model to train, e.g., LOGISTIC_REG.\n",
+        "\n",
+        "- `labels`: The column which are the labels.\n",
+        "\n",
+        "- `model_registry`: To register a BigQuery ML model to Vertex AI Model Registry, you must use `model_registry=\"vertex_ai\"`.\n",
+        "\n",
+        "Learn more about the <a href=\"https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-create\" target=\"_blank\">CREATE MODEL statement</a>.\n",
+        "\n",
+        "Learn more about <a href=\"https://cloud.google.com/bigquery-ml/docs/managing-models-vertex\" target=\"_blank\">Managing BigQuery ML models in the Vertex AI Model Registry</a>."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Q96rlZKRNPjU"
+      },
+      "outputs": [],
+      "source": [
+        "# Write the query to create Big Query ML model\n",
+        "\n",
+        "MODEL_NAME = \"penguins-lr\" + UUID\n",
+        "MODEL_QUERY = f\"\"\"\n",
+        "CREATE OR REPLACE MODEL `{BQ_DATASET_NAME}.{MODEL_NAME}`\n",
+        "OPTIONS(\n",
+        "    model_type='LOGISTIC_REG',\n",
+        "    labels = ['species'],\n",
+        "    model_registry='vertex_ai'\n",
+        "    )\n",
+        "AS\n",
+        "SELECT *\n",
+        "FROM `{BQ_TABLE}`\n",
+        "\"\"\""
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "eee158e2a375"
+      },
+      "source": [
+        "Create the BigQuery ML model using the query above and the BigQuery client that you created previously:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "LtdieY-BWILs"
+      },
+      "outputs": [],
+      "source": [
+        "# Run the model creation query using BigQuery client\n",
+        "\n",
+        "job = bqclient.query(MODEL_QUERY)\n",
+        "print(f\"Job state: {job.state}\\nJob Error:{job.errors}\")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "e4b007777e68"
+      },
+      "source": [
+        "Check the job status:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Y4C_3hTEXOE7"
+      },
+      "outputs": [],
+      "source": [
+        "job.result()\n",
+        "print(job.state)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5nHvVttrYfQ8"
+      },
+      "source": [
+        "### Find the model in the Vertex Model Registry\n",
+        "\n",
+        "You can use the `Vertex AI Model list()` method with a filter query to find the automatically registered model."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "E08IwUX5YpAG"
+      },
+      "outputs": [],
+      "source": [
+        "model = aiplatform.Model(model_name=MODEL_NAME)\n",
+        "\n",
+        "print(model.gca_resource)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "zgvXXlaZaYw3"
+      },
+      "source": [
+        "## Deploy Vertex AI Model resource to a Vertex AI Endpoint resource\n",
+        "You must deploy a model to an `endpoint` before that model can be used to serve online predictions; deploying a model associates physical resources with the model so it can serve online predictions with low latency. \n",
+        "\n",
+        "Learn more about <a href=\"https://cloud.google.com/vertex-ai/docs/predictions/deploy-model-api#aiplatform_deploy_model_custom_trained_model_sample-python\" target=\"_blank\">Deploy a model using the Vertex AI API</a>\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "N27z-_by5gti"
+      },
+      "source": [
+        "### Create a Vertex AI Endpoint resource\n",
+        "\n",
+        "If you are deploying a model to an existing endpoint, you can skip this cell.\n",
+        "\n",
+        "- `display_name`: Display name for the endpoint.\n",
+        "- `project`: The project ID on which you are creating an endpoint.\n",
+        "- `location`: The region where you are using Vertex AI."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "XmmyCtW055Ya"
+      },
+      "outputs": [],
+      "source": [
+        "ENDPOINT_DISPLAY_NAME = \"bqml-lr-model-endpoint\" + UUID\n",
+        "\n",
+        "endpoint = aiplatform.Endpoint.create(\n",
+        "    display_name=ENDPOINT_DISPLAY_NAME,\n",
+        "    project=PROJECT_ID,\n",
+        "    location=REGION,\n",
+        ")\n",
+        "\n",
+        "print(endpoint.display_name)\n",
+        "print(endpoint.resource_name)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "xSCIKdFo56YO"
+      },
+      "source": [
+        "### Deploy the Vertex AI Model resource to Vertex AI Endpoint resource"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "f7KfDgALE4aD"
+      },
+      "outputs": [],
+      "source": [
+        "DEPLOYED_NAME = \"bqml-lr-penguins\"\n",
+        "\n",
+        "model.deploy(endpoint=endpoint, deployed_model_display_name=DEPLOYED_NAME)\n",
+        "\n",
+        "print(model.display_name)\n",
+        "print(model.resource_name)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "H3K6SVplJ9Mg"
+      },
+      "source": [
+        "## Send prediction request to the Vertex AI Endpoint resource\n",
+        "\n",
+        "Now that your Vertex AI Model resource is deployed to a Vertex AI `Endpoint` resource, you can do online predictions by sending prediction requests to the `Endpoint` resource.\n",
+        "\n",
+        "Learn more about <a href=\"https://cloud.google.com/vertex-ai/docs/predictions/online-predictions-custom-models\" target=\"_blank\">Get online predictions from custom-trained models</a>"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2j7ioB3VKEtx"
+      },
+      "outputs": [],
+      "source": [
+        "instance = {\n",
+        "    \"island\": \"Dream\",\n",
+        "    \"culmen_length_mm\": 36.6,\n",
+        "    \"culmen_depth_mm\": 18.4,\n",
+        "    \"flipper_length_mm\": 184.0,\n",
+        "    \"body_mass_g\": 3475.0,\n",
+        "    \"sex\": \"FEMALE\",\n",
+        "}\n",
+        "\n",
+        "prediction = endpoint.predict([instance])\n",
+        "print(prediction)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "C39qOaBHZI1G"
+      },
+      "source": [
+        "## Batch Prediction on the BQML model"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "UBffk3GyaPY3"
+      },
+      "source": [
+        "Here you request batch predictions directly from the BigQuery ML model; you don't need to deploy the model to an endpoint. For data types that support both batch and online predictions, use batch predictions when you don't require an immediate response and want to process accumulated data by using a single request.\n",
+        "\n",
+        "Learn more abount <a href=\"https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-predict\" target=\"_blank\">The ML.PREDICT function</a>"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "_QxZb19_o6jx"
+      },
+      "outputs": [],
+      "source": [
+        "sql_ml_predict = f\"\"\"SELECT * FROM ML.PREDICT(MODEL `{PROJECT_ID}.{BQ_DATASET_NAME}.{MODEL_NAME}`, \n",
+        "(SELECT\n",
+        "      *\n",
+        "    FROM\n",
+        "      `{BQ_TABLE}` LIMIT 10))\"\"\"\n",
+        "\n",
+        "job = bqclient.query(sql_ml_predict)\n",
+        "prediction_result = job.result().to_arrow().to_pandas()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "TVpsLI5nrVii"
+      },
+      "outputs": [],
+      "source": [
+        "display(prediction_result.head())"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "TpV-iwP9qw9c"
+      },
+      "source": [
+        "## Cleaning up\n",
+        "\n",
+        "To clean up all Google Cloud resources used in this project, you can <a href=\"https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects\" target=\"_blank\">delete the Google Cloud\n",
+        "project</a> you used for the tutorial.\n",
+        "\n",
+        "Learn more about <a href=\"https://cloud.google.com/bigquery-ml/docs/managing-models-vertex\" target=\"_blank\">Deleting BigQuery ML models from Vertex AI Model Registry</a>\n",
+        "\n",
+        "Otherwise, you can delete the individual resources you created in this tutorial."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "sx_vKniMq9ZX"
+      },
+      "outputs": [],
+      "source": [
+        "# Delete the endpoint using the Vertex endpoint object\n",
+        "endpoint.undeploy_all()\n",
+        "endpoint.delete()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "3c5c8dc2f597"
+      },
+      "outputs": [],
+      "source": [
+        "# Delete BigQuery ML model\n",
+        "\n",
+        "delete_query = f\"\"\"DROP MODEL `{PROJECT_ID}.{BQ_DATASET_NAME}.{MODEL_NAME}`\"\"\"\n",
+        "job = bqclient.query(delete_query)\n",
+        "job.result()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "85dfc88f5472"
+      },
+      "outputs": [],
+      "source": [
+        "# Delete the created BigQuery dataset\n",
+        "! bq rm -r -f $PROJECT_ID:$BQ_DATASET_NAME"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "collapsed_sections": [],
+      "name": "bqml-vertexai-model-registry.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/official/pipelines/automl_tabular_classification_beans.ipynb

@@ -64,22 +64,11 @@
         "\n",
         "This notebook shows how to use the components defined in [`google_cloud_pipeline_components`](https://github.com/kubeflow/pipelines/tree/master/components/google-cloud) to build an AutoML tabular classification workflow on [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines).\n",
         "\n",
-        "You'll build a pipeline that looks like this:\n",
+        "You build a pipeline in this notebook that looks like this:\n",
         "\n",
         "<a href=\"https://storage.googleapis.com/amy-jo/images/mp/beans.png\" target=\"_blank\"><img src=\"https://storage.googleapis.com/amy-jo/images/mp/beans.png\" width=\"95%\"/></a>"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "dataset:beans,lcn"
-      },
-      "source": [
-        "### Dataset\n",
-        "\n",
-        "The dataset used for this tutorial is the UCI Machine Learning ['Dry beans dataset'](https://archive.ics.uci.edu/ml/datasets/Dry+Bean+Dataset), from: KOKLU, M. and OZKAN, I.A., (2020), \"Multiclass Classification of Dry Beans Using Computer Vision and Machine Learning Techniques.\"In Computers and Electronics in Agriculture, 174, 105507. [DOI](https://doi.org/10.1016/j.compag.2020.105507)."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -96,8 +85,8 @@
         "- `Vertex AI Pipelines`\n",
         "- `Google Cloud Pipeline Components`\n",
         "- `Vertex AutoML`\n",
-        "- `Vertex AI Model` resource\n",
-        "- `Vertex AI Endpoint` resource\n",
+        "- `Vertex AI Model`\n",
+        "- `Vertex AI Endpoint`\n",
         "\n",
         "The steps performed include:\n",
         "\n",
@@ -112,6 +101,17 @@
         "The components are [documented here](https://google-cloud-pipeline-components.readthedocs.io/en/latest/google_cloud_pipeline_components.aiplatform.html#module-google_cloud_pipeline_components.aiplatform)."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dataset:beans,lcn"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The dataset used for this tutorial is the UCI Machine Learning ['Dry beans dataset'](https://archive.ics.uci.edu/ml/datasets/Dry+Bean+Dataset), from: KOKLU, M. and OZKAN, I.A., (2020), \"Multiclass Classification of Dry Beans Using Computer Vision and Machine Learning Techniques.\"In Computers and Electronics in Agriculture, 174, 105507. [DOI](https://doi.org/10.1016/j.compag.2020.105507)."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -140,7 +140,7 @@
       "source": [
         "### Set up your local development environment\n",
         "\n",
-        "If you are using Colab or Vertex AI Workbench Notebook, your environment already meets all the requirements to run this notebook. You can skip this step.\n",
+        "If you are using Colab or Vertex AI Workbench Notebook, your environment already meets all the requirements to run this notebook.\n",
         "\n",
         "Otherwise, make sure your environment meets this notebook's requirements. You need the following:\n",
         "\n",
@@ -197,9 +197,10 @@
         "if IS_WORKBENCH_NOTEBOOK:\n",
         "    USER_FLAG = \"--user\"\n",
         "\n",
-        "! pip3 install --upgrade google-cloud-aiplatform {USER_FLAG} -q\n",
-        "! pip3 install -U google-cloud-storage $USER_FLAG -q\n",
-        "! pip3 install $USER kfp google-cloud-pipeline-components --upgrade -q"
+        "! pip3 install --upgrade {USER_FLAG} google-cloud-aiplatform \\\n",
+        "                                    google-cloud-storage \\\n",
+        "                                    kfp \\\n",
+        "                                    google-cloud-pipeline-components -q"
       ]
     },
     {
@@ -237,6 +238,8 @@
         "id": "check_versions"
       },
       "source": [
+        "### Check the package versions\n",
+        "\n",
         "Check the versions of the packages you installed.  The KFP SDK version should be >=1.8."
       ]
     },
@@ -282,6 +285,17 @@
         "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$`."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "af794e75b7e3"
+      },
+      "source": [
+        "#### Set your project ID\n",
+        "\n",
+        "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -290,7 +304,7 @@
       },
       "outputs": [],
       "source": [
-        "PROJECT_ID = \"python-docs-samples-tests\"  # @param {type:\"string\"}"
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -301,7 +315,7 @@
       },
       "outputs": [],
       "source": [
-        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"python-docs-samples-tests\":\n",
+        "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
         "    # Get your GCP project id from gcloud\n",
         "    shell_output = ! gcloud config list --format 'value(core.project)' 2>/dev/null\n",
         "    PROJECT_ID = shell_output[0]\n",
@@ -347,7 +361,10 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
       ]
     },
     {
@@ -356,9 +373,9 @@
         "id": "timestamp"
       },
       "source": [
-        "#### Timestamp\n",
+        "#### UUID\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
       ]
     },
     {
@@ -369,9 +386,16 @@
       },
       "outputs": [],
       "source": [
-        "from datetime import datetime\n",
+        "import random\n",
+        "import string\n",
         "\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
+        "\n",
+        "# Generate a uuid of a specifed length(default=8)\n",
+        "def generate_uuid(length: int = 8) -> str:\n",
+        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
+        "\n",
+        "\n",
+        "UUID = generate_uuid()"
       ]
     },
     {
@@ -382,7 +406,7 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Vertex AI Workbench Notebook**, your environment is already authenticated. Skip this step.\n",
+        "**If you are using Vertex AI Workbench Notebook**, your environment is already authenticated.\n",
         "\n",
         "**If you are using Colab**, run the cell below and follow the instructions\n",
         "when prompted to authenticate your account via oAuth.\n",
@@ -477,8 +501,8 @@
       "outputs": [],
       "source": [
         "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
-        "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
+        "    BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
     {
@@ -599,9 +623,6 @@
         "id": "setup_vars"
       },
       "source": [
-        "### Set up variables\n",
-        "\n",
-        "Next, set up some variables used throughout the tutorial.\n",
         "### Import libraries and define constants"
       ]
     },
@@ -613,7 +634,13 @@
       },
       "outputs": [],
       "source": [
-        "import google.cloud.aiplatform as aip"
+        "from typing import NamedTuple\n",
+        "\n",
+        "import kfp\n",
+        "from google.cloud import aiplatform\n",
+        "from kfp.v2 import dsl\n",
+        "from kfp.v2.dsl import (Artifact, ClassificationMetrics, Input, Metrics,\n",
+        "                        Output, component)"
       ]
     },
     {
@@ -624,9 +651,9 @@
       "source": [
         "#### Vertex AI constants\n",
         "\n",
-        "Setup up the following constants for Vertex AI:\n",
-        "\n",
-        "- `API_ENDPOINT`: The Vertex AI API service endpoint for `Dataset`, `Model`, `Job`, `Pipeline` and `Endpoint` services."
+        "Setup up the following constants for Vertex AI Pipeline:\n",
+        "- `PIPELINE_NAME`: Set name for the Pipeline.\n",
+        "- `PIPELINE_ROOT`: Cloud Storage bucket path to store pipeline artifacts."
       ]
     },
     {
@@ -637,65 +664,18 @@
       },
       "outputs": [],
       "source": [
-        "# API service endpoint\n",
-        "API_ENDPOINT = \"{}-aiplatform.googleapis.com\".format(REGION)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "pipeline_constants"
-      },
-      "source": [
-        "#### Vertex AI Pipelines constants\n",
-        "\n",
-        "Setup up the following constants for Vertex AI Pipelines:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "pipeline_constants"
-      },
-      "outputs": [],
-      "source": [
+        "# set path for storing the pipeline artifacts\n",
+        "PIPELINE_NAME = \"automl-tabular-beans-training\"\n",
         "PIPELINE_ROOT = \"{}/pipeline_root/beans\".format(BUCKET_URI)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "additional_imports"
-      },
-      "source": [
-        "Additional imports."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "import_pipelines"
-      },
-      "outputs": [],
-      "source": [
-        "from typing import NamedTuple\n",
-        "\n",
-        "import kfp\n",
-        "from google_cloud_pipeline_components import aiplatform as gcc_aip\n",
-        "from kfp.v2 import dsl\n",
-        "from kfp.v2.dsl import (Artifact, ClassificationMetrics, Input, Metrics,\n",
-        "                        Output, component)"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "init_aip:mbsdk"
       },
       "source": [
-        "## Initialize Vertex AI SDK for Python\n",
+        "### Initialize Vertex AI SDK for Python\n",
         "\n",
         "Initialize the Vertex AI SDK for Python for your project and corresponding bucket."
       ]
@@ -708,7 +688,7 @@
       },
       "outputs": [],
       "source": [
-        "aip.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
+        "aiplatform.init(project=PROJECT_ID, staging_bucket=BUCKET_URI)"
       ]
     },
     {
@@ -751,8 +731,7 @@
         ")\n",
         "def classification_model_eval_metrics(\n",
         "    project: str,\n",
-        "    location: str,  # \"us-central1\",\n",
-        "    api_endpoint: str,  # \"us-central1-aiplatform.googleapis.com\",\n",
+        "    location: str,\n",
         "    thresholds_dict_str: str,\n",
         "    model: Input[Artifact],\n",
         "    metrics: Output[Metrics],\n",
@@ -762,20 +741,21 @@
         "    import json\n",
         "    import logging\n",
         "\n",
-        "    from google.cloud import aiplatform as aip\n",
+        "    from google.cloud import aiplatform\n",
+        "\n",
+        "    aiplatform.init(project=project)\n",
         "\n",
         "    # Fetch model eval info\n",
-        "    def get_eval_info(client, model_name):\n",
-        "        from google.protobuf.json_format import MessageToDict\n",
-        "\n",
-        "        response = client.list_model_evaluations(parent=model_name)\n",
+        "    def get_eval_info(model):\n",
+        "        response = model.list_model_evaluations()\n",
         "        metrics_list = []\n",
         "        metrics_string_list = []\n",
         "        for evaluation in response:\n",
+        "            evaluation = evaluation.to_dict()\n",
         "            print(\"model_evaluation\")\n",
-        "            print(\" name:\", evaluation.name)\n",
-        "            print(\" metrics_schema_uri:\", evaluation.metrics_schema_uri)\n",
-        "            metrics = MessageToDict(evaluation._pb.metrics)\n",
+        "            print(\" name:\", evaluation[\"name\"])\n",
+        "            print(\" metrics_schema_uri:\", evaluation[\"metricsSchemaUri\"])\n",
+        "            metrics = evaluation[\"metrics\"]\n",
         "            for metric in metrics.keys():\n",
         "                logging.info(\"metric: %s, value: %s\", metric, metrics[metric])\n",
         "            metrics_str = json.dumps(metrics)\n",
@@ -783,7 +763,7 @@
         "            metrics_string_list.append(metrics_str)\n",
         "\n",
         "        return (\n",
-        "            evaluation.name,\n",
+        "            evaluation[\"name\"],\n",
         "            metrics_list,\n",
         "            metrics_string_list,\n",
         "        )\n",
@@ -832,20 +812,18 @@
         "            if metric != \"confidenceMetrics\":\n",
         "                val_string = json.dumps(metrics_list[0][metric])\n",
         "                metrics.log_metric(metric, val_string)\n",
-        "        # metrics.metadata[\"model_type\"] = \"AutoML Tabular classification\"\n",
         "\n",
         "    logging.getLogger().setLevel(logging.INFO)\n",
-        "    aip.init(project=project)\n",
+        "\n",
         "    # extract the model resource name from the input Model Artifact\n",
         "    model_resource_path = model.metadata[\"resourceName\"]\n",
         "    logging.info(\"model path: %s\", model_resource_path)\n",
         "\n",
-        "    client_options = {\"api_endpoint\": api_endpoint}\n",
-        "    # Initialize client that will be used to create and send requests.\n",
-        "    client = aip.gapic.ModelServiceClient(client_options=client_options)\n",
-        "    eval_name, metrics_list, metrics_str_list = get_eval_info(\n",
-        "        client, model_resource_path\n",
-        "    )\n",
+        "    # Get the trained model resource\n",
+        "    model = aiplatform.Model(model_resource_path)\n",
+        "\n",
+        "    # Get model evaluation metrics from the the trained model\n",
+        "    eval_name, metrics_list, metrics_str_list = get_eval_info(model)\n",
         "    logging.info(\"got evaluation name: %s\", eval_name)\n",
         "    logging.info(\"got metrics list: %s\", metrics_list)\n",
         "    log_metrics(metrics_list, metricsc)\n",
@@ -867,7 +845,9 @@
         "id": "define_pipeline:gcpc,beans,lcn"
       },
       "source": [
-        "## Define an AutoML tabular classification pipeline that uses components from `google_cloud_pipeline_components`"
+        "## Define pipeline \n",
+        "\n",
+        "Define the pipeline for AutoML tabular classification using the components from `google_cloud_pipeline_components`."
       ]
     },
     {
@@ -878,30 +858,34 @@
       },
       "outputs": [],
       "source": [
-        "DISPLAY_NAME = \"automl-beans{}\".format(TIMESTAMP)\n",
-        "PIPELINE_NAME = \"automl-tabular-beans-training-v2\"\n",
-        "MACHINE_TYPE = \"n1-standard-4\"\n",
-        "\n",
-        "\n",
         "@kfp.dsl.pipeline(name=PIPELINE_NAME, pipeline_root=PIPELINE_ROOT)\n",
         "def pipeline(\n",
-        "    bq_source: str = \"bq://aju-dev-demos.beans.beans1\",\n",
-        "    display_name: str = DISPLAY_NAME,\n",
-        "    project: str = PROJECT_ID,\n",
-        "    gcp_region: str = REGION,\n",
-        "    api_endpoint: str = API_ENDPOINT,\n",
-        "    thresholds_dict_str: str = '{\"auRoc\": 0.95}',\n",
+        "    bq_source: str,\n",
+        "    DATASET_DISPLAY_NAME: str,\n",
+        "    TRAINING_DISPLAY_NAME: str,\n",
+        "    MODEL_DISPLAY_NAME: str,\n",
+        "    ENDPOINT_DISPLAY_NAME: str,\n",
+        "    MACHINE_TYPE: str,\n",
+        "    project: str,\n",
+        "    gcp_region: str,\n",
+        "    thresholds_dict_str: str,\n",
         "):\n",
-        "    dataset_create_op = gcc_aip.TabularDatasetCreateOp(\n",
-        "        project=project, display_name=display_name, bq_source=bq_source\n",
+        "\n",
+        "    from google_cloud_pipeline_components.aiplatform import (\n",
+        "        AutoMLTabularTrainingJobRunOp, EndpointCreateOp, ModelDeployOp,\n",
+        "        TabularDatasetCreateOp)\n",
+        "\n",
+        "    dataset_create_op = TabularDatasetCreateOp(\n",
+        "        project=project, display_name=DATASET_DISPLAY_NAME, bq_source=bq_source\n",
         "    )\n",
         "\n",
-        "    training_op = gcc_aip.AutoMLTabularTrainingJobRunOp(\n",
+        "    training_op = AutoMLTabularTrainingJobRunOp(\n",
         "        project=project,\n",
-        "        display_name=display_name,\n",
+        "        display_name=TRAINING_DISPLAY_NAME,\n",
         "        optimization_prediction_type=\"classification\",\n",
         "        optimization_objective=\"minimize-log-loss\",\n",
         "        budget_milli_node_hours=1000,\n",
+        "        model_display_name=MODEL_DISPLAY_NAME,\n",
         "        column_specs={\n",
         "            \"Area\": \"numeric\",\n",
         "            \"Perimeter\": \"numeric\",\n",
@@ -924,10 +908,10 @@
         "        dataset=dataset_create_op.outputs[\"dataset\"],\n",
         "        target_column=\"Class\",\n",
         "    )\n",
+        "\n",
         "    model_eval_task = classification_model_eval_metrics(\n",
         "        project,\n",
         "        gcp_region,\n",
-        "        api_endpoint,\n",
         "        thresholds_dict_str,\n",
         "        training_op.outputs[\"model\"],\n",
         "    )\n",
@@ -937,13 +921,13 @@
         "        name=\"deploy_decision\",\n",
         "    ):\n",
         "\n",
-        "        endpoint_op = gcc_aip.EndpointCreateOp(\n",
+        "        endpoint_op = EndpointCreateOp(\n",
         "            project=project,\n",
         "            location=gcp_region,\n",
-        "            display_name=\"train-automl-beans\",\n",
+        "            display_name=ENDPOINT_DISPLAY_NAME,\n",
         "        )\n",
         "\n",
-        "        gcc_aip.ModelDeployOp(\n",
+        "        ModelDeployOp(\n",
         "            model=training_op.outputs[\"model\"],\n",
         "            endpoint=endpoint_op.outputs[\"endpoint\"],\n",
         "            dedicated_resources_min_replica_count=1,\n",
@@ -960,7 +944,7 @@
       "source": [
         "## Compile the pipeline\n",
         "\n",
-        "Next, compile the pipeline."
+        "Next, compile the pipeline to the specified json file."
       ]
     },
     {
@@ -971,11 +955,11 @@
       },
       "outputs": [],
       "source": [
-        "from kfp.v2 import compiler  # noqa: F811\n",
+        "from kfp.v2 import compiler\n",
         "\n",
         "compiler.Compiler().compile(\n",
         "    pipeline_func=pipeline,\n",
-        "    package_path=\"tabular classification_pipeline.json\".replace(\" \", \"_\"),\n",
+        "    package_path=\"tabular_classification_pipeline.json\",\n",
         ")"
       ]
     },
@@ -987,7 +971,53 @@
       "source": [
         "## Run the pipeline\n",
         "\n",
-        "Next, run the pipeline."
+        "Next, pass the input parameters required for the pipeline and run it. The defined pipeline takes the following parameters:\n",
+        "\n",
+        "- `bq_source`: BigQuery source for the tabular dataset.\n",
+        "- `DATASET_DISPLAY_NAME`: Display name for the Vertex AI managed dataset.\n",
+        "- `TRAINIG_DISPLAY_NAME`: Display name for the AutoML Training job.\n",
+        "- `MODEL_DISPLAY_NAME`: Display name for the Vertex AI Model generated as a result of the training job.\n",
+        "- `ENDPOINT_DISPLAY_NAME`: Display name for the Vertex AI Endpoint where the model is deployed.\n",
+        "- `MACHINE_TYPE`: Machine type for the serving container.\n",
+        "- `project`: Project-id where the pipeline is run.\n",
+        "- `gcp_region`: Region for setting the pipeline location.\n",
+        "- `thresholds_dict_str`: dictionary of thresholds based on which the model deployment is conditioned.\n",
+        "- `pipeline_root`:  To override the pipeline root path specified in the pipeline job's definition, specify a path that your pipeline job can access, such as a Cloud Storage bucket URI."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "1adf9b056954"
+      },
+      "outputs": [],
+      "source": [
+        "# Set the display-names for Vertex AI resources\n",
+        "PIPELINE_DISPLAY_NAME = \"[your-pipeline-display-name]\"  # @param {type:\"string\"}\n",
+        "DATASET_DISPLAY_NAME = \"[your-dataset-display-name]\"  # @param {type:\"string\"}\n",
+        "MODEL_DISPLAY_NAME = \"[your-model-display-name]\"  # @param {type:\"string\"}\n",
+        "TRAINING_DISPLAY_NAME = \"[your-training-job-display-name]\"  # @param {type:\"string\"}\n",
+        "ENDPOINT_DISPLAY_NAME = \"[your-endpoint-display-name]\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# Otherwise, use the default display-names\n",
+        "if PIPELINE_DISPLAY_NAME == \"[your-pipeline-display-name]\":\n",
+        "    PIPELINE_DISPLAY_NAME = f\"pipeline_beans_{UUID}\"\n",
+        "\n",
+        "if DATASET_DISPLAY_NAME == \"[your-dataset-display-name]\":\n",
+        "    DATASET_DISPLAY_NAME = f\"dataset_beans_{UUID}\"\n",
+        "\n",
+        "if MODEL_DISPLAY_NAME == \"[your-model-display-name]\":\n",
+        "    MODEL_DISPLAY_NAME = f\"model_beans_{UUID}\"\n",
+        "\n",
+        "if TRAINING_DISPLAY_NAME == \"[your-training-job-display-name]\":\n",
+        "    TRAINING_DISPLAY_NAME = f\"automl_training_beans_{UUID}\"\n",
+        "\n",
+        "if ENDPOINT_DISPLAY_NAME == \"[your-endpoint-display-name]\":\n",
+        "    ENDPOINT_DISPLAY_NAME = f\"endpoint_beans_{UUID}\"\n",
+        "\n",
+        "# Set machine type\n",
+        "MACHINE_TYPE = \"n1-standard-4\""
       ]
     },
     {
@@ -998,19 +1028,24 @@
       },
       "outputs": [],
       "source": [
-        "DISPLAY_NAME = \"beans_\" + TIMESTAMP\n",
-        "\n",
-        "job = aip.PipelineJob(\n",
-        "    display_name=DISPLAY_NAME,\n",
-        "    template_path=\"tabular classification_pipeline.json\".replace(\" \", \"_\"),\n",
+        "# Configure the pipeline\n",
+        "job = aiplatform.PipelineJob(\n",
+        "    display_name=PIPELINE_DISPLAY_NAME,\n",
+        "    template_path=\"tabular_classification_pipeline.json\",\n",
         "    pipeline_root=PIPELINE_ROOT,\n",
-        "    parameter_values={\"project\": PROJECT_ID, \"display_name\": DISPLAY_NAME},\n",
+        "    parameter_values={\n",
+        "        \"project\": PROJECT_ID,\n",
+        "        \"gcp_region\": REGION,\n",
+        "        \"bq_source\": \"bq://aju-dev-demos.beans.beans1\",\n",
+        "        \"thresholds_dict_str\": '{\"auRoc\": 0.95}',\n",
+        "        \"DATASET_DISPLAY_NAME\": DATASET_DISPLAY_NAME,\n",
+        "        \"TRAINING_DISPLAY_NAME\": TRAINING_DISPLAY_NAME,\n",
+        "        \"MODEL_DISPLAY_NAME\": MODEL_DISPLAY_NAME,\n",
+        "        \"ENDPOINT_DISPLAY_NAME\": ENDPOINT_DISPLAY_NAME,\n",
+        "        \"MACHINE_TYPE\": MACHINE_TYPE,\n",
+        "    },\n",
         "    enable_caching=False,\n",
-        ")\n",
-        "\n",
-        "job.submit()\n",
-        "\n",
-        "! rm tabular_classification_pipeline.json"
+        ")"
       ]
     },
     {
@@ -1019,11 +1054,19 @@
         "id": "view_pipeline_run:model"
       },
       "source": [
-        "Click on the generated link to see your run in the Cloud Console.\n",
-        "\n",
-        "<!-- It should look something like this as it is running:\n",
-        "\n",
-        "<a href=\"https://storage.googleapis.com/amy-jo/images/mp/automl_tabular_classif.png\" target=\"_blank\"><img src=\"https://storage.googleapis.com/amy-jo/images/mp/automl_tabular_classif.png\" width=\"40%\"/></a> -->"
+        "Run the pipeline job. Click on the generated link to see your run in the Cloud Console."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "114ab8ff24ac"
+      },
+      "outputs": [],
+      "source": [
+        "# Run the job\n",
+        "job.run()"
       ]
     },
     {
@@ -1045,7 +1088,7 @@
       },
       "outputs": [],
       "source": [
-        "pipeline_df = aip.get_pipeline_df(pipeline=PIPELINE_NAME)\n",
+        "pipeline_df = aiplatform.get_pipeline_df(pipeline=PIPELINE_NAME)\n",
         "print(pipeline_df.head(2))"
       ]
     },
@@ -1055,21 +1098,14 @@
         "id": "cleanup:pipelines"
       },
       "source": [
-        "# Cleaning up\n",
+        "## Cleaning up\n",
         "\n",
         "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
         "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
         "\n",
-        "Otherwise, you can delete the individual resources you created in this tutorial -- *Note:* this is auto-generated and not all resources may be applicable for this tutorial:\n",
+        "Otherwise, you can delete the individual resources you created in this tutorial:\n",
         "\n",
-        "- Dataset\n",
-        "- Pipeline\n",
-        "- Model\n",
-        "- Endpoint\n",
-        "- Batch Job\n",
-        "- Custom Job\n",
-        "- Hyperparameter Tuning Job\n",
-        "- Cloud Storage Bucket"
+        "(Set `delete_bucket` to **True** to delete the Cloud Storage bucket.)"
       ]
     },
     {
@@ -1080,94 +1116,37 @@
       },
       "outputs": [],
       "source": [
-        "delete_dataset = True\n",
-        "delete_pipeline = True\n",
-        "delete_model = True\n",
-        "delete_endpoint = True\n",
-        "delete_batchjob = True\n",
-        "delete_customjob = True\n",
-        "delete_hptjob = True\n",
-        "delete_bucket = True\n",
+        "delete_bucket = False\n",
         "\n",
-        "try:\n",
-        "    if delete_model and \"DISPLAY_NAME\" in globals():\n",
-        "        models = aip.Model.list(\n",
-        "            filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
-        "        )\n",
-        "        model = models[0]\n",
-        "        aip.Model.delete(model)\n",
-        "        print(\"Deleted model:\", model)\n",
-        "except Exception as e:\n",
-        "    print(e)\n",
+        "# Delete the Vertex AI Pipeline Job\n",
+        "job.delete()\n",
         "\n",
-        "try:\n",
-        "    if delete_endpoint and \"DISPLAY_NAME\" in globals():\n",
-        "        endpoints = aip.Endpoint.list(\n",
-        "            filter=f\"display_name={DISPLAY_NAME}_endpoint\", order_by=\"create_time\"\n",
-        "        )\n",
-        "        endpoint = endpoints[0]\n",
-        "        endpoint.undeploy_all()\n",
-        "        aip.Endpoint.delete(endpoint.resource_name)\n",
-        "        print(\"Deleted endpoint:\", endpoint)\n",
-        "except Exception as e:\n",
-        "    print(e)\n",
+        "# Delete the Vertex AI Endpoint\n",
+        "endpoints = aiplatform.Endpoint.list(\n",
+        "    filter=f\"display_name={ENDPOINT_DISPLAY_NAME}\", order_by=\"create_time\"\n",
+        ")\n",
         "\n",
-        "if delete_dataset and \"DISPLAY_NAME\" in globals():\n",
-        "    if \"tabular\" == \"tabular\":\n",
-        "        try:\n",
-        "            datasets = aip.TabularDataset.list(\n",
-        "                filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
-        "            )\n",
-        "            dataset = datasets[0]\n",
-        "            aip.TabularDataset.delete(dataset.resource_name)\n",
-        "            print(\"Deleted dataset:\", dataset)\n",
-        "        except Exception as e:\n",
-        "            print(e)\n",
+        "if len(endpoints) > 0:\n",
+        "    endpoint = endpoints[0]\n",
+        "    endpoint.delete(force=True)\n",
         "\n",
-        "    if \"tabular\" == \"image\":\n",
-        "        try:\n",
-        "            datasets = aip.ImageDataset.list(\n",
-        "                filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
-        "            )\n",
-        "            dataset = datasets[0]\n",
-        "            aip.ImageDataset.delete(dataset.resource_name)\n",
-        "            print(\"Deleted dataset:\", dataset)\n",
-        "        except Exception as e:\n",
-        "            print(e)\n",
+        "# Delete the Vertex AI model\n",
+        "models = aiplatform.Model.list(\n",
+        "    filter=f\"display_name={MODEL_DISPLAY_NAME}\", order_by=\"create_time\"\n",
+        ")\n",
+        "if len(models) > 0:\n",
+        "    model = models[0]\n",
+        "    model.delete()\n",
         "\n",
-        "    if \"tabular\" == \"text\":\n",
-        "        try:\n",
-        "            datasets = aip.TextDataset.list(\n",
-        "                filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
-        "            )\n",
-        "            dataset = datasets[0]\n",
-        "            aip.TextDataset.delete(dataset.resource_name)\n",
-        "            print(\"Deleted dataset:\", dataset)\n",
-        "        except Exception as e:\n",
-        "            print(e)\n",
-        "\n",
-        "    if \"tabular\" == \"video\":\n",
-        "        try:\n",
-        "            datasets = aip.VideoDataset.list(\n",
-        "                filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
-        "            )\n",
-        "            dataset = datasets[0]\n",
-        "            aip.VideoDataset.delete(dataset.resource_name)\n",
-        "            print(\"Deleted dataset:\", dataset)\n",
-        "        except Exception as e:\n",
-        "            print(e)\n",
-        "\n",
-        "try:\n",
-        "    if delete_pipeline and \"DISPLAY_NAME\" in globals():\n",
-        "        pipelines = aip.PipelineJob.list(\n",
-        "            filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
-        "        )\n",
-        "        pipeline = pipelines[0]\n",
-        "        aip.PipelineJob.delete(pipeline.resource_name)\n",
-        "        print(\"Deleted pipeline:\", pipeline)\n",
-        "except Exception as e:\n",
-        "    print(e)\n",
+        "# Delete the Vertex AI Dataset\n",
+        "datasets = aiplatform.TabularDataset.list(\n",
+        "    filter=f\"display_name={DATASET_DISPLAY_NAME}\", order_by=\"create_time\"\n",
+        ")\n",
+        "if len(datasets) > 0:\n",
+        "    dataset = datasets[0]\n",
+        "    dataset.delete()\n",
         "\n",
+        "# Delete the Cloud Storage bucket\n",
         "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
         "    ! gsutil rm -r $BUCKET_URI"
       ]

notebooks/official/pipelines/google_cloud_pipeline_components_automl_tabular.ipynb

@@ -64,19 +64,6 @@
         "This notebook shows how to use the components defined in [`google_cloud_pipeline_components`](https://github.com/kubeflow/pipelines/tree/master/components/google-cloud) to build an AutoML tabular regression workflow on [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines)."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "dataset:cal_housing,lrg"
-      },
-      "source": [
-        "### Dataset\n",
-        "\n",
-        "The dataset used for this tutorial is the [California Housing dataset from the 1990 Census](https://developers.google.com/machine-learning/crash-course/california-housing-data-description)\n",
-        "\n",
-        "The dataset predicts the median house price."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -109,6 +96,19 @@
         "The components are [documented here](https://google-cloud-pipeline-components.readthedocs.io/en/latest/google_cloud_pipeline_components.aiplatform.html#module-google_cloud_pipeline_components.aiplatform)."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "77de5c53ac91"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The dataset used for this tutorial is the [California Housing dataset from the 1990 Census](https://developers.google.com/machine-learning/crash-course/california-housing-data-description)\n",
+        "\n",
+        "The dataset predicts the median house price."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -234,6 +234,8 @@
         "id": "check_versions"
       },
       "source": [
+        "### Check installed package versions\n",
+        "\n",
         "Check the versions of the packages you installed.  The KFP SDK version should be >=1.6."
       ]
     },
@@ -344,7 +346,10 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
       ]
     },
     {
@@ -353,9 +358,9 @@
         "id": "timestamp"
       },
       "source": [
-        "#### Timestamp\n",
+        "#### UUID\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append the uuid onto the name of resources you create in this tutorial."
       ]
     },
     {
@@ -366,9 +371,16 @@
       },
       "outputs": [],
       "source": [
-        "from datetime import datetime\n",
+        "import random\n",
+        "import string\n",
         "\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
+        "\n",
+        "# Generate a uuid of length 8\n",
+        "def generate_uuid():\n",
+        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=8))\n",
+        "\n",
+        "\n",
+        "UUID = generate_uuid()"
       ]
     },
     {
@@ -474,7 +486,7 @@
       "outputs": [],
       "source": [
         "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
         "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
       ]
     },
@@ -718,10 +730,10 @@
         "    endpoint_op = EndpointCreateOp(\n",
         "        project=project,\n",
         "        location=region,\n",
-        "        display_name=\"train-automl-flowers\",\n",
+        "        display_name=\"train-automl-cal_housing_endpoint\",\n",
         "    )\n",
         "\n",
-        "    ModelDeployOp(\n",
+        "    _ = ModelDeployOp(\n",
         "        model=training_op.outputs[\"model\"],\n",
         "        endpoint=endpoint_op.outputs[\"endpoint\"],\n",
         "        dedicated_resources_machine_type=\"n1-standard-4\",\n",
@@ -753,7 +765,7 @@
         "\n",
         "compiler.Compiler().compile(\n",
         "    pipeline_func=pipeline,\n",
-        "    package_path=\"tabular regression_pipeline.json\".replace(\" \", \"_\"),\n",
+        "    package_path=\"tabular_regression_pipeline.json\",\n",
         ")"
       ]
     },
@@ -776,11 +788,11 @@
       },
       "outputs": [],
       "source": [
-        "DISPLAY_NAME = \"cal_housing_\" + TIMESTAMP\n",
+        "DISPLAY_NAME = \"cal_housing_\" + UUID\n",
         "\n",
         "job = aip.PipelineJob(\n",
         "    display_name=DISPLAY_NAME,\n",
-        "    template_path=\"tabular regression_pipeline.json\".replace(\" \", \"_\"),\n",
+        "    template_path=\"tabular_regression_pipeline.json\",\n",
         "    pipeline_root=PIPELINE_ROOT,\n",
         "    enable_caching=False,\n",
         ")\n",
@@ -818,16 +830,7 @@
         "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
         "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
         "\n",
-        "Otherwise, you can delete the individual resources you created in this tutorial -- *Note:* this is auto-generated and not all resources may be applicable for this tutorial:\n",
-        "\n",
-        "- Dataset\n",
-        "- Pipeline\n",
-        "- Model\n",
-        "- Endpoint\n",
-        "- Batch Job\n",
-        "- Custom Job\n",
-        "- Hyperparameter Tuning Job\n",
-        "- Cloud Storage Bucket"
+        "Otherwise, you can delete the individual resources you created in this tutorial "
       ]
     },
     {
@@ -842,89 +845,73 @@
         "delete_pipeline = True\n",
         "delete_model = True\n",
         "delete_endpoint = True\n",
-        "delete_batchjob = True\n",
-        "delete_customjob = True\n",
-        "delete_hptjob = True\n",
         "delete_bucket = True\n",
         "\n",
-        "try:\n",
-        "    if delete_model and \"DISPLAY_NAME\" in globals():\n",
-        "        models = aip.Model.list(\n",
-        "            filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
-        "        )\n",
-        "        model = models[0]\n",
-        "        aip.Model.delete(model)\n",
-        "        print(\"Deleted model:\", model)\n",
-        "except Exception as e:\n",
-        "    print(e)\n",
+        "dataset_display_name = \"housing\"\n",
+        "pipeline_display_name = \"automl-tab-training-v2\"\n",
+        "model_display_name = \"train-automl-cal_housing\"\n",
+        "endpoint_display_name = \"train-automl-cal_housing_endpoint\"\n",
         "\n",
-        "try:\n",
-        "    if delete_endpoint and \"DISPLAY_NAME\" in globals():\n",
-        "        endpoints = aip.Endpoint.list(\n",
-        "            filter=f\"display_name={DISPLAY_NAME}_endpoint\", order_by=\"create_time\"\n",
-        "        )\n",
+        "\n",
+        "if delete_endpoint:\n",
+        "    endpoints = aip.Endpoint.list(\n",
+        "        filter=f\"display_name={endpoint_display_name}\", order_by=\"create_time\"\n",
+        "    )\n",
+        "    if endpoints:\n",
         "        endpoint = endpoints[0]\n",
         "        endpoint.undeploy_all()\n",
-        "        aip.Endpoint.delete(endpoint.resource_name)\n",
+        "        endpoint.delete()\n",
         "        print(\"Deleted endpoint:\", endpoint)\n",
-        "except Exception as e:\n",
-        "    print(e)\n",
         "\n",
-        "if delete_dataset and \"DISPLAY_NAME\" in globals():\n",
+        "if delete_model:\n",
+        "    models = aip.Model.list(\n",
+        "        filter=f\"display_name={model_display_name}\", order_by=\"create_time\"\n",
+        "    )\n",
+        "    if models:\n",
+        "        model = models[0]\n",
+        "        model.delete()\n",
+        "        print(\"Deleted model:\", model)\n",
+        "\n",
+        "if delete_dataset:\n",
         "    if \"tabular\" == \"tabular\":\n",
-        "        try:\n",
-        "            datasets = aip.TabularDataset.list(\n",
-        "                filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
-        "            )\n",
+        "        datasets = aip.TabularDataset.list(\n",
+        "            filter=f\"display_name={dataset_display_name}\", order_by=\"create_time\"\n",
+        "        )\n",
+        "        if datasets:\n",
         "            dataset = datasets[0]\n",
-        "            aip.TabularDataset.delete(dataset.resource_name)\n",
+        "            dataset.delete()\n",
         "            print(\"Deleted dataset:\", dataset)\n",
-        "        except Exception as e:\n",
-        "            print(e)\n",
         "\n",
         "    if \"tabular\" == \"image\":\n",
-        "        try:\n",
-        "            datasets = aip.ImageDataset.list(\n",
-        "                filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
-        "            )\n",
+        "        datasets = aip.ImageDataset.list(\n",
+        "            filter=f\"display_name={dataset_display_name}\", order_by=\"create_time\"\n",
+        "        )\n",
+        "        if datasets:\n",
         "            dataset = datasets[0]\n",
-        "            aip.ImageDataset.delete(dataset.resource_name)\n",
+        "            dataset.delete()\n",
         "            print(\"Deleted dataset:\", dataset)\n",
-        "        except Exception as e:\n",
-        "            print(e)\n",
         "\n",
         "    if \"tabular\" == \"text\":\n",
-        "        try:\n",
-        "            datasets = aip.TextDataset.list(\n",
-        "                filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
-        "            )\n",
+        "        datasets = aip.TextDataset.list(\n",
+        "            filter=f\"display_name={dataset_display_name}\", order_by=\"create_time\"\n",
+        "        )\n",
+        "        if datasets:\n",
         "            dataset = datasets[0]\n",
-        "            aip.TextDataset.delete(dataset.resource_name)\n",
+        "            dataset.delete()\n",
         "            print(\"Deleted dataset:\", dataset)\n",
-        "        except Exception as e:\n",
-        "            print(e)\n",
         "\n",
         "    if \"tabular\" == \"video\":\n",
-        "        try:\n",
-        "            datasets = aip.VideoDataset.list(\n",
-        "                filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
-        "            )\n",
-        "            dataset = datasets[0]\n",
-        "            aip.VideoDataset.delete(dataset.resource_name)\n",
-        "            print(\"Deleted dataset:\", dataset)\n",
-        "        except Exception as e:\n",
-        "            print(e)\n",
-        "\n",
-        "try:\n",
-        "    if delete_pipeline and \"DISPLAY_NAME\" in globals():\n",
-        "        pipelines = aip.PipelineJob.list(\n",
-        "            filter=f\"display_name={DISPLAY_NAME}\", order_by=\"create_time\"\n",
+        "        datasets = aip.VideoDataset.list(\n",
+        "            filter=f\"display_name={dataset_display_name}\", order_by=\"create_time\"\n",
         "        )\n",
-        "        pipeline = pipelines[0]\n",
-        "        aip.PipelineJob.delete(pipeline.resource_name)\n",
-        "        print(\"Deleted pipeline:\", pipeline)\n",
-        "except Exception as e:\n",
-        "    print(e)\n",
+        "        if datasets:\n",
+        "            dataset = datasets[0]\n",
+        "            dataset.delete()\n",
+        "            print(\"Deleted dataset:\", dataset)\n",
+        "\n",
+        "if delete_pipeline:\n",
+        "    job.delete()\n",
+        "\n",
         "\n",
         "if delete_bucket and os.getenv(\"IS_TESTING\"):\n",
         "    ! gsutil rm -r $BUCKET_URI"

notebooks/official/pipelines/google_cloud_pipeline_components_dataproc_tabular.ipynb

@@ -425,9 +425,19 @@
         "3. In the **Service account name** field, enter a name, and\n",
         "   click **Create**.\n",
         "\n",
-        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
-        "into the filter box, and select\n",
-        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type the following role and select them\n",
+        "\n",
+        "      - Artifact Registry Administrator\n",
+        "      - Artifact Registry Repository Administrator\n",
+        "      - Cloud Build Editor\n",
+        "      - Compute Network Admin\n",
+        "      - Dataproc Administrator\n",
+        "      - Dataproc Worker\n",
+        "      - Service Account User\n",
+        "      - Storage Admin\n",
+        "      - Storage Object Admin\n",
+        "      - Vertex AI Administrator\n",
+        "\n",
         "\n",
         "5. Click *Create*. A JSON file that contains your key downloads to your\n",
         "local environment.\n",
@@ -639,15 +649,10 @@
       },
       "outputs": [],
       "source": [
-        "from pathlib import Path as path\n",
-        "\n",
-        "DATA_PATH = path(\"content/path/\")\n",
         "PUBLIC_DATA_URI = \"gs://cloud-samples-data/vertex-ai/dataset-management/datasets/loan_eligibilty/data.csv\"\n",
         "FEATURES_TRAIN_URI = f\"{BUCKET_URI}/data/features/snapshots/{UUID}\"\n",
         "\n",
-        "!mkdir -m 777 -p $DATA_PATH\n",
-        "!gsutil cp -r $PUBLIC_DATA_URI $DATA_PATH\n",
-        "!gsutil cp -r $DATA_PATH $FEATURES_TRAIN_URI"
+        "!gsutil cp -r $PUBLIC_DATA_URI $FEATURES_TRAIN_URI"
       ]
     },
     {
@@ -733,9 +738,7 @@
         "from google_cloud_pipeline_components import aiplatform as vertex_ai_components\n",
         "from kfp.v2 import compiler, dsl\n",
         "from kfp.v2.dsl import (Artifact, ClassificationMetrics, Condition, Input,\n",
-        "                        Metrics, Output, component)\n",
-        "\n",
-        "ID = random.randint(1, 10000)"
+        "                        Metrics, Output, component)"
       ]
     },
     {
@@ -767,7 +770,7 @@
         "MODEL_NAME = f\"{ML_APPLICATION}-{TASK}-{MODEL_TYPE}-{VERSION}\"\n",
         "\n",
         "# Preprocessing\n",
-        "PREPROCESSING_BATCH_ID = f\"data-preprocessing-{ID}\"\n",
+        "PREPROCESSING_BATCH_ID = f\"data-preprocessing-{UUID}\"\n",
         "PREPROCESSING_PYTHON_FILE_URI = f\"{BUCKET_URI}/src/data_preprocessing.py\"\n",
         "PROCESSED_DATA_URI = f\"{BUCKET_URI}/data/processed\"\n",
         "PREPROCESSING_ARGS = [\n",
@@ -778,11 +781,11 @@
         "]\n",
         "\n",
         "# Dataset\n",
-        "DATASET_NAME = f\"preprocessed-dataset-{ID}\"\n",
+        "DATASET_NAME = f\"preprocessed-dataset-{UUID}\"\n",
         "GCS_PREPROCESSED_URI = f\"{PROCESSED_DATA_URI}/*/?.csv\"\n",
         "\n",
         "# Training\n",
-        "TRAINING_BATCH_ID = f\"model-training-{ID}\"\n",
+        "TRAINING_BATCH_ID = f\"model-training-{UUID}\"\n",
         "TRAINING_PYTHON_FILE_URI = f\"{BUCKET_URI}/src/model_training.py\"\n",
         "MODEL_URI = f\"{BUCKET_URI}/deliverables/model/rfor/{UUID}/train_model\"\n",
         "METRICS_URI = f\"{BUCKET_URI}/deliverables/metrics/rfor/{UUID}/train_metrics.json\"\n",
@@ -800,7 +803,7 @@
         "AUPR_HYPERTUNE_CONDITION = \"[AUPR_HYPERTUNE]\"\n",
         "\n",
         "# Hypertuning\n",
-        "HPT_TRAINING_BATCH_ID = f\"hyper-tuning-{ID}\"\n",
+        "HPT_TRAINING_BATCH_ID = f\"hyper-tuning-{UUID}\"\n",
         "HPT_PYTHON_FILE_URI = f\"{BUCKET_URI}/src/hp_tuning.py\"\n",
         "HPT_MODEL_URI = f\"{BUCKET_URI}/deliverables/model/rfor/{UUID}/model\"\n",
         "HPT_METRICS_URI = f\"{BUCKET_URI}/deliverables/metrics/rfor/{UUID}/metrics.json\"\n",
@@ -909,6 +912,7 @@
         "\n",
         "\"\"\"\n",
         "data_preprocessing.py is the module for\n",
+        "\n",
         "  - ingest data\n",
         "  - do simple preprocessing tasks\n",
         "  - upload processed data to gcs\n",
@@ -1021,8 +1025,7 @@
         "\n",
         "    spark = (SparkSession.builder\n",
         "             .master(\"local[*]\")\n",
-        "             .appName(\"spark go live\")\n",
-        "             .config('spark.ui.port', '4050')\n",
+        "             .appName(\"loan eligibility\")\n",
         "             .getOrCreate())\n",
         "    try:\n",
         "        logger.info(f'spark version: {spark.sparkContext.version}')\n",
@@ -1041,12 +1044,7 @@
         "        training_data_raw_df.show(truncate=False)\n",
         "\n",
         "        logger.info(f'load prepared data to {output_data_path}.')\n",
-        "        if output_data_path.startswith('gs://'):\n",
-        "            training_data_raw_df.write.mode('overwrite').csv(str(output_data_path), header=True)\n",
-        "        else:\n",
-        "            output_file_path = Path(output_data_path)\n",
-        "            output_file_path.mkdir(parents=True, exist_ok=True)\n",
-        "            training_data_raw_df.write.mode('overwrite').csv(str(output_file_path), header=True)\n",
+        "        training_data_raw_df.write.mode('overwrite').csv(str(output_data_path), header=True)\n",
         "    except RuntimeError as main_error:\n",
         "        logger.error(main_error)\n",
         "    else:\n",
@@ -1376,8 +1374,7 @@
         "        logger.info('start spark session.')\n",
         "        spark = (SparkSession.builder\n",
         "                 .master(\"local[*]\")\n",
-        "                 .appName(\"spark go live\")\n",
-        "                 .config('spark.ui.port', '4050')\n",
+        "                 .appName(\"loan eligibility\")\n",
         "                 .getOrCreate())\n",
         "        logger.info(f'spark version: {spark.sparkContext.version}')\n",
         "        logger.info('start bulding pipeline.')\n",
@@ -1402,23 +1399,12 @@
         "\n",
         "        logger.info(f'load model pipeline in {model_path}.')\n",
         "        pipeline.write().overwrite().save(model_path)\n",
-        "        if model_path.startswith('gs://'):\n",
-        "            pipeline.write().overwrite().save(model_path)\n",
-        "        else:\n",
-        "            path(model_path).mkdir(parents=True, exist_ok=True)\n",
-        "            pipeline.write().overwrite().save(model_path)\n",
         "\n",
-        "        logger.info(f'Upload metrics under {metrics_path}.')\n",
-        "        if metrics_path.startswith('gs://'):\n",
-        "            bucket = urlparse(model_path).netloc\n",
-        "            metrics_file_path = urlparse(metrics_path).path.strip('/')\n",
-        "            write_metrics(bucket, metrics, metrics_file_path)\n",
-        "        else:\n",
-        "            metrics_version_path = path(metrics_path).parents[0]\n",
-        "            metrics_version_path.mkdir(parents=True, exist_ok=True)\n",
-        "            with open(metrics_path, 'w') as json_file:\n",
-        "                json.dump(metrics, json_file)\n",
-        "            json_file.close()\n",
+        "        logger.info(f'Upload metrics under {metrics_path}.')     \n",
+        "        bucket = urlparse(model_path).netloc\n",
+        "        metrics_file_path = urlparse(metrics_path).path.strip('/')\n",
+        "        write_metrics(bucket, metrics, metrics_file_path)\n",
+        "        \n",
         "    except RuntimeError as main_error:\n",
         "        logger.error(main_error)\n",
         "    else:\n",
@@ -1748,12 +1734,7 @@
         "        logger.info('start spark session.')\n",
         "        spark = (SparkSession.builder\n",
         "                 .master(\"local[*]\")\n",
-        "                 .appName(\"spark go live\")\n",
-        "                 .config('spark.ui.port', '4050')\n",
-        "                 .config('spark.jars.packages', 'ml.combust.mleap:mleap-runtime_2.12:0.19.0')\n",
-        "                 .config('spark.jars.packages', 'ml.combust.mleap:mleap-base_2.12:0.19.0')\n",
-        "                 .config('spark.jars.packages', 'ml.combust.mleap:mleap-spark_2.12:0.19.0')\n",
-        "                 .config('spark.jars.packages', 'ml.combust.mleap:mleap-spark-extension_2.12:0.19.0')\n",
+        "                 .appName(\"loan eligibility\")\n",
         "                 .getOrCreate())\n",
         "        logger.info(f'spark version: {spark.sparkContext.version}')\n",
         "        logger.info('start building pipeline.')\n",
@@ -1763,12 +1744,7 @@
         "        pipeline_cross_validator = build_hp_pipeline(preprocessing_stages, feature_engineering_stages,\n",
         "                                                     model_training_stage)\n",
         "        logger.info(f'load train data from {train_path}.')\n",
-        "        if train_path.startswith('bq://'):\n",
-        "            raw_data = spark.read.format('bigquery') \\\n",
-        "                .option('table', train_path.replace('bq://', '')) \\\n",
-        "                .load()\n",
-        "        else:\n",
-        "            raw_data = (spark.read.format('csv')\n",
+        "        raw_data = (spark.read.format('csv')\n",
         "                        .option(\"header\", \"true\")\n",
         "                        .schema(DATA_SCHEMA)\n",
         "                        .load(train_path))\n",
@@ -1781,23 +1757,12 @@
         "            print(f'{m}: {v}')\n",
         "\n",
         "        logger.info(f'load model pipeline in {model_path}.')\n",
-        "        if model_path.startswith('gs://'):\n",
-        "            pipeline_model.write().overwrite().save(model_path)\n",
-        "        else:\n",
-        "            path(model_path).mkdir(parents=True, exist_ok=True)\n",
-        "            pipeline_model.write().overwrite().save(model_path)\n",
+        "        pipeline_model.write().overwrite().save(model_path)\n",
         "\n",
         "        logger.info(f'Upload metrics under {metrics_path}.')\n",
-        "        if metrics_path.startswith('gs://'):\n",
-        "            bucket = urlparse(model_path).netloc\n",
-        "            metrics_file_path = urlparse(metrics_path).path.strip('/')\n",
-        "            write_metrics(bucket, metrics, metrics_file_path)\n",
-        "        else:\n",
-        "            metrics_version_path = path(metrics_path).parents[0]\n",
-        "            metrics_version_path.mkdir(parents=True, exist_ok=True)\n",
-        "            with open(metrics_path, 'w') as json_file:\n",
-        "                json.dump(metrics, json_file)\n",
-        "            json_file.close()\n",
+        "        bucket = urlparse(model_path).netloc\n",
+        "        metrics_file_path = urlparse(metrics_path).path.strip('/')\n",
+        "        write_metrics(bucket, metrics, metrics_file_path)\n",
         "    except RuntimeError as main_error:\n",
         "        logger.error(main_error)\n",
         "    else:\n",
@@ -1926,8 +1891,7 @@
         "      python \\\n",
         "      scikit-image \\\n",
         "      scikit-learn \\\n",
-        "      scipy \\\n",
-        "      mleap \n",
+        "      scipy  \n",
         "\n",
         "# (Required) Create the 'spark' group/user.\n",
         "# The GID and UID must be 1099. Home directory is required.\n",
@@ -2433,6 +2397,17 @@
         "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
         "    ! gsutil -m rm -r $BUCKET_URI"
       ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "bc7f1247f0ec"
+      },
+      "outputs": [],
+      "source": [
+        "!rm -rf $SRC $BUILD_PATH"
+      ]
     }
   ],
   "metadata": {