migration: move to pipelines folder

.cloud-build/cleanup/resource_cleanup_manager.py

@@ -103,6 +103,7 @@ class EndpointResourceCleanupManager(VertexAIResourceCleanupManager):
             models.id for models in resource._gca_resource.deployed_models
         ]:
             resource._undeploy(deployed_model_id=deployed_model_id)
+
         resource.delete(force=True)
 
 
@@ -116,7 +117,3 @@ class MatchingEngineIndexResourceCleanupManager(VertexAIResourceCleanupManager):
 
 class MatchingEngineIndexEndpointResourceCleanupManager(VertexAIResourceCleanupManager):
     vertex_ai_resource = aiplatform.MatchingEngineIndexEndpoint
-
-    def delete(self, resource):
-        resource.undeploy_all()
-        resource.delete(force=True)

.cloud-build/requirements.txt

@@ -11,4 +11,4 @@ google-cloud-storage
 google-cloud-build
 ratemate
 GitPython
-tqdm
+google-api-core==2.10

.github/workflows/linter/requirements.txt

@@ -2,9 +2,9 @@ git+https://github.com/tensorflow/docs
 ipython
 jupyter
 nbconvert
-black==22.12.0
+black==22.10.0
 pyupgrade==2.38.4
-isort==5.12.0
-flake8==6.0.0
+isort==5.10.1
+flake8==4.0.1
 nbqa==1.5.3
 

community-content/pytorch_image_classification_single_gpu_with_vertex_sdk_and_torchserve/trainer/requirements.txt

@@ -1,3 +1,3 @@
-torch==1.13.1
+torch==1.8.1
 torchvision==0.9.1
 tensorboard==2.5.0

community-content/pytorch_pre_built_images_deployment/pytorch-text-classification-vertex-ai-deploy.ipynb

@@ -31,7 +31,17 @@
       "source": [
         "# Deploying a PyTorch Text Classification Model on [Vertex AI](https://cloud.google.com/vertex-ai)\n",
         "\n",
-        "**Kindly reach out to Vertex AI before you run any scale tests or you have any questions.**\n"
+        "**This is an Experimental release**, covered by the Pre-GA Offerings Terms of your Google Cloud Platform [Terms of Service](https://cloud.google.com/terms).\n",
+        "\n",
+        "Experiments are focused on validating a prototype and are not guaranteed to be released. They are not intended for production use or covered by any SLA, support obligation, or deprecation policy and might be subject to backward-incompatible changes.\n",
+        "\n",
+        "**Kindly drop us a note before you run any scale tests.**\n",
+        "\n",
+        "**Do not hesitate to contact vertexai-prediction-preview-feedback@google.com if you have any questions or run into any issues.**\n",
+        "\n",
+        "The usage of the product is free during the Experimental release period: you will still incur charges for other GCP products usage, such as storage.\n",
+        "\n",
+        "The projects need to be added to the allowlist in order to deploy PyTorch models using Vertex AI Prediction pre-built PyTorch images. If you are interested in the feature, please send an email to vertexai-prediction-preview-feedback@google.com to provide your project numbers OR project ids."
       ]
     },
     {

notebooks/community/CODEOWNERS

@@ -40,4 +40,4 @@
 /notebooks/community/pipelines/google_cloud_pipeline_components_bqml_pipeline_anomaly_detection.ipynb @inardini
 /notebooks/community/pipelines/google_cloud_pipeline_components_cloud_natural_language_pipeline.ipynb @Narwhalprime
 /notebooks/community/pipelines/google_cloud_pipeline_components_ready_to_go_text_classification_pipeline.ipynb @Narwhalprime
-/notebooks/community/feature_store/get_started_vertex_feature_store.ipynb @junkourata
+

notebooks/community/ml_ops/stage3/get_started_with_hpt_pipeline_components.ipynb

@@ -1333,6 +1333,7 @@
         "Next, you compile the pipeline and then exeute it. The pipeline takes the following parameters, which are passed as the dictionary `parameter_values`:\n",
         "\n",
         "- `display_name`: A human readable name for the pipeline job.\n",
+        "- `import_file`: The Cloud Storage location to the dataset.\n",
         "- `worker_pool_specs`: The the machine and container, and auto-scaling requirements, as well as command line arguments.\n",
         "- `study_spec_metrics`: The metrics to optimize in the study trials.\n",
         "- `study_spec_parameters`: The parameters to tune."

notebooks/community/ml_ops/stage5/get_started_with_vertex_endpoint_and_shared_vm.ipynb

@@ -1028,9 +1028,6 @@
         "deployment_resource_pool.dedicated_resources.min_replica_count = MIN_NODES\n",
         "deployment_resource_pool.dedicated_resources.max_replica_count = MAX_NODES\n",
         "deployment_resource_pool.dedicated_resources.machine_spec.machine_type = DEPLOY_COMPUTE\n",
-        "if DEPLOY_NGPU:\n",
-        "    deployment_resource_pool.dedicated_resources.machine_spec.accelerator_type = DEPLOY_GPU\n",
-        "    deployment_resource_pool.dedicated_resources.machine_spec.accelerator_count = DEPLOY_NGPU\n",
         "\n",
         "request = aip_beta.CreateDeploymentResourcePoolRequest(\n",
         "    parent=f\"projects/{PROJECT_ID}/locations/{REGION}\",\n",

notebooks/community/pipelines/google_cloud_pipeline_components_bqml_pipeline_demand_forecasting.ipynb

@@ -54,20 +54,18 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "239ba71252d3"
+        "id": "tvgnzT1CKxrO"
       },
       "source": [
         "## Overview\n",
         "\n",
-        "This notebook shows how to use `Vertex AI Pipelines` and `BigQuery ML pipeline components` to train and evaluate a demand forecasting model."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "25c28706c23e"
-      },
-      "source": [
+        "This notebook shows how to use `Vertex AI Pipelines` and `BigQuery ML pipeline components` to train and evaluate a demand forecasting model.\n",
+        "\n",
+        "### Dataset\n",
+        "\n",
+        "The dataset is a modified version of the dataset in [Build and visualize demand forecast predictions using Datastream, Dataflow, BigQuery ML, and Looker\n",
+        "](https://cloud.google.com/architecture/build-visualize-demand-forecast-prediction-datastream-dataflow-bigqueryml-looker) solution architecture\n",
+        "\n",
         "### Objective\n",
         "\n",
         "In this tutorial, you learn how to train and evaluate a BigQuery ML model using Vertex AI Pipelines and BigQuery ML pipeline components. \n",
@@ -89,27 +87,8 @@
         "  - Generate the ARIMA Plus forecasts\n",
         "  - Generate the ARIMA PLUS forecast explainations\n",
         "- Compile the pipeline.\n",
-        "- Execute the pipeline."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "586acfa9b502"
-      },
-      "source": [
-        "### Dataset\n",
+        "- Execute the pipeline.\n",
         "\n",
-        "The dataset is a modified version of the dataset in [Build and visualize demand forecast predictions using Datastream, Dataflow, BigQuery ML, and Looker\n",
-        "](https://cloud.google.com/architecture/build-visualize-demand-forecast-prediction-datastream-dataflow-bigqueryml-looker) solution architecture\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "tvgnzT1CKxrO"
-      },
-      "source": [
         "### Costs \n",
         "\n",
         "This tutorial uses billable components of Google Cloud:\n",
@@ -373,8 +352,9 @@
         "id": "06571eb4063b"
       },
       "source": [
-        "#### UUID\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
+        "#### Timestamp\n",
+        "\n",
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append it onto the name of resources you create in this tutorial."
       ]
     },
     {
@@ -385,16 +365,9 @@
       },
       "outputs": [],
       "source": [
-        "import random\n",
-        "import string\n",
+        "from datetime import datetime\n",
         "\n",
-        "\n",
-        "# Generate a uuid of a specifed length(default=8)\n",
-        "def generate_uuid(length: int = 8) -> str:\n",
-        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
-        "\n",
-        "\n",
-        "UUID = generate_uuid()"
+        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
       ]
     },
     {
@@ -512,7 +485,7 @@
       "outputs": [],
       "source": [
         "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"-aip-\" + UUID\n",
+        "    BUCKET_NAME = PROJECT_ID + \"-aip-\" + TIMESTAMP\n",
         "    BUCKET_URI = f\"gs://{BUCKET_NAME}\""
       ]
     },
@@ -733,7 +706,6 @@
         "KFP_COMPONENTS_PATH = \"components\"\n",
         "PIPELINES_PATH = \"pipelines\"\n",
         "\n",
-        "! mkdir -m 777 -p {DATA_PATH}\n",
         "! mkdir -m 777 -p {KFP_COMPONENTS_PATH}\n",
         "! mkdir -m 777 -p {PIPELINES_PATH}"
       ]
@@ -799,7 +771,7 @@
         "  --location={LOCATION} \\\n",
         "  --source_format=CSV \\\n",
         "  --skip_leading_rows=1\\\n",
-        "  fast_fresh.orders_{UUID} \\\n",
+        "  fast_fresh.orders_{TIMESTAMP} \\\n",
         "  {RAW_DATA_URI} \\\n",
         "  time_of_sale:DATETIME,order_id:INTEGER,product_name:STRING,price:NUMERIC,quantity:NUMERIC,payment_method:STRING,store_id:INTEGER,user_id:INTEGER"
       ]
@@ -810,7 +782,7 @@
         "id": "ZrgOD30o7HcL"
       },
       "source": [
-        "## BigQuery ML Training Formalization\n",
+        "## BQML Training Formalization\n",
         "\n",
         "In the next cells, you build the components and pipeline to train and evaluate the BQML demand forecasting model."
       ]
@@ -848,13 +820,13 @@
         "BQ_EVALUATE_MODEL_TABLE_PREFIX = \"orders_arima_model_evaluate\"\n",
         "BQ_FORECAST_TABLE_PREFIX = \"orders_arima_forecast\"\n",
         "BQ_EXPLAIN_FORECAST_TABLE_PREFIX = \"orders_arima_explain_forecast\"\n",
-        "BQ_ORDERS_TABLE = f\"{BQ_ORDERS_TABLE_PREFIX}_{UUID}\"\n",
-        "BQ_TRAINING_TABLE = f\"{BQ_TRAINING_TABLE_PREFIX}_{UUID}\"\n",
-        "BQ_MODEL_TABLE = f\"{BQ_MODEL_TABLE_PREFIX}_{UUID}\"\n",
-        "BQ_EVALUATE_TS_TABLE = f\"{BQ_EVALUATE_TS_TABLE_PREFIX}_{UUID}\"\n",
-        "BQ_EVALUATE_MODEL_TABLE = f\"{BQ_EVALUATE_MODEL_TABLE_PREFIX}_{UUID}\"\n",
-        "BQ_FORECAST_TABLE = f\"{BQ_FORECAST_TABLE_PREFIX}_{UUID}\"\n",
-        "BQ_EXPLAIN_FORECAST_TABLE = f\"{BQ_EXPLAIN_FORECAST_TABLE_PREFIX}_{UUID}\"\n",
+        "BQ_ORDERS_TABLE = f\"{BQ_ORDERS_TABLE_PREFIX}_{TIMESTAMP}\"\n",
+        "BQ_TRAINING_TABLE = f\"{BQ_TRAINING_TABLE_PREFIX}_{TIMESTAMP}\"\n",
+        "BQ_MODEL_TABLE = f\"{BQ_MODEL_TABLE_PREFIX}_{TIMESTAMP}\"\n",
+        "BQ_EVALUATE_TS_TABLE = f\"{BQ_EVALUATE_TS_TABLE_PREFIX}_{TIMESTAMP}\"\n",
+        "BQ_EVALUATE_MODEL_TABLE = f\"{BQ_EVALUATE_MODEL_TABLE_PREFIX}_{TIMESTAMP}\"\n",
+        "BQ_FORECAST_TABLE = f\"{BQ_FORECAST_TABLE_PREFIX}_{TIMESTAMP}\"\n",
+        "BQ_EXPLAIN_FORECAST_TABLE = f\"{BQ_EXPLAIN_FORECAST_TABLE_PREFIX}_{TIMESTAMP}\"\n",
         "\n",
         "BQ_TRAIN_CONFIGURATION = {\n",
         "    \"destinationTable\": {\n",
@@ -1050,7 +1022,7 @@
         "id": "pcSL1FHk69KT"
       },
       "source": [
-        "### Build the BigQuery ML training pipeline\n",
+        "### Build the BQML training pipeline\n",
         "\n",
         "Define your workflow using Kubeflow Pipelines DSL package. \n",
         "\n",
@@ -1122,8 +1094,8 @@
         "        location=location,\n",
         "    ).set_display_name(\"get train data\")\n",
         "\n",
-        "    # Run an ARIMA PLUS experiment\n",
-        "    bq_arima_model_exp_op = (\n",
+        "    # Train the ARIMA PLUS model\n",
+        "    bq_arima_model_op = (\n",
         "        BigqueryCreateModelJobOp(\n",
         "            query=f\"\"\"\n",
         "        -- create model table\n",
@@ -1132,7 +1104,10 @@
         "        MODEL_TYPE = \\'ARIMA_PLUS\\',\n",
         "        TIME_SERIES_TIMESTAMP_COL = \\'hourly_timestamp\\',\n",
         "        TIME_SERIES_DATA_COL = \\'total_sold\\',\n",
-        "        TIME_SERIES_ID_COL = [\\'product_name\\']\n",
+        "        TIME_SERIES_ID_COL = [\\'product_name\\'],\n",
+        "        MODEL_REGISTRY = \\'vertex_ai\\',\n",
+        "        VERTEX_AI_MODEL_ID = \\'order_demand_forecasting\\',\n",
+        "        VERTEX_AI_MODEL_VERSION_ALIASES = [\\'staging\\']\n",
         "        ) AS\n",
         "        SELECT\n",
         "          hourly_timestamp,\n",
@@ -1144,7 +1119,7 @@
         "            project=project,\n",
         "            location=location,\n",
         "        )\n",
-        "        .set_display_name(\"run arima+ model experiment\")\n",
+        "        .set_display_name(\"train arima plus model\")\n",
         "        .after(create_training_dataset_op)\n",
         "    )\n",
         "\n",
@@ -1153,12 +1128,12 @@
         "        BigqueryMLArimaEvaluateJobOp(\n",
         "            project=project,\n",
         "            location=location,\n",
-        "            model=bq_arima_model_exp_op.outputs[\"model\"],\n",
+        "            model=bq_arima_model_op.outputs[\"model\"],\n",
         "            show_all_candidate_models=False,\n",
         "            job_configuration_query=bq_evaluate_time_series_configuration,\n",
         "        )\n",
         "        .set_display_name(\"evaluate arima plus time series\")\n",
-        "        .after(bq_arima_model_exp_op)\n",
+        "        .after(bq_arima_model_op)\n",
         "    )\n",
         "\n",
         "    # Evaluate ARIMA Plus model\n",
@@ -1166,12 +1141,12 @@
         "        BigqueryEvaluateModelJobOp(\n",
         "            project=project,\n",
         "            location=location,\n",
-        "            model=bq_arima_model_exp_op.outputs[\"model\"],\n",
+        "            model=bq_arima_model_op.outputs[\"model\"],\n",
         "            query_statement=f\"\"\"SELECT * FROM `{project}.{bq_dataset}.{bq_training_table}` WHERE split='TEST'\"\"\",\n",
         "            job_configuration_query=bq_evaluate_model_configuration,\n",
         "        )\n",
         "        .set_display_name(\"evaluate arima plus model\")\n",
-        "        .after(bq_arima_model_exp_op)\n",
+        "        .after(bq_arima_model_op)\n",
         "    )\n",
         "\n",
         "    # Plot model metrics\n",
@@ -1189,34 +1164,6 @@
         "        < PERF_THRESHOLD,\n",
         "        name=\"avg. mae good\",\n",
         "    ):\n",
-        "        # Train the ARIMA PLUS model\n",
-        "        bq_arima_model_op = (\n",
-        "            BigqueryCreateModelJobOp(\n",
-        "                query=f\"\"\"\n",
-        "        -- create model table\n",
-        "        CREATE OR REPLACE MODEL `{project}.{bq_dataset}.{bq_model_table}`\n",
-        "        OPTIONS(\n",
-        "        MODEL_TYPE = \\'ARIMA_PLUS\\',\n",
-        "        TIME_SERIES_TIMESTAMP_COL = \\'hourly_timestamp\\',\n",
-        "        TIME_SERIES_DATA_COL = \\'total_sold\\',\n",
-        "        TIME_SERIES_ID_COL = [\\'product_name\\'],\n",
-        "        MODEL_REGISTRY = \\'vertex_ai\\',\n",
-        "        VERTEX_AI_MODEL_ID = \\'order_demand_forecasting\\',\n",
-        "        VERTEX_AI_MODEL_VERSION_ALIASES = [\\'staging\\']\n",
-        "        ) AS\n",
-        "        SELECT\n",
-        "          DATETIME_TRUNC(time_of_sale, HOUR) as hourly_timestamp,\n",
-        "          product_name,\n",
-        "          SUM(quantity) AS total_sold,\n",
-        "          FROM `{project}.{bq_dataset}.{bq_orders_table}`\n",
-        "        GROUP BY hourly_timestamp, product_name;\n",
-        "        \"\"\",\n",
-        "                project=project,\n",
-        "                location=location,\n",
-        "            )\n",
-        "            .set_display_name(\"train arima+ model\")\n",
-        "            .after(get_evaluation_model_metrics_op)\n",
-        "        )\n",
         "\n",
         "        # Generate the ARIMA PLUS forecasts\n",
         "        bq_arima_forecast_op = (\n",
@@ -1277,7 +1224,7 @@
       "source": [
         "### Execute your pipeline\n",
         "\n",
-        "Next, we execute the pipeline. It takes the following parameters which we set as default:\n",
+        "Next, you execute the pipeline. It takes the following parameters which we set as default:\n",
         "\n",
         "- `bq_dataset`: The BigQuery dataset to train on.\n",
         "- `bq_orders_table` : The BigQuery table of raw data.\n",
@@ -1319,7 +1266,7 @@
       "source": [
         "### View BigQuery ML training pipeline results\n",
         "\n",
-        "Finally, you view the artifact outputs of each task in the pipeline."
+        "Finally, you will view the artifact outputs of each task in the pipeline."
       ]
     },
     {
@@ -1395,8 +1342,8 @@
         "print(\"bigquery-ml-arima-evaluate-job\")\n",
         "artifacts = print_pipeline_output(bqml_pipeline, \"bigquery-ml-arima-evaluate-job\")\n",
         "print(\"\\n\\n\")\n",
-        "print(\"bigquery-evaluate-model-job\")\n",
-        "artifacts = print_pipeline_output(bqml_pipeline, \"bigquery-evaluate-model-job\")\n",
+        "print(\"get-model-evaluation-metrics\")\n",
+        "artifacts = print_pipeline_output(bqml_pipeline, \"get-model-evaluation-metrics\")\n",
         "print(\"\\n\\n\")\n",
         "print(\"bigquery-forecast-model-job\")\n",
         "artifacts = print_pipeline_output(bqml_pipeline, \"bigquery-forecast-model-job\")\n",
@@ -1460,8 +1407,7 @@
         "\n",
         "# Remove local resorces\n",
         "! rm -rf {KFP_COMPONENTS_PATH}\n",
-        "! rm -rf {PIPELINES_PATH}\n",
-        "! rm -rf {DATA_PATH}"
+        "! rm -rf {PIPELINES_PATH}"
       ]
     }
   ],

notebooks/community/pipelines/google_cloud_pipeline_components_cloud_natural_language_pipeline.ipynb

@@ -42,12 +42,12 @@
         "<table align=\"left\">\n",
         "\n",
         "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/pipelines/google_cloud_pipeline_components_cloud_natural_language_pipeline.ipynb\">\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/natural_language/cloud_natural_language_pipeline.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/pipelines/google_cloud_pipeline_components_cloud_natural_language_pipeline.ipynb\">\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/natural_language/cloud_natural_language_pipeline.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
@@ -328,7 +328,7 @@
       },
       "outputs": [],
       "source": [
-        "PROJECT_ID = \"your-project-id\"  # @param {type:\"string\"}\n",
+        "PROJECT_ID = \"cloud-ml-language-test\"  # @param {type:\"string\"}\n",
         "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n",
         "    # Get your GCP project id from gcloud\n",
         "    shell_output = !gcloud config list --format 'value(core.project)' 2>/dev/null\n",
@@ -368,7 +368,7 @@
       "source": [
         "REGION = \"us\"  # @param {type:\"string\"}\n",
         "LOCATION = \"us-central1\"  # @param {type:\"string\"}\n",
-        "TRAINING_DATA_LOCATION = \"gs://your-training-data-location\"  # @param {type:\"string\"}\n",
+        "TRAINING_DATA_LOCATION = \"gs://dougchen-20221130-pipeline-colab-test/data-00001-of-00001.jsonl\"  # @param {type:\"string\"}\n",
         "TASK_TYPE = \"CLASSIFICATION\"  # @param [\"CLASSIFICATION\", \"MULTILABEL_CLASSIFICATION\"]"
       ]
     },
@@ -740,7 +740,7 @@
         "\n",
         "This sends a create pipeline job request to Vertex Pipelines. Note that this  task run synchronously and may take a while to complete.\n",
         "\n",
-        "You may view the progress of the job at any time by clicking on the generated links (after \"View Pipeline Job\" in the console output of the cell below). Once the pipeline finishes, you may examine the artifacts produced from this pipeline."
+        "You may view the progress of the job at any time by clicking on the generated links (after \"View Pipeline Job\" in the console output of the cell below). Once the pipeline finishes, you may examine the artifacts produced from this pipeline. See "
       ]
     },
     {

notebooks/community/pipelines/google_cloud_pipeline_components_ready_to_go_text_classification_pipeline.ipynb

@@ -42,12 +42,12 @@
         "<table align=\"left\">\n",
         "\n",
         "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/pipelines/google_cloud_pipeline_components_ready_to_go_text_classification_pipeline.ipynb\">\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/master/community-content/ready_to_go_text_classification_pipeline/ready_to_go_text_classification_pipeline.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/pipelines/google_cloud_pipeline_components_ready_to_go_text_classification_pipeline.ipynb\">\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/community-content/ready_to_go_text_classification_pipeline/ready_to_go_text_classification_pipeline.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
@@ -706,7 +706,7 @@
         ")\n",
         "\n",
         "# The GCS directory for keeping staging files for model evaluation.\n",
-        "ROOT_DIR = \"'f\\\"{BASE_OUTPUT_DIR}/root\\\"'\"  # @param {type:\"string\"}"
+        "ROOT_DIR = 'f\"{BASE_OUTPUT_DIR}/root\"'  # @param {type:\"string\"}"
       ]
     },
     {

notebooks/community/vertex_endpoints/optimized_tensorflow_runtime/benchmark.py

@@ -112,7 +112,7 @@ def benchmark(
 
     results = []
     for qps in qps_list:
-        num_requests = int(max(qps * duration_sec, 10))
+        num_requests = max(qps * duration_sec, 10)
         requests_for_qps = list(
             itertools.islice(itertools.cycle(requests), num_requests)
         )

notebooks/community/vertex_endpoints/torchserve/dreambooth_stablediffusion.ipynb

@@ -1,987 +0,0 @@
-{
-  "cells": [
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "18ebbd838e32"
-      },
-      "outputs": [],
-      "source": [
-        "# Copyright 2023 Google LLC\n",
-        "#\n",
-        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
-        "# you may not use this file except in compliance with the License.\n",
-        "# You may obtain a copy of the License at\n",
-        "#\n",
-        "# https://www.apache.org/licenses/LICENSE-2.0\n",
-        "#\n",
-        "# Unless required by applicable law or agreed to in writing, software\n",
-        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
-        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
-        "# See the License for the specific language governing permissions and\n",
-        "# limitations under the License."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "219f1b1fe8fe"
-      },
-      "source": [
-        "# Deploy and host a Stable Diffusion model on Vertex AI\n"
-      ]
-    },
-    {
-      "attachments": {},
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "JAPoU8Sm5E6e"
-      },
-      "source": [
-        "<table align=\"left\">\n",
-        "\n",
-        "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/vertex_endpoints/torchserve/dreambooth_stablediffusion.ipynb\">\n",
-        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
-        "    </a>\n",
-        "  </td>\n",
-        "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/vertex_endpoints/torchserve/dreambooth_stablediffusion.ipynb\">\n",
-        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
-        "      View on GitHub\n",
-        "    </a>\n",
-        "  </td>\n",
-        " <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/vertex_endpoints/torchserve/dreambooth_stablediffusion.ipynb\">\n",
-        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-        "      Open in Vertex AI Workbench\n",
-        "    </a>\n",
-        "  </td>\n",
-        "</table>"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "fce05a8186d6"
-      },
-      "source": [
-        "## Overview\n",
-        "\n",
-        "This notebook demonstrates how to deploy and host a fine-tuned [Stable Diffusion 1.5](https://huggingface.co/runwayml/stable-diffusion-v1-5) model on Vertex AI. For hosting, you use the PyTorch 3 container built for Vertex AI with [TorchServe](https://pytorch.org/serve/index.html)."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "c76216b03fec"
-      },
-      "source": [
-        "### Objective\n",
-        "\n",
-        "In this tutorial, you learn how to host and deploy a Stable Diffusion 1.5 model on Vertex AI.\n",
-        "\n",
-        "This tutorial uses the following Google Cloud ML services:\n",
-        "\n",
-        "+ Vertex AI `Model` resource\n",
-        "+ Vertex AI `Endpoint` resource\n",
-        "\n",
-        "The steps performed include:\n",
-        "\n",
-        "+ Create a `torchserve` handler for responding to prediction requests.\n",
-        "+ Upload a Stable Diffusion 1.5 model on a prebuilt PyTorch container in Vertex AI.\n",
-        "+ Deploy a model to a Vertex AI Endpoint.\n",
-        "+ Send requests to the endpoint and parse the responses using Vertex AI Prediction service."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "c6deba5a8557"
-      },
-      "source": [
-        "### Model\n",
-        "\n",
-        "This notebook uses a collection of model artifacts fine-tuned to generate images of a small dog. These are the same images used in the original [DreamBooth paper](https://dreambooth.github.io/)."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "911dc651ea9c"
-      },
-      "source": [
-        "### Costs\n",
-        "\n",
-        "This tutorial uses billable components of Google Cloud:\n",
-        "\n",
-        "* Vertex AI models\n",
-        "* Vertex AI endpoints\n",
-        "* Vertex AI prediction\n",
-        "* Cloud Storage\n",
-        "* (Optionally) Vertex AI Workbench\n",
-        "\n",
-        "Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing) and [Cloud Storage pricing](https://cloud.google.com/storage/pricing), and use the [Pricing Calculator](https://cloud.google.com/products/calculator/) to generate a cost estimate based on your projected usage."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "0d36bd3d53fa"
-      },
-      "source": [
-        "## Hardware requirements\n",
-        "\n",
-        "This notebook requires that you use a GPU with a sufficient amount of VRAM available. It was tested on a `NVIDIA Tesla A100 GPU` with 85 GB of VRAM. Run the following cell to ensure that you have the correct hardware configuration."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "3dd4022552e5"
-      },
-      "outputs": [],
-      "source": [
-        "!nvidia-smi --query-gpu=name,memory.total,memory.free --format=\"csv,noheader\""
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "a782627e5f73"
-      },
-      "source": [
-        "### Create a user-managed notebook on Vertex AI\n",
-        "\n",
-        "If you are using Vertex AI Workbench, you can create a notebook with the correct configuration by doing the following:\n",
-        "\n",
-        "+ Go to [Vertex AI Workbench](https://console.cloud.google.com/vertex-ai/workbench/user-managed) in the Google Cloud Console.\n",
-        "+ Click **New Notebook** and then click **PyTorch 1.13** > **With 1 NVIDIA T4**.\n",
-        "+ In the **New notebook** dialog box, click **Advanced Options**. The **Create a user-managed notebook** page opens up.\n",
-        "+ In the **Create a user-managed notebook** page, do the following:\n",
-        "  * In the **Notebook name** box, type a name for your notebook, for example \"my-stablediffusion-nb\".\n",
-        "  * In the **Machine type** drop-down, select **A2 highgpu** > **a2-highgpu-1g**.\n",
-        "  * In the **GPU type** drop-down, select **NVIDIA Tesla A100**.\n",
-        "  * Check the box next to **Install NVIDIA GPU driver automatically for me**\n",
-        "  * Expand **Disk(s)** and do the following:\n",
-        "    - Under **Boot disk type**, select **SSD Persistent Disk**.\n",
-        "    - Under **Data disk type**, select **SSD Persistent Disk**.\n",
-        "  * Click **Create**."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "FAPoU8Sm5E6e"
-      },
-      "source": [
-        "<div style=\"background:#feefe3; padding:5px; color:#aa0000\">\n",
-        "<strong>Caution:</strong> Using a Vertex AI Workbench notebook with the above configuration can increase your costs significantly. You can estimate your costs using the <a href=\"https://cloud.google.com/products/calculator\"><u>costs calculator</u></a>.</div>"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "0bb4201cc99a"
-      },
-      "source": [
-        "## Installation\n",
-        "\n",
-        "Install the following packages required to execute this notebook.\n",
-        "\n",
-        "**Note**: You might need to change the version of PyTorch (`torch`) installed by `pip`."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "9c769df171a6"
-      },
-      "outputs": [],
-      "source": [
-        "%%writefile requirements.txt\n",
-        "diffusers\n",
-        "ftfy\n",
-        "google-cloud-aiplatform\n",
-        "gradio\n",
-        "ninja\n",
-        "tensorboard==1.15.0\n",
-        "torch\n",
-        "torchaudio\n",
-        "torchvision\n",
-        "torchserve\n",
-        "torch-model-archiver\n",
-        "torch-workflow-archiver\n",
-        "transformers"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "e46804ac90d8"
-      },
-      "outputs": [],
-      "source": [
-        "%pip install -r requirements.txt"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "58707a750154"
-      },
-      "source": [
-        "### Colab only: Uncomment the following cell to restart the kernel."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "77c11549298a"
-      },
-      "outputs": [],
-      "source": [
-        "# Automatically restart kernel after installs so that your environment can access the new packages\n",
-        "# import IPython\n",
-        "\n",
-        "# app = IPython.Application.instance()\n",
-        "# app.kernel.do_shutdown(True)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "294df346a918"
-      },
-      "source": [
-        "## Before you begin"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "BF1j6f9HApxa"
-      },
-      "source": [
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "2. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-        "\n",
-        "3. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com).\n",
-        "\n",
-        "4. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk)."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "WReHDGG5g0XY"
-      },
-      "source": [
-        "#### Set your project ID\n",
-        "\n",
-        "**If you don't know your project ID**, try the following:\n",
-        "* Run `gcloud config list`.\n",
-        "* Run `gcloud projects list`.\n",
-        "* See the support page: [Locate the project ID](https://support.google.com/googleapi/answer/7014113)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "oM1iC_MfAts1"
-      },
-      "outputs": [],
-      "source": [
-        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}\n",
-        "\n",
-        "# Set the project id\n",
-        "! gcloud config set project {PROJECT_ID}"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "region"
-      },
-      "source": [
-        "#### Region\n",
-        "\n",
-        "You can also change the `REGION` variable used by Vertex AI. Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "region"
-      },
-      "outputs": [],
-      "source": [
-        "REGION = \"us-central1\""
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "sBCra4QMA2wR"
-      },
-      "source": [
-        "### Authenticate your Google Cloud account\n",
-        "\n",
-        "Depending on your Jupyter environment, you may have to manually authenticate. Follow the relevant instructions below."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "74ccc9e52986"
-      },
-      "source": [
-        "**1. Vertex AI Workbench**\n",
-        "* Do nothing as you are already authenticated."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "de775a3773ba"
-      },
-      "source": [
-        "**2. Local JupyterLab instance, uncomment and run:**"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "254614fa0c46"
-      },
-      "outputs": [],
-      "source": [
-        "# ! gcloud auth login"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "ef21552ccea8"
-      },
-      "source": [
-        "**3. Colab, uncomment and run:**"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "603adbbf0532"
-      },
-      "outputs": [],
-      "source": [
-        "# from google.colab import auth\n",
-        "# auth.authenticate_user()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "f6b2ccc891ed"
-      },
-      "source": [
-        "**4. Service account or other**\n",
-        "* See how to grant Cloud Storage permissions to your service account at https://cloud.google.com/storage/docs/gsutil/commands/iam#ch-examples."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "cb5c4ca3e851"
-      },
-      "source": [
-        "### Import libraries"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "7348591eda51"
-      },
-      "outputs": [],
-      "source": [
-        "import base64\n",
-        "import math\n",
-        "\n",
-        "import torch\n",
-        "from diffusers import StableDiffusionPipeline\n",
-        "from google.cloud import aiplatform\n",
-        "from IPython import display\n",
-        "from PIL import Image\n",
-        "from torch import autocast"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "697566b5f660"
-      },
-      "source": [
-        "## Optional: View model inferences\n",
-        "\n",
-        "Before uploading the model to Vertex AI, you can review the expected output from the model. The model used in this notebook is available for your use and can be downloaded from Cloud Storage. This download may take a few minutes to complete."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "d63df8d91215"
-      },
-      "outputs": [],
-      "source": [
-        "!gsutil -m cp gs://cloud-samples-data/vertex-ai/model-deployment/models/stable-diffusion/model_artifacts.zip \\\n",
-        "   ."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "89d613ae9573"
-      },
-      "outputs": [],
-      "source": [
-        "!unzip model_artifacts.zip"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "9192ea4f3b57"
-      },
-      "source": [
-        "### Create new images\n",
-        "\n",
-        "With everything in place, you can now generate new images from the Stable Diffusion model. First you must load your model into a `StableDiffusionPipeline`."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "cd1f30223b79"
-      },
-      "outputs": [],
-      "source": [
-        "model_path = \"model_artifacts\"\n",
-        "\n",
-        "pipe = StableDiffusionPipeline.from_pretrained(\n",
-        "    model_path, torch_dtype=torch.float16\n",
-        ").to(\"cuda\")\n",
-        "\n",
-        "g_cuda = None"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "dd91520f58cf"
-      },
-      "outputs": [],
-      "source": [
-        "g_cuda = torch.Generator(device=\"cuda\")\n",
-        "seed = 52362\n",
-        "g_cuda.manual_seed(seed)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "1e9c5d734b9f"
-      },
-      "source": [
-        "With the model loaded into a `StableDiffusionPipeline`, you can now generate results (inferences) from the model. Each set of inference requires an input (called a [prompt](https://learnprompting.org/)) that specifies what the model should create.\n",
-        "\n",
-        "You can also vary other inputs into the model, as shown in the following cell."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "a28b73de55ce"
-      },
-      "outputs": [],
-      "source": [
-        "prompt = \"photo of examplePup dog in a Monet style\"\n",
-        "\n",
-        "num_samples = 4\n",
-        "num_batches = 1\n",
-        "num_columns = 2\n",
-        "guidance_scale = 10\n",
-        "num_inference_steps = 50\n",
-        "height = 512\n",
-        "width = 512"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "11a7bb79de59"
-      },
-      "outputs": [],
-      "source": [
-        "def image_grid(imgs, cols):\n",
-        "    total = len(imgs)\n",
-        "    rows = math.ceil(total / cols)\n",
-        "\n",
-        "    w, h = imgs[0].size\n",
-        "    grid = Image.new(\"RGB\", size=(cols * w, rows * h))\n",
-        "    grid_w, grid_h = grid.size\n",
-        "\n",
-        "    for i, img in enumerate(imgs):\n",
-        "        grid.paste(img, box=(i % cols * w, i // cols * h))\n",
-        "    return grid\n",
-        "\n",
-        "\n",
-        "all_images = []\n",
-        "for _ in range(num_batches):\n",
-        "    with autocast(\"cuda\"):\n",
-        "        images = pipe(\n",
-        "            [prompt] * num_samples,\n",
-        "            height=height,\n",
-        "            width=width,\n",
-        "            num_inference_steps=num_inference_steps,\n",
-        "            guidance_scale=guidance_scale,\n",
-        "        ).images\n",
-        "        all_images.extend(images)\n",
-        "\n",
-        "\n",
-        "grid = image_grid(all_images, num_columns)\n",
-        "grid"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "bbc72963ff88"
-      },
-      "source": [
-        "## Deploy the model to Vertex AI\n",
-        "\n",
-        "You can host your Stable Diffusion 1.5 model on a Vertex AI endpoint where you can get inferences from it online. Uploading your model is a four step process: \n",
-        "\n",
-        "1. Create a custom TorchServe handler.\n",
-        "1. Upload the model artifacts onto Cloud Storage.\n",
-        "2. Create a Vertex AI model with the model artifacts and a prebuilt PyTorch container image.\n",
-        "3. Deploy the Vertex AI model onto an endpoint."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "eafbb0e0-40e6-43a0-a38e-edc54323da51"
-      },
-      "source": [
-        "### Create the custom TorchServe handler\n",
-        "\n",
-        "The model deployed to Vertex AI uses [TorchServe](https://pytorch.org/serve/) to handle requests and return responses from the model. You must create a custom TorchServe handler to include in with the model artifacts uploaded to Vertex AI.\n",
-        "\n",
-        "The handler file should be included in the directory with the other model artifacts."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "94567a87-9d74-4c87-a749-306ddaf01b61"
-      },
-      "outputs": [],
-      "source": [
-        "%%writefile model_artifacts/handler.py\n",
-        "\n",
-        "\"\"\"Customized handler for Stable Diffusion 1.5.\"\"\"\n",
-        "import base64\n",
-        "import logging\n",
-        "from io import BytesIO\n",
-        "\n",
-        "import torch\n",
-        "from diffusers import EulerDiscreteScheduler\n",
-        "from diffusers import StableDiffusionPipeline\n",
-        "from ts.torch_handler.base_handler import BaseHandler\n",
-        "\n",
-        "logger = logging.getLogger(__name__)\n",
-        "model_id = 'runwayml/stable-diffusion-v1-5'\n",
-        "\n",
-        "\n",
-        "class ModelHandler(BaseHandler):\n",
-        "\n",
-        "  def __init__(self):\n",
-        "    self.initialized = False\n",
-        "    self.map_location = None\n",
-        "    self.device = None\n",
-        "    self.use_gpu = True\n",
-        "    self.store_avg = True\n",
-        "    self.pipe = None\n",
-        "\n",
-        "  def initialize(self, context):\n",
-        "    \"\"\"Initializes the pipe.\"\"\"\n",
-        "    properties = context.system_properties\n",
-        "    gpu_id = properties.get('gpu_id')\n",
-        "\n",
-        "    self.map_location, self.device, self.use_gpu = \\\n",
-        "      ('cuda', torch.device('cuda:' + str(gpu_id)),\n",
-        "       True) if torch.cuda.is_available() else \\\n",
-        "        ('cpu', torch.device('cpu'), False)\n",
-        "\n",
-        "    # Use the Euler scheduler here instead\n",
-        "    scheduler = EulerDiscreteScheduler.from_pretrained(model_id,\n",
-        "                                                       subfolder='scheduler')\n",
-        "    pipe = StableDiffusionPipeline.from_pretrained(model_id,\n",
-        "                                                   scheduler=scheduler,\n",
-        "                                                   torch_dtype=torch.float16)\n",
-        "    pipe = pipe.to('cuda')\n",
-        "    # Uncomment the following line to reduce the GPU memory usage.\n",
-        "    # pipe.enable_attention_slicing()\n",
-        "    self.pipe = pipe\n",
-        "\n",
-        "    self.initialized = True\n",
-        "\n",
-        "  def preprocess(self, requests):\n",
-        "    \"\"\"Noting to do here.\"\"\"\n",
-        "    logger.info('requests: %s', requests)\n",
-        "    return requests\n",
-        "\n",
-        "  def inference(self, preprocessed_data, *args, **kwargs):\n",
-        "    \"\"\"Run the inference.\"\"\"\n",
-        "    images = []\n",
-        "    for pd in preprocessed_data:\n",
-        "      prompt = pd['prompt']\n",
-        "      images.extend(self.pipe(prompt).images)\n",
-        "    return images\n",
-        "\n",
-        "  def postprocess(self, output_batch):\n",
-        "    \"\"\"Converts the images to base64 string.\"\"\"\n",
-        "    postprocessed_data = []\n",
-        "    for op in output_batch:\n",
-        "      fp = BytesIO()\n",
-        "      op.save(fp, format='JPEG')\n",
-        "      postprocessed_data.append(base64.b64encode(fp.getvalue()).decode('utf-8'))\n",
-        "      fp.close()\n",
-        "    return postprocessed_data\n"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "6ace1dac0af0"
-      },
-      "source": [
-        "After creating the handler file, you must package the handler as a model archiver (MAR) file. The output file must be named 'model.mar'."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "67707f95d440"
-      },
-      "outputs": [],
-      "source": [
-        "!torch-model-archiver \\\n",
-        "  -f \\\n",
-        "  --model-name model \\\n",
-        "  --version 1.0 \\\n",
-        "  --handler model_artifacts/handler.py \\\n",
-        "  --export-path model_artifacts"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "ffab030f4bc8"
-      },
-      "source": [
-        "### Upload the model artifacts to Cloud Storage\n",
-        "\n",
-        "Create a new folder in your Cloud Storage bucket to hold the model artifacts"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "MzGDU7TWdts_"
-      },
-      "outputs": [],
-      "source": [
-        "BUCKET_NAME = \"your-bucket-name-unique\"  # @param {type:\"string\"}\n",
-        "BUCKET_URI = f\"gs://{BUCKET_NAME}/\"\n",
-        "FULL_GCS_PATH = f\"{BUCKET_URI}model_artifacts\""
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "-EcIXiGsCePi"
-      },
-      "source": [
-        "**Only if your bucket doesn't already exist**: Run the following cell to create your Cloud Storage bucket."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "NIq7R4HZCfIc"
-      },
-      "outputs": [],
-      "source": [
-        "! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "971232e28657"
-      },
-      "source": [
-        "Next, upload the model archive file and your trained Stable Diffusion 1.5 model to the folder on Cloud Storage."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "ef6baf44c808"
-      },
-      "outputs": [],
-      "source": [
-        "!gsutil cp -r model_artifacts $BUCKET_URI"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "402370ca9396"
-      },
-      "source": [
-        "### Create the Vertex AI model\n",
-        "\n",
-        "Once you've uploaded the model artifacts into a Cloud Storage bucket, you can create a new Vertex AI model. This notebook uses the [Vertex AI SDK](https://cloud.google.com/vertex-ai/docs/start/use-vertex-ai-python-sdk) to create the model."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "b6c58a74a0fd"
-      },
-      "outputs": [],
-      "source": [
-        "PYTORCH_PREDICTION_IMAGE_URI = (\n",
-        "    \"us-docker.pkg.dev/vertex-ai/prediction/pytorch-gpu.1-12:latest\"\n",
-        ")\n",
-        "APP_NAME = \"my-stable-diffusion\"\n",
-        "VERSION = 1\n",
-        "MODEL_DISPLAY_NAME = \"stable_diffusion_1_5-unique\"\n",
-        "MODEL_DESCRIPTION = \"stable_diffusion_1_5 container\"\n",
-        "ENDPOINT_DISPLAY_NAME = f\"{APP_NAME}-endpoint\""
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "07c3503a1a2e"
-      },
-      "outputs": [],
-      "source": [
-        "aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=BUCKET_NAME)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "FAPoU8Sm5E6e"
-      },
-      "source": [
-        "<div style=\"background:#e3effe; padding:5px; color:#0000aa\">\n",
-        "<strong>Note:</strong> The next cell fails if you haven't <a href=\"https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com\"><u>enabled the Vertex API</u></a>.</div>"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "a776324dd16f"
-      },
-      "outputs": [],
-      "source": [
-        "model = aiplatform.Model.upload(\n",
-        "    display_name=MODEL_DISPLAY_NAME,\n",
-        "    description=MODEL_DESCRIPTION,\n",
-        "    serving_container_image_uri=PYTORCH_PREDICTION_IMAGE_URI,\n",
-        "    artifact_uri=FULL_GCS_PATH,\n",
-        ")\n",
-        "\n",
-        "model.wait()\n",
-        "\n",
-        "print(model.display_name)\n",
-        "print(model.resource_name)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "0fbc3d371574"
-      },
-      "source": [
-        "### Deploy the model to an endpoint\n",
-        "\n",
-        "To get online preductions from your Stable Diffusion 2.0 model, you must [deploy it to a Vertex AI endpoint](https://cloud.google.com/vertex-ai/docs/predictions/overview). You can again use the Vertex AI SDK to create the endpoint and deploy your model."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "ab29f0a770cb"
-      },
-      "outputs": [],
-      "source": [
-        "endpoint = aiplatform.Endpoint.create(display_name=ENDPOINT_DISPLAY_NAME)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "25f703df88c7"
-      },
-      "outputs": [],
-      "source": [
-        "model.deploy(\n",
-        "    endpoint=endpoint,\n",
-        "    deployed_model_display_name=MODEL_DISPLAY_NAME,\n",
-        "    machine_type=\"n1-standard-8\",\n",
-        "    accelerator_type=\"NVIDIA_TESLA_P100\",\n",
-        "    accelerator_count=1,\n",
-        "    traffic_percentage=100,\n",
-        "    deploy_request_timeout=1200,\n",
-        "    sync=True,\n",
-        ")"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "c9fc560df0a9"
-      },
-      "source": [
-        "The previous cell, which deploys your model to the endpoint, can take a while to complete. If the previous cell times out before returning, your endpoint might still be successfully deployed to an endpoint. Check the [Cloud Console](https://console.cloud.google.com/vertex-ai/endpoints) to verify the results.\n",
-        "\n",
-        "You can also extend the time to wait for deployment by changing the `deploy_request_timeout` argument passed to `model.deploy()`."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "88a5304dfdc9"
-      },
-      "source": [
-        "## Get online predictions\n",
-        "\n",
-        "Finally, with your Stable Diffusion 1.5 model deployed to a Vertex AI endpoint, you can now get online predictions from it. Using the Vertex AI SDK, you only need a few lines of code to get an inference."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "0d6bc4aa34d6"
-      },
-      "outputs": [],
-      "source": [
-        "instances = [{\"prompt\": \"An examplePup dog with a baseball jersey.\"}]\n",
-        "response = endpoint.predict(instances=instances)\n",
-        "\n",
-        "with open(\"img5.jpg\", \"wb\") as g:\n",
-        "    g.write(base64.b64decode(response.predictions[0]))"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "65bafefda60c"
-      },
-      "outputs": [],
-      "source": [
-        "display.Image(\"img5.jpg\")"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "TpV-iwP9qw9c"
-      },
-      "source": [
-        "## Cleaning up\n",
-        "\n",
-        "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
-        "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
-        "\n",
-        "Otherwise, you can delete the individual resources you created in this tutorial:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "sx_vKniMq9ZX"
-      },
-      "outputs": [],
-      "source": [
-        "import os\n",
-        "\n",
-        "# Delete endpoint resource\n",
-        "endpoint.undeploy_all()\n",
-        "endpoint.delete()\n",
-        "\n",
-        "# Delete model resource\n",
-        "model.delete()\n",
-        "\n",
-        "# Delete Cloud Storage objects that were created\n",
-        "delete_bucket = False\n",
-        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
-        "    ! gsutil -m rm -r $BUCKET_URI"
-      ]
-    }
-  ],
-  "metadata": {
-    "colab": {
-      "name": "dreambooth_stablediffusion.ipynb",
-      "toc_visible": true
-    },
-    "kernelspec": {
-      "display_name": "Python 3",
-      "name": "python3"
-    }
-  },
-  "nbformat": 4,
-  "nbformat_minor": 0
-}

notebooks/notebook_template.ipynb

@@ -74,9 +74,7 @@
       "source": [
         "## Overview\n",
         "\n",
-        "{TODO: Include a paragraph or two explaining what this example demonstrates, who should be interested in it, and what you need to know before you get started.}\n",
-        "\n",
-        "Learn more about [web-doc-title](linkback-to-webdoc-page). {TODO: if more than one primary feature, add tag/linkback for each one}"
+        "{TODO: Include a paragraph or two explaining what this example demonstrates, who should be interested in it, and what you need to know before you get started.}"
       ]
     },
     {

notebooks/notebook_template_review.py

@@ -12,10 +12,6 @@
         --errors-codes: A list of error codes to report errors. Otherwise, all errors are reported.
         --errors-csv: Report errors in CSV format
         
-        # options for automatic fixing
-        --fix: Automatic fix
-        --fix-codes: A list of fix codes to fix. Otherwise, all fix codes are enabled.
-        
         # index generatation
         --repo: Generate index in markdown format
         --web: Generate index in HTML format
@@ -23,7 +19,6 @@
         --desc: Add description to index
         --steps: Add steps to index
         --uses: Add "resources" used to index
-        --linkback: Add linkback to index
         
     Format of CSV file for notebooks to review:
     
@@ -695,26 +690,7 @@ class ObjectiveRule(NotebookRule):
             ret = notebook.report_error(ErrorCode.ERROR_OBJECTIVE_MISSING_DESC, "Objective section missing desc")
         else:
             self.desc = self.desc.lstrip()
-            
-            bracket = False
-            paren = False
-            sentences = ""
-            for _ in range(len(self.desc)):
-                if self.desc[_] == '[':
-                    bracket = True
-                    continue
-                elif self.desc[_] == ']':
-                    bracket = False
-                    continue
-                elif self.desc[_] == '(':
-                    paren = True
-                elif self.desc[_] == ')':
-                    paren = False
-                    continue
-                    
-                if not paren:
-                    sentences += self.desc[_]
-            sentences = sentences.split('.')
+            sentences = self.desc.split('.')
             if len(sentences) > 1:
                 self.desc = sentences[0] + '.\n'
             if self.desc.startswith('In this tutorial, you learn') or self.desc.startswith('In this notebook, you learn'):
@@ -1140,44 +1116,32 @@ def add_index(path: str,
             print(f'            {tag.strip()}<br/>\n')
         print('        </td>')
         print('        <td>')
-        print(f'            <b>{title}</b>. ')
+        print(f'            <b>{title}</b><br/>\n')
         if args.desc:
             desc = replace_cl(desc.replace('`', ''))
             print('<br/>')
-            print(f'            {desc}\n')
+            print(f'            {desc}<br/>\n')
             
+        if args.steps:
+            steps = replace_cl(steps.replace('\n', '<br/>').replace('-', '&nbsp;&nbsp;-').replace('**', '').replace('*', '&nbsp;&nbsp;-').replace('`', ''))
+            print('<br/>' + steps +  '<br/>')
             
         if args.linkback and linkbacks:
             num = len(tags)
             for _ in range(num):
                 if linkbacks[_].startswith("vertex-ai"):
-                    print(f' Learn more about <a href="https://cloud.google.com/{linkbacks[_]}" target="_blank">{replace_cl(tags[_])}</a>.\n')
+                    print(f'<br/>            Learn more about <a href="https://cloud.google.com/{linkbacks[_]}" target="_blank">{replace_cl(tags[_])}</a>.\n')
                 else:
-                    print(f' Learn more about <a href="{linkbacks[_]}" target="_blank">{replace_cl(tags[_])}</a>.\n')
-                    
-        if args.steps:
-            print("<devsite-expandable>\n")
-            print('  <p class="showalways">Tutorial steps</p>\n')
-            print('  <ul>\n')
-            
-            if ":" in steps:
-                steps = steps.split(':')[1].replace('*', '').replace('-', '').strip().split('\n')
-            else:
-                steps = []
-              
-            for step in steps:
-                print(f'    <li>{replace_cl(step)}</li>\n')
-            print('  </ul>\n')
-            print("</devsite-expandable>\n")
+                    print(f'<br/>            Learn more about <a href="{linkbacks[_]}" target="_blank">{replace_cl(tags[_])}</a>.\n')
                     
         print('        </td>')
         print('        <td>')
         if colab_link:
-            print(f'            <a href="{colab_link}" target="_blank" class="external" track-type="notebookTutorial" track-name="colabLink">Colab</a><br/>\n')
+            print(f'            <a href="{colab_link}" target="_blank">Colab</a><br/>\n')
         if git_link:
-            print(f'            <a href="{git_link}" target="_blank" class="external" track-type="notebookTutorial" track-name="gitHubLink">GitHub</a><br/>\n')
+            print(f'            <a href="{git_link}" target="_blank">GitHub</a><br/>\n')
         if workbench_link:
-            print(f'            <a href="{workbench_link}" target="_blank" class="external" track-type="notebookTutorial" track-name="workbenchLink">Vertex AI Workbench</a><br/>\n')
+            print(f'            <a href="{workbench_link}" target="_blank">Vertex AI Workbench</a><br/>\n')
         print('        </td>')
         print('    </tr>\n')
     elif args.repo:
@@ -1240,16 +1204,14 @@ def replace_cl(text : str ) -> str:
         'Vertex AI Prediction': '{{vertex_prediction_name}}',
         'Vertex TensorBoard': '{{vertex_tensorboard_name}}',
         'Vertex AI TensorBoard': '{{vertex_tensorboard_name}}',
-        'TensorBoard': '{{vertex_tensorboard_name}}',
-        'Tensorboard': '{{vertex_tensorboard_name}}',
         'Vertex ML Metadata': '{{vertex_metadata_name}}',
         'Vertex Pipelines': '{{vertex_pipelines_name}}',
         'Vertex AI Pipelines': '{{vertex_pipelines_name}}',
         'Vertex AI Data Labeling': '{{vertex_data_labeling_name}}',
         'Vertex AI Experiments': '{{vertex_experiments_name}}',
         'Vertex Experiments': '{{vertex_experiments_name}}',
-        'Vertex AI Matching Engine': '{{vertex_matching_engine_name}}',
-        'Vertex Matching Engine': '{{vertex_matching_engine_name}}',
+        'Vertex AI Matching Engine': '{vertex_matching_engine_name}}',
+        'Vertex Matching Engine': '{vertex_matching_engine_name}}',
         'Vertex Model Monitoring': '{{vertex_model_monitoring_name}}',
         'Vertex AI Model Monitoring': '{{vertex_model_monitoring_name}}',
         'Vertex Feature Store': '{{vertex_featurestore_name}}',
@@ -1267,8 +1229,6 @@ def replace_cl(text : str ) -> str:
         'Vertex AI': '{{vertex_ai_name}}',
         
         'Cloud Storage': '{{storage_name}}',
-        'GCS': '{{storage_name}}',
-        'GCP': '{{gcp_name}}',
         'TensorFlow Enterprise': '{{tf4gcp_name}}',
         'TensorFlow': '{{tensorflow_name}}',
     }
@@ -1317,14 +1277,9 @@ if args.web:
     print('}')
     print('</style>')
     print('<table>')
-    print('    <thead>')
-    print('        <tr>')
-    print('            <th width="180px">Services</th>')
-    print('            <th>Description</th>')
-    print('            <th width="80px">Open in</th>')
-    print('        </tr>')
-    print('    </thead>')
-    print('    <tbody class="list">')
+    print('    <th width="180px">Services</th>')
+    print('    <th>Description</th>')
+    print('    <th width="80px">Open in</th>')
 
 if args.notebook_dir:
     if not os.path.isdir(args.notebook_dir):
@@ -1360,7 +1315,6 @@ else:
     exit(1)
 
 if args.web:
-    print('    </tbody>\n')
     print('</table>\n')
 
 exit(exit_code)

notebooks/notebook_template_writing_guidelines.md

@@ -23,62 +23,7 @@
     - Incorrect examples: "Let's update the field", "We'll update the field", "The user should update the field"
 - **Googlers**: Please follow our [branding guidelines](http://goto/cloud-branding).
 
-
-## Authoring guidelines
-
-### Focus
-
-Notebooks for official are expected to be narrow focused, which highlight a subset of features of a Vertex AI product/service.
-The product/feature is to be highlighted in the Overview section. For example:
-
-```
-This tutorial demonstrates using Vertex AI Training to train an XGBoost model using a XGBoost pre-built training container.
-```
-
-In the above example, the Vertex AI product/service is `Vertex AI Training` and the feature is `XGBoost pre-built training container`.
-
-### Scope
-
-Notebooks for official are expected to be narrow in scope, without extra extraneous steps. For example, if the notebook is about training, we discourage ending the notebook with deploying the model and doing an online/batch prediction. On the later, we recommend a separate notebook about prediction that uses a pretrained model.
-
-#### Training
-
-Notebooks for training should be constructed as follows:
-
-1. If the training script(s) are small, embed them in the notebook and use %writefile to store them locally.
-2. If the training script(s) are large, store them in our public bucket: gs://cloud-samples-data/vertex-ai/dataset-management/script, and use !wgets to retrieve and store the script locally.
-3. Train the model using the Vertex AI SDK methods for custom training.
-4. Preferrable have the service upload the trained model to the Vertex AI Model Registry.
-4. Have the script do an evaluation.
-5. Retrieve the evaluation metrics and attach them as an artifact to the corresponding entry in the Model Registry.
-6. Optionally, download the model artifacts and test locally -- i.e., make a local prediction request.
-
-#### Evaluation
-
-Notebooks for evaluation should be constructed as follows:
-
-1. Use a pretrained model from a public repository.
-2. Upload the pretrained model to the Vertex AI Model Registry.
-3. Perform a model evaluation.
-4. Review the model evaluation.
-4. Attach the model evaluation to the corresponding entry in the Model Registry.
-
-#### Prediction
-
-Notebooks for prediction should be constructed as follows:
-
-1. Use a pretrained model from a public repository.
-2. If relevant, attach a serving function to the model artifacts.
-3. Upload the pretrained model to the Vertex AI Model Registry.
-4. For online:<br/>
-    A. Deploy the model.<br/>
-    B. Perform an online prediction.</br>
-    C. Review the result.
-5. For batch:<br/>
-    A. Perform a batch prediction.</br/>
-    B. Review the result.
-    
-## Code
+### Code
 
 - Put all your installs and imports in a setup section.
 - Save the notebook with the Table of Contents open.
@@ -86,7 +31,7 @@ Notebooks for prediction should be constructed as follows:
 - Follow the [Google Python Style guide](https://github.com/google/styleguide/blob/gh-pages/pyguide.md) and write readable code.
 - Keep cells small (max ~20 lines).
 
-### TensorFlow code style
+## TensorFlow code style
 
 Use the highest level API that gets the job done (unless the goal is to demonstrate the low level API). For example, when using Tensorflow:
 

notebooks/official/CODEOWNERS

@@ -29,14 +29,17 @@
 /pipelines/google_cloud_pipelines_dataproc_tabular @inardini
 /automl/automl_forecasting_bqml_arima_plus_comparison.ipynb @TheMichaelHu
 /automl/automl_tabular_on_vertex_pipelines.ipynb @helinwang
-/custom/custom_training_tensorboard_profiler.ipynb @gericdong
-/custom/get_started_with_vertex_endpoint_and_shared_vm.ipynb @andrewferlitsch
+/custom/custom_training_tensorboard_profiler.ipynb @itseric
 /workbench/spark/spark_sample_notebook.ipynb @bradmiro
 /workbench/spark/spark_ml.ipynb @bradmiro
 /model_registry/bqml_vertexai_model_registry.ipynb @soheilazangeneh
+/workbench/exploratory_data_analysis/explore_data_in_bigquery_with_workbench.ipynb @alokpattani
 /model_evaluation/automl_tabular_classification_model_evaluation.ipynb @soheilazangeneh
 /model_evaluation/automl_tabular_regression_model_evaluation.ipynb @soheilazangeneh
-/tabular_workflows/prophet_on_vertex_pipelines.ipynb @TheMichaelHu
+/tabular_workflows/tabnet_on_vertex_pipelines.ipynb @sakagarwal
+/tabular_workflows/wide_and_deep_on_vertex_pipelines.ipynb @sakagarwal
 /model_evaluation/custom_tabular_classification_model_evaluation.ipynb @soheilazangeneh
 /sdk/SDK_FBProphet_Forecasting_Online.ipynb @brianchunkang
-/pipelines/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines @Ark-kun
+/automl/sdk_automl_forecasting_hierarchical_batch.ipynb @ivanmkc
+/prediction/custom_batch_prediction_feature_filter.ipynb @soheilazangeneh
+/feature_store/feature_store_streaming_ingestion_sdk.ipynb @soheilazangeneh

notebooks/official/automl/README.md

@@ -14,7 +14,7 @@ The steps performed include the following:
 
 ```
 
-   Learn more about [Classification for tabular data](https://cloud.google.com/vertex-ai/docs/tabular-data/classification-regression/overview).
+   Learn more about [Tabular classification](https://cloud.google.com/vertex-ai/docs/tabular-data/classification-regression/overview).
 
 
 [Create, train, and deploy an AutoML text classification model](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/automl-text-classification.ipynb)
@@ -34,7 +34,7 @@ The steps performed include:
 
 ```
 
-   Learn more about [Classification for text data](https://cloud.google.com/vertex-ai/docs/training-overview#classification_for_text).
+   Learn more about [AutoML Text classification](https://cloud.google.com/vertex-ai/docs/text-data/classification/train-model).
 
 
 [Compare Vertex AI Forecasting and BigQuery ML ARIMA_PLUS](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/automl_forecasting_bqml_arima_plus_comparison.ipynb)
@@ -54,7 +54,9 @@ The steps performed are:
 
 ```
 
-   Learn more about [BQML ARIMA+ forecasting for tabular data](https://cloud.google.com/vertex-ai/docs/tabular-data/forecasting-arima/overview).
+   Learn more about [AutoML Forecasting](https://cloud.google.com/vertex-ai/docs/tabular-data/forecasting/overview).
+
+   Learn more about [BQML Forecasting](https://cloud.google.com/vertex-ai/docs/tabular-data/forecasting-arima/overview).
 
 
 [AutoML Tabular Workflow pipelines](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/automl_tabular_on_vertex_pipelines.ipynb)
@@ -69,26 +71,7 @@ The steps performed are:
 
 ```
 
-   Learn more about [Tabular Workflow for E2E AutoML](https://cloud.google.com/vertex-ai/docs/tabular-data/tabular-workflows/e2e-automl).
-
-
-[Get started with AutoML Training](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/get_started_automl_training.ipynb)
-
-```
-Learn how to use `AutoML` for training with `Vertex AI`.
-
-The steps performed include:
-
-- Train an image model
-- Export the image model as an edge model
-- Train a tabular model
-- Export the tabular model as a cloud model
-- Train a text model
-- Train a video model
-
-```
-
-   Learn more about [Vertex AI for AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users).
+   Learn more about [AutoML Tabular Workflows](https://cloud.google.com/vertex-ai/docs/tabular-data/tabular-workflows/e2e-automl).
 
 
 [AutoML training hierarchical forecasting for batch prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_forecasting_hierarchical_batch.ipynb)
@@ -107,7 +90,7 @@ The steps performed include:
 
 ```
 
-   Learn more about [Hierarchical forecasting for tabular data](https://cloud.google.com/vertex-ai/docs/tabular-data/forecasting/hierarchical).
+   Learn more about [AutoML Forecasting](https://cloud.google.com/vertex-ai/docs/tabular-data/forecasting/overview).
 
 
 [AutoML training image object detection model for batch prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_image_object_detection_batch.ipynb)
@@ -124,7 +107,7 @@ The steps performed include:
 
 ```
 
-   Learn more about [Object detection for image data](https://cloud.google.com/vertex-ai/docs/training-overview#object_detection_for_images).
+   Learn more about [AutoML Image](https://cloud.google.com/vertex-ai/docs/image-data/object-detection/train-model).
 
 
 [AutoML tabular forecasting model for batch prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_tabular_forecasting_batch.ipynb)
@@ -141,7 +124,7 @@ The steps performed include:
 
 ```
 
-   Learn more about [Forecasting for tabular data](https://cloud.google.com/vertex-ai/docs/tabular-data/forecasting/overview).
+   Learn more about [AutoML Forecasting](https://cloud.google.com/vertex-ai/docs/tabular-data/forecasting/tutorials-samples).
 
 
 [AutoML training tabular regression model for batch prediction using BigQuery](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_tabular_regression_batch_bq.ipynb)
@@ -160,7 +143,7 @@ The steps performed include:
 
 ```
 
-   Learn more about [Regression for tabular data](https://cloud.google.com/vertex-ai/docs/tabular-data/classification-regression/overview).
+   Learn more about [AutoML Tabular](https://cloud.google.com/vertex-ai/docs/training-overview#tabular_data).
 
 
 [AutoML training tabular regression model for online prediction using BigQuery](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_tabular_regression_online_bq.ipynb)
@@ -179,7 +162,7 @@ The steps performed include:
 
 ```
 
-   Learn more about [Regression for tabular data](https://cloud.google.com/vertex-ai/docs/tabular-data/classification-regression/overview).
+   Learn more about [AutoML Tabular](https://cloud.google.com/vertex-ai/docs/training-overview#tabular_data).
 
 
 [AutoML training text entity extraction model for online prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_text_entity_extraction_online.ipynb)
@@ -198,7 +181,7 @@ The steps performed include:
 
 ```
 
-   Learn more about [Entity extraction for text data](https://cloud.google.com/vertex-ai/docs/training-overview#entity_extraction_for_text).
+   Learn more about [AutoML Text](https://cloud.google.com/vertex-ai/docs/text-data/entity-extraction/train-model).
 
 
 [Training an AutoML text sentiment analysis model for online predictions](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_text_sentiment_analysis_online.ipynb)
@@ -218,7 +201,7 @@ The steps performed include:
 
 ```
 
-   Learn more about [Sentiment analysis for text data](https://cloud.google.com/vertex-ai/docs/training-overview#sentiment_analysis_for_text).
+   Learn more about [AutoML Text](https://cloud.google.com/vertex-ai/docs/text-data/sentiment-analysis/train-model).
 
 
 [AutoML training video action recognition model for batch prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_video_action_recognition_batch.ipynb)
@@ -235,7 +218,7 @@ The steps performed include:
 
 ```
 
-   Learn more about [Action recognition for video data](https://cloud.google.com/vertex-ai/docs/training-overview#action_recognition_for_videos).
+   Learn more about [AutoML Video](https://cloud.google.com/vertex-ai/docs/video-data/action-recognition/train-model).
 
 
 [AutoML training video classification model for batch prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_video_classification_batch.ipynb)
@@ -252,7 +235,7 @@ The steps performed include:
 
 ```
 
-   Learn more about [Classification for video data](https://cloud.google.com/vertex-ai/docs/training-overview#classification_for_videos).
+   Learn more about [AutoML Video](https://cloud.google.com/vertex-ai/docs/video-data/classification/train-model).
 
 
 [AutoML training video object tracking model for batch prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_video_object_tracking_batch.ipynb)
@@ -269,5 +252,5 @@ The steps performed include:
 
 ```
 
-   Learn more about [Object tracking for video data](https://cloud.google.com/vertex-ai/docs/training-overview#object_tracking_for_videos).
+   Learn more about [AutoML Video](https://cloud.google.com/vertex-ai/docs/video-data/object-tracking/train-model).
 

notebooks/official/automl/automl_tabular_on_vertex_pipelines.ipynb

@@ -170,7 +170,7 @@
       },
       "outputs": [],
       "source": [
-        "!pip install -U google-cloud-pipeline-components==1.0.25 -q"
+        "!pip install -U google-cloud-pipeline-components -q"
       ]
     },
     {
@@ -333,10 +333,7 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"us-central1\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if REGION == \"[your-region]\":\n",
-        "    REGION = \"us-central1\""
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}"
       ]
     },
     {
@@ -382,11 +379,8 @@
         "import os\n",
         "import sys\n",
         "\n",
-        "# If on Vertex AI Workbench, then don't execute this code\n",
-        "IS_COLAB = \"google.colab\" in sys.modules\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
-        "    \"DL_ANACONDA_HOME\"\n",
-        "):\n",
+        "# If on Google Cloud Notebook, then don't execute this code\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
         "    if \"google.colab\" in sys.modules:\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
@@ -394,9 +388,10 @@
         "\n",
         "    # If you are running this notebook locally, replace the string below with the\n",
         "    # path to your service account key and run this cell to authenticate your GCP\n",
-        "    # account.\n",
+        "    # account. Alternatively, you may edit this notebook to authenticate using\n",
+        "    # gcloud.\n",
         "    elif not os.getenv(\"IS_TESTING\"):\n",
-        "        %env GOOGLE_APPLICATION_CREDENTIALS '[your-service-account-key-path]'"
+        "        %env GOOGLE_APPLICATION_CREDENTIALS ''"
       ]
     },
     {
@@ -483,77 +478,6 @@
         "! gsutil ls -al $BUCKET_URI"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "44accda192d5"
-      },
-      "source": [
-        "#### Service Account\n",
-        "\n",
-        "You use a service account to create Vertex AI Pipeline jobs. If you do not want to use your project's Compute Engine service account, set `SERVICE_ACCOUNT` to another service account ID."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "e0c9c4f84849"
-      },
-      "outputs": [],
-      "source": [
-        "SERVICE_ACCOUNT = \"[your-service-account]\""
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "604ae09ab6d3"
-      },
-      "outputs": [],
-      "source": [
-        "if (\n",
-        "    SERVICE_ACCOUNT == \"\"\n",
-        "    or SERVICE_ACCOUNT is None\n",
-        "    or SERVICE_ACCOUNT == \"[your-service-account]\"\n",
-        "):\n",
-        "    # Get your service account from gcloud\n",
-        "    if not IS_COLAB:\n",
-        "        shell_output = !gcloud auth list 2>/dev/null\n",
-        "        SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
-        "\n",
-        "    else:  # IS_COLAB:\n",
-        "        shell_output = ! gcloud projects describe  $PROJECT_ID\n",
-        "        project_number = shell_output[-1].split(\":\")[1].strip().replace(\"'\", \"\")\n",
-        "        SERVICE_ACCOUNT = f\"{project_number}-compute@developer.gserviceaccount.com\"\n",
-        "\n",
-        "    print(\"Service Account:\", SERVICE_ACCOUNT)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "d1ecb60964d5"
-      },
-      "source": [
-        "#### Set service account access for Vertex AI Pipelines\n",
-        "Run the following commands to grant your service account access to read and write pipeline artifacts in the bucket that you created in the previous step. You only need to run this step once per service account."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "a592f0a380c2"
-      },
-      "outputs": [],
-      "source": [
-        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
-        "\n",
-        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -912,7 +836,6 @@
         "\n",
         "job.run()\n",
         "\n",
-        "\n",
         "pipeline_task_details = job.gca_resource.job_detail.task_details\n",
         "\n",
         "if export_additional_model_without_custom_ops:\n",
@@ -954,7 +877,6 @@
         "stage_1_tuner_task = get_task_detail(\n",
         "    pipeline_task_details, \"automl-tabular-stage-1-tuner\"\n",
         ")\n",
-        "\n",
         "stage_1_tuning_result_artifact_uri = (\n",
         "    stage_1_tuner_task.outputs[\"tuning_result_output\"].artifacts[0].uri\n",
         ")"

notebooks/official/automl/sdk_automl_tabular_forecasting_batch.ipynb

@@ -833,7 +833,6 @@
       ]
     },
     {
-      "attachments": {},
       "cell_type": "markdown",
       "metadata": {
         "id": "99b7a9287ba6"
@@ -841,7 +840,7 @@
       "source": [
         "For AutoML models, manual scaling can be adjusted by setting both min and max nodes i.e., `starting_replica_count` and `max_replica_count` as the same value(in this example, set to 1). The node count can be increased or decreased as required by load.\n",
         " \n",
-        "`batch_predict` can export predictions either to BigQuery or GCS. This example exports to BigQuery."
+        "`batch_predict` can export predictions either to BigQuery or GCS. The BigQuery options are commented out below and the predictions will be exported to the BUCKET_URI."
       ]
     },
     {

notebooks/official/bigquery_ml/README.md

@@ -15,25 +15,5 @@ The steps performed include:
 
 ```
 
-   Learn more about [BigQuery ML](https://cloud.google.com/vertex-ai/docs/beginner/bqml).
-
-
-[Get started with BigQuery ML Training](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/bigquery_ml/get_started_with_bqml_training.ipynb)
-
-```
-Learn how to use `BigQueryML` for training with `Vertex AI`.
-
-The steps performed include:
-
-- Create a local BigQuery table in your project
-- Train a BigQuery ML model
-- Evaluate the BigQuery ML model
-- Export the BigQuery ML model as a cloud model
-- Upload the exported model as a `Vertex AI Model` resource
-- Hyperparameter tune a BigQuery ML model with `Vertex AI Vizier`
-- Automatically register a BigQuery ML model to `Vertex AI Model Registry`
-
-```
-
-   Learn more about [BigQuery ML](https://cloud.google.com/vertex-ai/docs/beginner/bqml).
+   Learn more about [BigQuery ML](https://cloud.google.com/bigquery-ml/docs/introduction).
 

notebooks/official/bigquery_ml/bqml-online-prediction.ipynb

@@ -63,7 +63,7 @@
         "\n",
         "This notebook is aimed at data analysts and data scientists who have data in BigQuery, want to train a model using BigQuery ML, register the model to Vertex AI Model Registry, and deploy it to an endpoint for real-time prediction. \n",
         "\n",
-        "Learn more about [BigQuery ML](https://cloud.google.com/vertex-ai/docs/beginner/bqml)."
+        "Learn more about [BigQuery ML](https://cloud.google.com/bigquery-ml/docs/introduction)."
       ]
     },
     {

notebooks/official/custom/SDK_Custom_Container_Prediction.ipynb

@@ -60,7 +60,7 @@
         "\n",
         "This tutorial walks you through building a custom container to serve a scikit-learn model on Vertex AI. You use the FastAPI Python web server framework to create a prediction and health endpoint. You also incorporate a pre-processor from training pipeline into your online serving application.\n",
         "\n",
-        "Learn more about [Custom training](https://cloud.google.com/vertex-ai/docs/training/custom-training) and [Vertex AI Prediction](https://cloud.google.com/vertex-ai/docs/predictions/get-predictions)."
+        "Learn more about [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training)  and [Vertex AI Prediction](https://cloud.google.com/vertex-ai/docs/predictions/get-predictions)."
       ]
     },
     {

notebooks/official/custom/SDK_FBProphet_Forecasting_Online.ipynb

@@ -64,7 +64,7 @@
         "\n",
         "Learn more about serving an FBProphet model from this [article on testdriven.io: Deploying and Hosting a Machine Learning Model with FastAPI and Heroku](https://testdriven.io/blog/fastapi-machine-learning/).\n",
         "\n",
-        "Learn more about [Custom training](https://cloud.google.com/vertex-ai/docs/training/custom-training) and [Vertex AI Prediction](https://cloud.google.com/vertex-ai/docs/predictions/get-predictions).\n"
+        "Learn more about [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training) and [Vertex AI Prediction](https://cloud.google.com/vertex-ai/docs/predictions/get-predictions).\n"
       ]
     },
     {

notebooks/official/custom/custom-tabular-bq-managed-dataset.ipynb

@@ -147,7 +147,8 @@
         "# Install the packages\n",
         "! pip3 install --upgrade google-cloud-aiplatform \\\n",
         "                        google-cloud-storage \\\n",
-        "                        'google-cloud-bigquery[pandas]'"
+        "                        google-cloud-bigquery \\\n",
+        "                        pyarrow"
       ]
     },
     {
@@ -631,12 +632,12 @@
         "    df_train_x, df_train_y = df_train, df_train.pop(LABEL_COLUMN)\n",
         "    df_validation_x, df_validation_y = df_validation, df_validation.pop(LABEL_COLUMN)\n",
         "\n",
-        "    y_train = tf.convert_to_tensor(np.asarray(df_train_y).astype(\"float32\"))\n",
-        "    y_validation = tf.convert_to_tensor(np.asarray(df_validation_y).astype(\"float32\"))\n",
+        "    y_train = np.asarray(df_train_y).astype(\"float32\")\n",
+        "    y_validation = np.asarray(df_validation_y).astype(\"float32\")\n",
         "\n",
         "    # Convert to numpy representation\n",
-        "    x_train = tf.convert_to_tensor(np.asarray(df_train_x).astype(\"float32\"))\n",
-        "    x_test = tf.convert_to_tensor(np.asarray(df_validation_x).astype(\"float32\"))\n",
+        "    x_train = np.asarray(df_train_x) \n",
+        "    x_test = np.asarray(df_validation_x)\n",
         "\n",
         "    # Convert to one-hot representation\n",
         "    num_species = len(df_train_y.unique())\n",
@@ -734,7 +735,7 @@
         "    display_name=JOB_NAME,\n",
         "    script_path=\"task.py\",\n",
         "    container_uri=\"us-docker.pkg.dev/vertex-ai/training/tf-cpu.2-8:latest\",\n",
-        "    requirements=[\"google-cloud-bigquery[pandas]\", \"protobuf<3.20.0\"],\n",
+        "    requirements=[\"google-cloud-bigquery>=2.20.0\", \"db-dtypes\"],\n",
         "    model_serving_container_image_uri=\"us-docker.pkg.dev/vertex-ai/prediction/tf2-cpu.2-8:latest\",\n",
         ")\n",
         "\n",

notebooks/official/custom/sdk-custom-image-classification-batch.ipynb

@@ -64,7 +64,7 @@
         "\n",
         "This tutorial demonstrates how to use the Vertex AI SDK for Python to train and deploy a custom image classification model for batch prediction.\n",
         "\n",
-        "Learn more about [Custom training](https://cloud.google.com/vertex-ai/docs/training/custom-training) and [Vertex AI Batch Prediction](https://cloud.google.com/vertex-ai/docs/tabular-data/classification-regression/get-batch-predictions)."
+        "Learn more about [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training)  and [Vertex AI Batch Prediction](https://cloud.google.com/vertex-ai/docs/tabular-data/classification-regression/get-batch-predictions)."
       ]
     },
     {

notebooks/official/custom/sdk-custom-image-classification-online.ipynb

@@ -64,7 +64,7 @@
         "\n",
         "This tutorial demonstrates how to use the Vertex AI SDK for Python to train and deploy a custom image classification model for online prediction.\n",
         "\n",
-        "Learn more about [Custom training](https://cloud.google.com/vertex-ai/docs/training/custom-training) and [Vertex AI Prediction](https://cloud.google.com/vertex-ai/docs/predictions/get-predictions)."
+        "Learn more about [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training)  and [Vertex AI Prediction](https://cloud.google.com/vertex-ai/docs/predictions/get-predictions)."
       ]
     },
     {

notebooks/official/datasets/get_started_bq_datasets.ipynb

@@ -29,7 +29,7 @@
         "id": "title:generic,gcp"
       },
       "source": [
-        "# Get started with BigQuery datasets\n",
+        "# E2E ML on GCP: MLOps stage 1 : data management: get started with BigQuery datasets\n",
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
@@ -61,7 +61,7 @@
         "## Overview\n",
         "\n",
         "\n",
-        "This tutorial demonstrates how to use Vertex AI in production. This tutorial covers data management: get started with BigQuery datasets.\n",
+        "This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 1 : data management: get started with BigQuery datasets.\n",
         "\n",
         "Learn more about [BigQuery Datasets](https://cloud.google.com/bigquery/docs/datasets-intro)."
       ]

notebooks/official/datasets/get_started_with_data_labeling.ipynb

@@ -29,7 +29,7 @@
         "id": "title:generic,gcp"
       },
       "source": [
-        "# Get started with Vertex AI Data Labeling\n",
+        "# E2E ML on GCP: MLOps stage 1 : formalization: get started with Vertex AI Data Labeling\n",
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
@@ -62,7 +62,7 @@
         "## Overview\n",
         "\n",
         "\n",
-        "This tutorial demonstrates how to use Vertex AI in production. This tutorial covers data management: get started with Vertex AI Data Labeling service.\n",
+        "This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 1 : data management: get started with Vertex AI Data Labeling service.\n",
         "\n",
         "Learn more about [Vertex AI Data Labeling](https://cloud.google.com/vertex-ai/docs/datasets/data-labeling-job)."
       ]
@@ -352,10 +352,7 @@
       },
       "outputs": [],
       "source": [
-        "EMAIL = \"[your-email-address]\"  # @param {type: \"string\"}\n",
-        "\n",
-        "if os.getenv(\"IS_TESTING\"):\n",
-        "    EMAIL = \"noreply@google.com\""
+        "EMAIL = \"[your-email-address]\"  # @param {type: \"string\"}"
       ]
     },
     {

notebooks/official/experiments/README.md

@@ -55,36 +55,3 @@ The steps performed include:
 
    Learn more about [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction).
 
-
-[Get started with Vertex AI Experiments](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/experiments/get_started_with_vertex_experiments.ipynb)
-
-```
-Learn how to use `Vertex AI Experiments` when training with `Vertex AI`.
-
-The steps performed include:
-
-- Local (notebook) Training
-    - Create an experiment
-    - Create a first run in the experiment
-    - Log parameters and metrics
-    - Create artifact lineage
-    - Visualize the experiment results
-    - Execute a second run
-    - Compare the two runs in the experiment
-- Cloud (`Vertex AI`) Training
-    - Within the training script:
-        - Create an experiment
-        - Log parameters and metrics
-        - Create artifact lineage
-    - Create a `Vertex AI Training` custom job
-    - Execute the custom job
-    - Visualize the experiment results
-
-```
-
-   Learn more about [Vertex AI Experiments](https://cloud.google.com/vertex-ai/docs/experiments/intro-vertex-ai-experiments).
-
-   Learn more about [Vertex ML Metadata](https://cloud.google.com/vertex-ai/docs/ml-metadata).
-
-   Learn more about [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training).
-

notebooks/official/explainable_ai/sdk_automl_tabular_binary_classification_batch_explain.ipynb

@@ -64,7 +64,7 @@
         "\n",
         "This tutorial demonstrates how to use the Vertex AI SDK to create tabular binary classification models and do batch prediction with explanation using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) model.\n",
         "\n",
-        "Learn more about [Classification for tabular data](https://cloud.google.com/vertex-ai/docs/tabular-data/classification-regression/overview). Learn more about [Vertex Explainable AI](https://cloud.google.com/vertex-ai/docs/explainable-ai/overview)."
+        "Learn more about [AutoML Tabular](https://cloud.google.com/vertex-ai/docs/tabular-data/overview) and [Vertex Explainable AI](https://cloud.google.com/vertex-ai/docs/explainable-ai/overview)."
       ]
     },
     {

notebooks/official/explainable_ai/sdk_automl_tabular_classification_online_explain.ipynb

@@ -64,7 +64,7 @@
         "\n",
         "This tutorial demonstrates how to use the Vertex AI SDK to create tabular classification models and do online prediction with explanation using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) model.\n",
         "\n",
-        "Learn more about [Classification for tabular data](https://cloud.google.com/vertex-ai/docs/tabular-data/classification-regression/overview) and [Vertex Explainable AI](https://cloud.google.com/vertex-ai/docs/explainable-ai/overview)."
+        "Learn more about [AutoML Tabular](https://cloud.google.com/vertex-ai/docs/tabular-data/overview) and [Vertex Explainable AI](https://cloud.google.com/vertex-ai/docs/explainable-ai/overview)."
       ]
     },
     {

notebooks/official/matching_engine/sdk_matching_engine_for_indexing.ipynb

@@ -292,7 +292,8 @@
       },
       "outputs": [],
       "source": [
-        "VPC_NETWORK = \"[your-vpc-network-name]\"  # @param {type:\"string\"}\n",
+        "# VPC_NETWORK = \"[your-vpc-network-name]\"  # @param {type:\"string\"}\n",
+        "VPC_NETWORK = \"matching-engine-test\"  # @param {type:\"string\"}\n",
         "\n",
         "PEERING_RANGE_NAME = \"ann-haystack-range\""
       ]
@@ -543,7 +544,7 @@
         "        json.dumps(\n",
         "            {\n",
         "                \"id\": str(index),\n",
-        "                \"embedding\": [str(value) for value in embedding],\n",
+        "                \"embedding\": [str(value) for value in train[index]],\n",
         "                \"restricts\": [\n",
         "                    {\n",
         "                        \"namespace\": \"class\",\n",
@@ -645,7 +646,7 @@
       "outputs": [],
       "source": [
         "tree_ah_index = aiplatform.MatchingEngineIndex.create_tree_ah_index(\n",
-        "    display_name=DISPLAY_NAME,\n",
+        "    display_name=DISPLAY_NAME_BRUTE_FORCE,\n",
         "    contents_delta_uri=EMBEDDINGS_INITIAL_URI,\n",
         "    dimensions=DIMENSIONS,\n",
         "    approximate_neighbors_count=150,\n",
@@ -711,7 +712,7 @@
       "outputs": [],
       "source": [
         "brute_force_index = aiplatform.MatchingEngineIndex.create_brute_force_index(\n",
-        "    display_name=DISPLAY_NAME_BRUTE_FORCE,\n",
+        "    display_name=DISPLAY_NAME,\n",
         "    contents_delta_uri=EMBEDDINGS_INITIAL_URI,\n",
         "    dimensions=DIMENSIONS,\n",
         "    distance_measure_type=\"DOT_PRODUCT_DISTANCE\",\n",

notebooks/official/migration/UJ1 Vertex SDK AutoML Image Classification.ipynb

@@ -64,7 +64,7 @@
         "\n",
         "This tutorial demonstrates how to use the Vertex AI SDK for Python to train and deploy an AutoML image classification model.\n",
         "\n",
-        "Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai) and [Classification for image data](https://cloud.google.com/vertex-ai/docs/training-overview#classification_for_images)."
+        "Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai) and [AutoML Image](https://cloud.google.com/vertex-ai/docs/tutorials/image-recognition-automl/training)."
       ]
     },
     {

notebooks/official/migration/UJ10 Vertex SDK Custom Scikit-Learn with pre-built training container.ipynb

@@ -64,7 +64,7 @@
         "\n",
         "This tutorial demonstrates how to use the Vertex AI SDK for Python to train and deploy a custom tabular classification scikit-learn model for batch prediction.\n",
         "\n",
-        "Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai) and [Custom training](https://cloud.google.com/vertex-ai/docs/training/custom-training)."
+        "Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai) and [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training)."
       ]
     },
     {

notebooks/official/migration/UJ11 Vertex SDK Hyperparameter Tuning.ipynb

@@ -65,7 +65,7 @@
         "\n",
         "This tutorial demonstrates how to use the Vertex AI SDK for Python to hyperparamer tune a custom tabular classification TemsorFlow model.\n",
         "\n",
-        "Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/training/hyperparameter-tuning-overview) and [Custom training](https://cloud.google.com/vertex-ai/docs/training/custom-training)."
+        "Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/training/hyperparameter-tuning-overview) and [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training)."
       ]
     },
     {

notebooks/official/migration/UJ14 Vertex SDK AutoML Video Classification.ipynb

@@ -29,7 +29,7 @@
         "id": "title:migration,new"
       },
       "source": [
-        "# Vertex AI: Vertex AI Migration: AutoML Video Classification\n",
+        "# Vertex AI: Vertex AI Migration: AutoML Video Classificaton\n",
         "\n",
         "<table align=\"left\">\n",
         "\n",
@@ -64,7 +64,7 @@
         "\n",
         "This tutorial demonstrates how to use the Vertex AI SDK for Python to train a AutoML video classification model and do a batch prediction.\n",
         "\n",
-        "Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai) and [Classification for video data](https://cloud.google.com/vertex-ai/docs/training-overview#classification_for_videos)."
+        "Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai) and [AutoML Video](https://cloud.google.com/vertex-ai/docs/tutorials/video-classification-automl/training)."
       ]
     },
     {
@@ -568,7 +568,7 @@
       },
       "outputs": [],
       "source": [
-        "IMPORT_FILE = \"gs://automl-video-demo-data/hmdb_split1_train_40_mp4_gs.csv\""
+        "IMPORT_FILE = \"gs://automl-video-demo-data/hmdb_split1_5classes_train_inf.csv\""
       ]
     },
     {
@@ -604,19 +604,6 @@
         "! gsutil cat $FILE | head"
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "f483d8d80f64"
-      },
-      "outputs": [],
-      "source": [
-        "import datetime\n",
-        "\n",
-        "start = datetime.datetime.now()"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -770,19 +757,7 @@
         "\n",
         "The `run` method when completed returns the `Model` resource.\n",
         "\n",
-        "The execution of the training pipeline may take over 24 hrs."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "8f135100f5d9"
-      },
-      "outputs": [],
-      "source": [
-        "if os.getenv(\"IS_TESTING\"):\n",
-        "    sys.exit(0)"
+        "The execution of the training pipeline will take upto 20 minutes."
       ]
     },
     {
@@ -831,18 +806,6 @@
         "    INFO:google.cloud.aiplatform.training_jobs:Model available at projects/759209241365/locations/us-central1/models/1899701006099283968"
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "48ca7f0980e0"
-      },
-      "outputs": [],
-      "source": [
-        "end = datetime.datetime.now()\n",
-        "end - start"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {

notebooks/official/migration/UJ15 Vertex SDK AutoML Object Tracking.ipynb

@@ -64,7 +64,7 @@
         "\n",
         "This tutorial demonstrates how to use the Vertex AI SDK for Python to train a AutoML video object tracking model and do a batch prediction.\n",
         "\n",
-        "Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai) and [Object tracking for video data](https://cloud.google.com/vertex-ai/docs/training-overview#object_tracking_for_videos)."
+        "Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai) and [AutoML Video](https://cloud.google.com/video-intelligence/automl/object-tracking/docs/index-object-tracking)."
       ]
     },
     {

notebooks/official/migration/UJ2,12 Vertex SDK Custom Image Classification with pre-built training container.ipynb

@@ -64,7 +64,7 @@
         "\n",
         "This tutorial demonstrates how to use the Vertex AI SDK for Python to train using a pre-built container and deploy a custom image classification model for online and batch prediction.\n",
         "\n",
-        "Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai) and [Custom training](https://cloud.google.com/vertex-ai/docs/training/custom-training)."
+        "Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai) and [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training)."
       ]
     },
     {

notebooks/official/migration/UJ3 Vertex SDK Custom Image Classification with custom training container.ipynb

@@ -63,7 +63,7 @@
         "\n",
         "This notebook demonstrates training a custom image classification model using Tensorflow and Vertex AI SDK by creating a custom training container. Additionally, the notebooks also deploys the trained model to Vertex AI and predictions are generated from it.\n",
         "\n",
-        "Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai) and [Custom training](https://cloud.google.com/vertex-ai/docs/training/custom-training)."
+        "Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai) and [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training)."
       ]
     },
     {

notebooks/official/migration/UJ4 Vertex SDK AutoML Tabular Binary Classification.ipynb

@@ -64,7 +64,7 @@
         "\n",
         "This tutorial demonstrates how to use the Vertex AI SDK to create tabular binary classification models and do online prediction using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) model.\n",
         "\n",
-        "Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai). Learn more about [Classification for tabular data](https://cloud.google.com/vertex-ai/docs/tabular-data/classification-regression/overview)."
+        "Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai) and [AutoML Tabular](https://cloud.google.com/vertex-ai/docs/start/automl-users#tables)."
       ]
     },
     {

notebooks/official/migration/UJ5 Vertex SDK AutoML Image Object Detection.ipynb

@@ -65,7 +65,7 @@
         "\n",
         "This tutorial demonstrates how to use the Vertex AI SDK for Python to train and deploy an AutoML object detection model.\n",
         "\n",
-        "Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai) and [Object detection for image data](https://cloud.google.com/vertex-ai/docs/training-overview#object_detection_for_images)."
+        "Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai) and [AutoML Image](https://cloud.google.com/vertex-ai/docs/tutorials/image-recognition-automl/training)."
       ]
     },
     {

notebooks/official/migration/UJ6 Vertex SDK AutoML Text Classification.ipynb

@@ -61,11 +61,11 @@
       "source": [
         "## Overview\n",
         "\n",
-        "This notebook demonstrates how to create an AutoML Text Classification Model, with a Vertex AI text dataset, and how to serve the model for batch prediction. It requires you provide a bucket where the dataset will be stored.\n",
+        "This notebook demonstrates how to create an AutoML Video Classification Model, with a Vertex AI video dataset, and how to serve the model for batch prediction. It requires you provide a bucket where the dataset will be stored.\n",
         "\n",
         "Note: you may incur charges for training, prediction, storage or usage of other GCP products in connection with testing this SDK.\n",
         "\n",
-        "Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai) and [Classification for text data](https://cloud.google.com/vertex-ai/docs/training-overview#classification_for_text)."
+        "Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai) and [AutoML Text](https://cloud.google.com/vertex-ai/docs/text-data/classification/prepare-data)."
       ]
     },
     {
@@ -76,7 +76,7 @@
       "source": [
         "### Objective\n",
         "\n",
-        "The objective of this notebook is to build a AutoML Text Classification Model. The following steps have been followed:\n",
+        "The objective of this notebook is to build a AutoML Video Classification Model. The following steps have been followed:\n",
         "This tutorial uses the following Google Cloud ML services :\n",
         "\n",
         "* Vertex AI Dataset resource\n",
@@ -87,11 +87,11 @@
         "The steps performed include the following:\n",
         "\n",
         "* Set your task name, and GCS prefix\n",
-        "* Copy AutoML text demo train data for creating managed dataset\n",
+        "* Copy AutoML video demo train data for creating managed dataset\n",
         "* Create a dataset on Vertex AI.\n",
         "* Configure a training job\n",
         "* Launch a training job and create a model on Vertex AI\n",
-        "* Copy AutoML Text Demo Prediction Data for creating batch prediction job\n",
+        "* Copy AutoML Video Demo Prediction Data for creating batch prediction job\n",
         "* Perform batch prediction job on the model"
       ]
     },

notebooks/official/migration/UJ7 Vertex SDK AutoML Text Entity Extraction.ipynb

@@ -65,7 +65,7 @@
         "\n",
         "Note: you may incur charges for training, prediction, storage or usage of other Google Cloud products in connection with testing this SDK.\n",
         "\n",
-        "Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai) and [Entity extraction for text data](https://cloud.google.com/vertex-ai/docs/training-overview#entity_extraction_for_text).\n"
+        "Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai) and [AutoML Text](https://cloud.google.com/vertex-ai/docs/text-data/entity-extraction/prepare-data)."
       ]
     },
     {
@@ -76,7 +76,7 @@
       "source": [
         "### Objective\n",
         "\n",
-        "The objective of this notebook is to build a AutoML Text Entity Extraction model. The following steps have been followed:\n",
+        "The objective of this notebook is to build a AutoML Text Entity Extraction Model. The following steps have been followed:\n",
         "This tutorial uses the following Google Cloud ML services :\n",
         "\n",
         "* Vertex AI Dataset resource\n",

notebooks/official/migration/UJ8 Vertex SDK AutoML Text Sentiment Analysis.ipynb

@@ -66,7 +66,7 @@
         "\n",
         "Note: you may incur charges for training, prediction, storage or usage of other Google Cloud products in connection with testing this SDK.\n",
         "\n",
-        "Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai) and [Sentiment analysis for text data](https://cloud.google.com/vertex-ai/docs/training-overview#sentiment_analysis_for_text)."
+        "Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai) and [AutoML Text](https://cloud.google.com/vertex-ai/docs/text-data/sentiment-analysis/prepare-data)."
       ]
     },
     {

notebooks/official/migration/UJ9 Vertex SDK Custom XGBoost with pre-built training container.ipynb

@@ -64,7 +64,7 @@
         "\n",
         "This tutorial demonstrates how to use the Vertex AI SDK for Python to train and deploy a custom tabular classification XGBoost model for batch prediction.\n",
         "\n",
-        "Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai) and [Custom training](https://cloud.google.com/vertex-ai/docs/training/custom-training)."
+        "Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai) and [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training)."
       ]
     },
     {

notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-custom-jobs.ipynb

@@ -63,7 +63,7 @@
         "\n",
         "This notebook demonstrates how to track metrics and parameters for Vertex AI custom training jobs, and how to perform detailed analysis using this data.\n",
         "\n",
-        "Learn more about [Vertex ML Metadata](https://cloud.google.com/vertex-ai/docs/ml-metadata) and [Custom training](https://cloud.google.com/vertex-ai/docs/training/custom-training)."
+        "Learn more about [Vertex ML Metadata](https://cloud.google.com/vertex-ai/docs/ml-metadata) and [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training)."
       ]
     },
     {

notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-locally-trained-models.ipynb

@@ -140,7 +140,7 @@
       "outputs": [],
       "source": [
         "! pip install --upgrade -q google-cloud-aiplatform \\\n",
-        "                            tensorflow -q"
+        "                            tensorflow==2.8 -q"
       ]
     },
     {

notebooks/official/model_evaluation/automl_tabular_classification_model_evaluation.ipynb

@@ -63,7 +63,7 @@
         "\n",
         "This notebook demonstrates how to use the Vertex AI classification model evaluation component to evaluate an AutoML Tabular classification model. Model evaluation helps determine your model's performance based on the evaluation metrics and improve the model whenever necessary. \n",
         "\n",
-        "Learn more about [Vertex AI Model Evaluation](https://cloud.google.com/vertex-ai/docs/evaluation/introduction). Learn more about [Classification for tabular data](https://cloud.google.com/vertex-ai/docs/tabular-data/classification-regression/overview)."
+        "Learn more about [Vertex AI Model Evaluation](https://cloud.google.com/vertex-ai/docs/evaluation/introduction) and [AutoML Tabular](https://cloud.google.com/vertex-ai/docs/start/automl-users#tables)."
       ]
     },
     {

notebooks/official/model_evaluation/automl_tabular_regression_model_evaluation.ipynb

@@ -63,7 +63,7 @@
         "\n",
         "This notebook demonstrates how to use Vertex AI regression model evaluation component to evaluate an AutoML Tabular regression model. Model evaluation helps you determine your model performance based on the evaluation metrics and improve the model if necessary. \n",
         "\n",
-        "Learn more about [Vertex AI Model Evaluation](https://cloud.google.com/vertex-ai/docs/evaluation/introduction). Learn more about [Regression for tabular data](https://cloud.google.com/vertex-ai/docs/tabular-data/classification-regression/overview)."
+        "Learn more about [Vertex AI Model Evaluation](https://cloud.google.com/vertex-ai/docs/evaluation/introduction) and [AutoML Tabular](https://cloud.google.com/vertex-ai/docs/start/automl-users#tables)."
       ]
     },
     {
@@ -92,7 +92,7 @@
         "- Run the `AutoMLTabularTrainingJob` which returns a model\n",
         "- Import a pre-trained `AutoML model resource` into the pipeline\n",
         "- Run a `batch prediction` job in the pipeline\n",
-        "- Evaluate the AutoML model using the `regression evaluation component`\n",
+        "- Evaulate the AutoML model using the `regression evaluation component`\n",
         "- Import the Regression Metrics to the AutoML model resource"
       ]
     },

notebooks/official/model_evaluation/automl_video_classification_model_evaluation.ipynb

@@ -63,7 +63,7 @@
         "\n",
         "This notebook demonstrates how to use the Vertex AI classification model evaluation component to evaluate an AutoML video classification model. Model evaluation helps you determine your model performance based on the evaluation metrics and improve the model if necessary. \n",
         "\n",
-        "Learn more about [Vertex AI Model Evaluation](https://cloud.google.com/vertex-ai/docs/evaluation/introduction) and [Classification for video data](https://cloud.google.com/vertex-ai/docs/training-overview#classification_for_videos)."
+        "Learn more about [Vertex AI Model Evaluation](https://cloud.google.com/vertex-ai/docs/evaluation/introduction) and [AutoML Video](https://cloud.google.com/vertex-ai/docs/video-data/classification/prepare-data)."
       ]
     },
     {
@@ -92,7 +92,7 @@
         "- Train a Automl Video Classification model on the `Vertex AI Dataset` resource.\n",
         "- Import the trained `AutoML Vertex AI Model resource` into the pipeline.\n",
         "- Run a batch prediction job inside the pipeline.\n",
-        "- Evaluate the AutoML model using the classification evaluation component.\n",
+        "- Evaulate the AutoML model using the classification evaluation component.\n",
         "- Import the classification metrics to the AutoML Vertex AI Model resource."
       ]
     },
@@ -792,19 +792,7 @@
         "\n",
         "The `run` method when completed returns the `Model` resource.\n",
         "\n",
-        "The execution of the training pipeline can take over 24 hours to complete."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "3eaba926cdfa"
-      },
-      "outputs": [],
-      "source": [
-        "if os.getenv(\"IS_TESTING\"):\n",
-        "    sys.exit(0)"
+        "The execution of the training pipeline can take over 3 hours to complete."
       ]
     },
     {

notebooks/official/model_evaluation/custom_tabular_regression_model_evaluation.ipynb

@@ -63,7 +63,7 @@
         "\n",
         "This notebook demonstrates how to use the Vertex AI regression model evaluation component to evaluate a custom regression model. Model evaluation helps you determine your model performance based on the evaluation metrics and improve the model if necessary. \n",
         "\n",
-        "Learn more about [Vertex AI Model Evaluation](https://cloud.google.com/vertex-ai/docs/evaluation/introduction) and [Custom training](https://cloud.google.com/vertex-ai/docs/training/custom-training)."
+        "Learn more about [Vertex AI Model Evaluation](https://cloud.google.com/vertex-ai/docs/evaluation/introduction) and [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training)."
       ]
     },
     {
@@ -93,7 +93,7 @@
         "- Upload the model as a Vertex AI Model resource.\n",
         "- Import a pre-trained `Vertex AI model resource` into the pipeline.\n",
         "- Run a `batch prediction` job in the pipeline.\n",
-        "- Evaluate the model using the `regression evaluation component`.\n",
+        "- Evaulate the model using the `regression evaluation component`.\n",
         "- Import the Regression Metrics to the Vertex AI model resource."
       ]
     },

notebooks/official/model_evaluation/get_started_with_custom_model_evaluation_import.ipynb

@@ -1,865 +0,0 @@
-{
-  "cells": [
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "ur8xi4C7S06n"
-      },
-      "outputs": [],
-      "source": [
-        "# Copyright 2023 Google LLC\n",
-        "#\n",
-        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
-        "# you may not use this file except in compliance with the License.\n",
-        "# You may obtain a copy of the License at\n",
-        "#\n",
-        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
-        "#\n",
-        "# Unless required by applicable law or agreed to in writing, software\n",
-        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
-        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
-        "# See the License for the specific language governing permissions and\n",
-        "# limitations under the License."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "JAPoU8Sm5E6e"
-      },
-      "source": [
-        "# Get started with importing a custom model evaluation to the Vertex AI Model Registry\n",
-        "\n",
-        "<table align=\"left\">\n",
-        "\n",
-        "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model_evaluation/get_started_with_custom_model_evaluation_import.ipynb\">\n",
-        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
-        "    </a>\n",
-        "  </td>\n",
-        "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model_evaluation/get_started_with_custom_model_evaluation_import.ipynb\">\n",
-        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
-        "      View on GitHub\n",
-        "    </a>\n",
-        "  </td>\n",
-        "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/model_evaluation/get_started_with_custom_model_evaluation_import.ipynb\">\n",
-        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
-        "      Open in Vertex AI Workbench\n",
-        "    </a>\n",
-        "  </td>                                                                                               \n",
-        "</table>"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "24743cf4a1e1"
-      },
-      "source": [
-        "**_NOTE_**: This notebook has been tested in the following environment:\n",
-        "\n",
-        "* Python version = 3.9"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "tvgnzT1CKxrO"
-      },
-      "source": [
-        "## Overview\n",
-        "\n",
-        "This tutorial shows how to use Vertex AI Model Evaluation to import a custom model evaluation to an existing Vertex AI Model Registry entry."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "d975e698c9a4"
-      },
-      "source": [
-        "### Objective\n",
-        "\n",
-        "In this tutorial, you learn how to construct and upload a custom model evaluation, and upload the custom model evaluation to a Model resource entry in Vertex AI Model Registry.\n",
-        "\n",
-        "This tutorial uses the following Google Cloud ML services and resources:\n",
-        "\n",
-        "- Vertex AI Model Evaluation\n",
-        "- Vertex AI Model Registry\n",
-        "\n",
-        "The steps performed include:\n",
-        "\n",
-        "- Import a pretrained (blessed) model to the Vertex AI Model Registry.\n",
-        "- Construct a custom model evaluation.\n",
-        "- Import the model evaluation metrics to the corresponding model in the Vertex AI Model Registry.\n",
-        "- List the model evaluation for the corresponding model in the Vertex AI Model Registry.\n",
-        "- Construct a second custom model evaluation.\n",
-        "- Import the second model evaluation metrics to the corresponding model in the Vertex AI Model Registry.\n",
-        "- List the second model evaluation for the corresponding model in the Vertex AI Model Registry.\n",
-        "\n",
-        "Learn more about [Model Evaluation in Vertex AI](https://cloud.google.com/vertex-ai/docs/evaluation/introduction)."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "08d289fa873f"
-      },
-      "source": [
-        "### Model\n",
-        "\n",
-        "This tutorial uses a pre-trained image classification model from TensorFlow Hub, which is trained on ImageNet dataset.\n",
-        "\n",
-        "Learn more about [ResNet V2 pretained model](https://tfhub.dev/google/imagenet/resnet_v2_101/classification/5). "
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "aed92deeb4a0"
-      },
-      "source": [
-        "### Costs \n",
-        "\n",
-        "This tutorial uses billable components of Google Cloud:\n",
-        "\n",
-        "* Vertex AI\n",
-        "* Cloud Storage\n",
-        "\n",
-        "Learn about [Vertex AI pricing](https://cloud.google.com/vertex-ai/pricing),\n",
-        "and [Cloud Storage pricing](https://cloud.google.com/storage/pricing), \n",
-        "and use the [Pricing Calculator](https://cloud.google.com/products/calculator/)\n",
-        "to generate a cost estimate based on your projected usage."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "i7EUnXsZhAGF"
-      },
-      "source": [
-        "## Installation\n",
-        "\n",
-        "Install the following packages required to execute this notebook. "
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "2b4ef9b72d43"
-      },
-      "outputs": [],
-      "source": [
-        "# Install the packages\n",
-        "USER=''\n",
-        "! pip3 install {USER} --upgrade google-cloud-aiplatform \\\n",
-        "                                tensorflow==2.5 \\\n",
-        "                                tensorflow-hub"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "58707a750154"
-      },
-      "source": [
-        "### Colab only: Uncomment the following cell to restart the kernel."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "f200f10a1da3"
-      },
-      "outputs": [],
-      "source": [
-        "# Automatically restart kernel after installs so that your environment can access the new packages\n",
-        "# import IPython\n",
-        "\n",
-        "# app = IPython.Application.instance()\n",
-        "# app.kernel.do_shutdown(True)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "BF1j6f9HApxa"
-      },
-      "source": [
-        "## Before you begin\n",
-        "\n",
-        "### Set up your Google Cloud project\n",
-        "\n",
-        "**The following steps are required, regardless of your notebook environment.**\n",
-        "\n",
-        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
-        "\n",
-        "2. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
-        "\n",
-        "3. [Enable the Vertex AI API]\n",
-        "\n",
-        "4. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk)."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "WReHDGG5g0XY"
-      },
-      "source": [
-        "#### Set your project ID\n",
-        "\n",
-        "**If you don't know your project ID**, try the following:\n",
-        "* Run `gcloud config list`.\n",
-        "* Run `gcloud projects list`.\n",
-        "* See the support page: [Locate the project ID](https://support.google.com/googleapi/answer/7014113)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "oM1iC_MfAts1"
-      },
-      "outputs": [],
-      "source": [
-        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}\n",
-        "\n",
-        "# Set the project id\n",
-        "! gcloud config set project {PROJECT_ID}"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "region"
-      },
-      "source": [
-        "#### Region\n",
-        "\n",
-        "You can also change the `REGION` variable used by Vertex AI. Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "region"
-      },
-      "outputs": [],
-      "source": [
-        "REGION = \"us-central1\"  # @param {type: \"string\"}"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "sBCra4QMA2wR"
-      },
-      "source": [
-        "### Authenticate your Google Cloud account\n",
-        "\n",
-        "Depending on your Jupyter environment, you may have to manually authenticate. Follow the relevant instructions below."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "74ccc9e52986"
-      },
-      "source": [
-        "**1. Vertex AI Workbench**\n",
-        "* Do nothing as you are already authenticated."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "de775a3773ba"
-      },
-      "source": [
-        "**2. Local JupyterLab instance, uncomment and run:**"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "254614fa0c46"
-      },
-      "outputs": [],
-      "source": [
-        "# ! gcloud auth login"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "ef21552ccea8"
-      },
-      "source": [
-        "**3. Colab, uncomment and run:**"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "603adbbf0532"
-      },
-      "outputs": [],
-      "source": [
-        "# from google.colab import auth\n",
-        "# auth.authenticate_user()"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "f6b2ccc891ed"
-      },
-      "source": [
-        "**4. Service account or other**\n",
-        "* See how to grant Cloud Storage permissions to your service account at https://cloud.google.com/storage/docs/gsutil/commands/iam#ch-examples."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "zgPO1eR3CYjk"
-      },
-      "source": [
-        "### Create a Cloud Storage bucket\n",
-        "\n",
-        "Create a storage bucket to store intermediate artifacts such as datasets.\n",
-        "\n",
-        "- *{Note to notebook author: For any user-provided strings that need to be unique (like bucket names or model ID's), append \"-unique\" to the end so proper testing can occur}*"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "MzGDU7TWdts_"
-      },
-      "outputs": [],
-      "source": [
-        "BUCKET_URI = f\"gs://your-bucket-name-unique-{PROJECT_ID}\"  # @param {type:\"string\"}"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "-EcIXiGsCePi"
-      },
-      "source": [
-        "**Only if your bucket doesn't already exist**: Run the following cell to create your Cloud Storage bucket."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "NIq7R4HZCfIc"
-      },
-      "outputs": [],
-      "source": [
-        "! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "960505627ddf"
-      },
-      "source": [
-        "### Import libraries"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "PyQmSRbKA8r-"
-      },
-      "outputs": [],
-      "source": [
-        "import tensorflow as tf\n",
-        "import tensorflow_hub as hub\n",
-        "from google.cloud import aiplatform\n",
-        "from google.cloud.aiplatform import gapic"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "init_aip:mbsdk,all"
-      },
-      "source": [
-        "### Initialize Vertex AI SDK for Python\n",
-        "\n",
-        "Initialize the Vertex AI SDK for Python for your project."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "init_aip:mbsdk,all"
-      },
-      "outputs": [],
-      "source": [
-        "aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=BUCKET_URI)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "accelerators:training,cpu,prediction,cpu,mbsdk"
-      },
-      "source": [
-        "#### Set hardware accelerators\n",
-        "\n",
-        "You can set hardware accelerators for prediction.\n",
-        "\n",
-        "Set the variables `DEPLOY_GPU/DEPLOY_NGPU` to use a container image supporting a GPU and the number of GPUs allocated to the virtual machine (VM) instance. For example, to use a GPU container image with 4 Nvidia Telsa K80 GPUs allocated to each VM, you would specify:\n",
-        "\n",
-        "    (aip.gapic.AcceleratorType.NVIDIA_TESLA_K80, 4)\n",
-        "\n",
-        "\n",
-        "Otherwise specify `(None, None)` to use a container image to run on a CPU.\n",
-        "\n",
-        "Learn more about [hardware accelerator support for your region](https://cloud.google.com/vertex-ai/docs/general/locations#accelerators), and [GPU pricing](https://cloud.google.com/compute/gpus-pricing)."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "accelerators:training,cpu,prediction,cpu,mbsdk"
-      },
-      "outputs": [],
-      "source": [
-        "DEPLOY_GPU, DEPLOY_NGPU = (None, None)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "container:training,prediction"
-      },
-      "source": [
-        "#### Set pre-built containers\n",
-        "\n",
-        "Set the pre-built Docker container image for prediction.\n",
-        "\n",
-        "For the latest list, see [Pre-built containers for prediction](https://cloud.google.com/ai-platform-unified/docs/predictions/pre-built-containers)."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "container:training,prediction"
-      },
-      "outputs": [],
-      "source": [
-        "TF = \"2.5\".replace(\".\", \"-\")\n",
-        "\n",
-        "if DEPLOY_GPU:\n",
-        "    DEPLOY_VERSION = \"tf2-gpu.{}\".format(TF)\n",
-        "else:\n",
-        "    DEPLOY_VERSION = \"tf2-cpu.{}\".format(TF)\n",
-        "\n",
-        "DEPLOY_IMAGE = \"{}-docker.pkg.dev/vertex-ai/prediction/{}:latest\".format(\n",
-        "    REGION.split(\"-\")[0], DEPLOY_VERSION\n",
-        ")\n",
-        "\n",
-        "print(\"Deployment:\", DEPLOY_IMAGE, DEPLOY_GPU, DEPLOY_NGPU)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "d8128b8ff025"
-      },
-      "source": [
-        "## Get pretrained model from TensorFlow Hub\n",
-        "\n",
-        "For demonstration purposes, this tutorial uses a pretrained model from TensorFlow Hub (TFHub), which is then uploaded to a `Vertex AI Model` resource. Once you have a `Vertex AI Model` resource, the model can be deployed to a `Vertex AI Endpoint` resource.\n",
-        "\n",
-        "### Download the pretrained model\n",
-        "\n",
-        "First, you download the pretrained model from TensorFlow Hub. The model gets downloaded as a TF.Keras layer. To finalize the model, in this example, you create a `Sequential()` model with the downloaded TFHub model as a layer, and specify the input shape to the model."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "c55fa4c826f7"
-      },
-      "outputs": [],
-      "source": [
-        "tfhub_model = tf.keras.Sequential(\n",
-        "    [hub.KerasLayer(\"https://tfhub.dev/google/imagenet/resnet_v2_101/classification/5\")]\n",
-        ")\n",
-        "\n",
-        "tfhub_model.build([None, 32, 32, 3])\n",
-        "\n",
-        "tfhub_model.summary()"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "64618c713db9"
-      },
-      "outputs": [],
-      "source": [
-        "MODEL_DIR = BUCKET_URI + \"/model\"\n",
-        "tfhub_model.save(MODEL_DIR)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "e8ce91147c93"
-      },
-      "source": [
-        "### Upload the TensorFlow Hub model to a `Vertex AI Model` resource\n",
-        "\n",
-        "Finally, you upload the model artifacts from the TFHub model into a `Vertex AI Model` resource using the method `upload()`, with the following parameters:\n",
-        "\n",
-        "- `display_name`: A human readable name for the `Model` resource.\n",
-        "- `artifact_uri`: The Cloud Storage location of the model package.\n",
-        "- `serving_container_image_uri`: The serving container image.\n",
-        "\n",
-        "Uploading a model into a Vertex AI Model resource returns a long running operation, since it may take a few moments. \n",
-        "\n",
-        "*Note:* When you upload the model artifacts to a `Vertex AI Model` resource, you specify the corresponding deployment container image."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "ad61e1429512"
-      },
-      "outputs": [],
-      "source": [
-        "model = aiplatform.Model.upload(\n",
-        "    display_name=\"resnet\",\n",
-        "    artifact_uri=MODEL_DIR,\n",
-        "    serving_container_image_uri=DEPLOY_IMAGE,\n",
-        "    is_default_version=True,\n",
-        "    version_aliases=[\"v1\"],\n",
-        ")\n",
-        "\n",
-        "print(model)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "c11e98ef5391"
-      },
-      "source": [
-        "## Introduction to custom model evaluations\n",
-        "\n",
-        "When training a custom model, one generally performs some form of an evaluation of the trained model. Your custom model evaluation can then be imported to the corresponding model in the Vertex AI Model Registry using the `import_model_evaluation()` method. Once imported, the custom model evaluation can be subsequently retreived with the `list_model_evaluations()` method. \n",
-        "\n",
-        "The Vertex AI Model Registry supports importing multiple model evaluations for a model where each evaluation is distinquished by a unique `display_name`.\n",
-        "\n",
-        "\n",
-        "### Create a model evaluation\n",
-        "\n",
-        "First, you create a model evaluation in a format that corresponds to one of the predefined schemas for model evaluations. In this example, you use the schema for a classification metric, and specify the following subset of evaluation metrics as a dictionary:\n",
-        "\n",
-        "- `logLoss`: The log loss.\n",
-        "- `auPrc`: The accuracy.\n",
-        "\n",
-        "You then construct the `ModelEvaluation` object with the following parameters:\n",
-        "\n",
-        "- `display_name`: The human readable name for the evaluation metric.\n",
-        "- `metrics_schema_uri`: The schema for the specific type of evaluation metrics.\n",
-        "- `metrics`: The dictionary with the evaluation metrics.\n",
-        "\n",
-        "Learn more about [Schemas for evaluation metrics](https://cloud.google.com/vertex-ai/docs/evaluation/introduction#features)"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "e9af222db292"
-      },
-      "outputs": [],
-      "source": [
-        "metrics = {\"logLoss\": 1.4, \"auPrc\": 0.85}\n",
-        "print(metrics)\n",
-        "\n",
-        "model_eval = gapic.ModelEvaluation(\n",
-        "    display_name=\"eval\",\n",
-        "    metrics_schema_uri=\"gs://google-cloud-aiplatform/schema/modelevaluation/classification_metrics_1.0.0.yaml\",\n",
-        "    metrics=metrics,\n",
-        ")"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "68870bd8194d"
-      },
-      "source": [
-        "### Upload the evaluation metrics to the Model Registry\n",
-        "\n",
-        "Next, upload the model's evaluation from the custom training job to the corresponding entry in the Vertex AI Model Registry.\n",
-        "\n",
-        "Currently, there is not yet support for this method in the SDK. Instead, you use the lower level GAPIC API interface."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "3044788848af"
-      },
-      "outputs": [],
-      "source": [
-        "API_ENDPOINT = f\"{REGION}-aiplatform.googleapis.com\"\n",
-        "client = gapic.ModelServiceClient(client_options={\"api_endpoint\": API_ENDPOINT})\n",
-        "\n",
-        "client.import_model_evaluation(parent=model.resource_name, model_evaluation=model_eval)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "337319a19efe"
-      },
-      "source": [
-        "### List the custom model evaluation\n",
-        "\n",
-        "Now that your custom metric has been uploaded to the corresponding model in the Vertex AI Model Registry, you can retriv"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "1b3374e988cf"
-      },
-      "outputs": [],
-      "source": [
-        "evaluation = model.list_model_evaluations()[0]\n",
-        "print(evaluation.gca_resource)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "68870bd8194d"
-      },
-      "source": [
-        "### Upload a second evaluation metrics to the Model Registry\n",
-        "\n",
-        "Next, upload a second model evaluation to the corresponding entry in the Vertex AI Model Registry. In this example, we refer to first evaluation metric as `eval` (from training) and the second as `prod` (from production data)."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "9b864c0ba07f"
-      },
-      "outputs": [],
-      "source": [
-        "metrics = {\"logLoss\": 1.2, \"auPrc\": 0.87}\n",
-        "print(metrics)\n",
-        "\n",
-        "model_prod = gapic.ModelEvaluation(\n",
-        "    display_name=\"prod\",\n",
-        "    metrics_schema_uri=\"gs://google-cloud-aiplatform/schema/modelevaluation/classification_metrics_1.0.0.yaml\",\n",
-        "    metrics=metrics,\n",
-        ")\n",
-        "\n",
-        "client.import_model_evaluation(parent=model.resource_name, model_evaluation=model_prod)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "337319a19efe"
-      },
-      "source": [
-        "### List a specific custom model evaluation\n",
-        "\n",
-        "Now that your custom second metric has been uploaded to the corresponding model in the Vertex AI Model Registry, you can retrieve this specific evaluation by filtering using the `display_name`:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "1b3374e988cf"
-      },
-      "outputs": [],
-      "source": [
-        "evaluations = model.list_model_evaluations()\n",
-        "for evaluation in evaluations:\n",
-        "    if evaluation.display_name == \"prod\":\n",
-        "        print(evaluation.gca_resource)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "b3a3993dc813"
-      },
-      "source": [
-        "### Upload version 2 of the TFHub model to the `Vertex AI Model Registry`\n",
-        "\n",
-        "Next, you upload the second version of the TFHub model as a `Model` resource in the `Vertex AI Model Registry`, with the additional following parameters:\n",
-        "\n",
-        "- `parent_model`: The existing `Model` resource for which to add this model as the next model version.\n",
-        "- `is_default_version`: Whether this will be the default version for the `Model` resource. In this example, you change from the default from the first version to the second version of the model.\n",
-        "- `version_ailiases`: User defined list of alternative alias names for the model version, such as `production`.\n",
-        "- `version_description`: User description of the model version.\n",
-        "\n",
-        "When a subsequent model version is created in the `Vertex AI Model Registry`, the property `version_id` will automatically be incremented. In this example, it will be set to 2 (2nd version)."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "852be5e5f964"
-      },
-      "outputs": [],
-      "source": [
-        "model_v2 = aiplatform.Model.upload(\n",
-        "    display_name=\"resnet\",\n",
-        "    artifact_uri=MODEL_DIR,\n",
-        "    serving_container_image_uri=DEPLOY_IMAGE,\n",
-        "    parent_model=model.resource_name,\n",
-        "    is_default_version=True,\n",
-        "    version_aliases=[\"v2\"],\n",
-        "    version_description=\"This is the second version of the model\",\n",
-        ")\n",
-        "\n",
-        "print(model_v2)"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "68870bd8194d"
-      },
-      "source": [
-        "### Upload an evaluation metrics for version 2 of the model to the Model Registry\n",
-        "\n",
-        "Next, upload a model evaluation to the corresponding model version in the Vertex AI Model Registry. Note, you referenced `model_v2.resource_name` to refer to version 2 of this model."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "9b864c0ba07f"
-      },
-      "outputs": [],
-      "source": [
-        "metrics = {\"logLoss\": 1.0, \"auPrc\": 0.91}\n",
-        "print(metrics)\n",
-        "\n",
-        "model_eval = gapic.ModelEvaluation(\n",
-        "    display_name=\"eval\",\n",
-        "    metrics_schema_uri=\"gs://google-cloud-aiplatform/schema/modelevaluation/classification_metrics_1.0.0.yaml\",\n",
-        "    metrics=metrics,\n",
-        ")\n",
-        "\n",
-        "client.import_model_evaluation(\n",
-        "    parent=model_v2.resource_name, model_evaluation=model_eval\n",
-        ")"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "d4e72b21b06b"
-      },
-      "source": [
-        "### List the evaluations for both versions of the model\n",
-        "\n",
-        "Finally, list the number of evaluations for both versions of the model."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "369a23d392a4"
-      },
-      "outputs": [],
-      "source": [
-        "evaluations = model.list_model_evaluations()\n",
-        "print(\"Model v1 no. of evaluations\", len(evaluations))\n",
-        "evaluations = model_v2.list_model_evaluations()\n",
-        "print(\"Model v2 no. of evaluations\", len(evaluations))"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "TpV-iwP9qw9c"
-      },
-      "source": [
-        "## Cleaning up\n",
-        "\n",
-        "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
-        "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
-        "\n",
-        "Otherwise, you can delete the individual resources you created in this tutorial:\n"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "sx_vKniMq9ZX"
-      },
-      "outputs": [],
-      "source": [
-        "import os\n",
-        "\n",
-        "# Delete model resource\n",
-        "model.delete()\n",
-        "\n",
-        "# Delete Cloud Storage objects that were created\n",
-        "delete_bucket = False\n",
-        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
-        "    ! gsutil -m rm -r $BUCKET_URI"
-      ]
-    }
-  ],
-  "metadata": {
-    "colab": {
-      "collapsed_sections": [],
-      "name": "get_started_with_custom_model_evaluation_import.ipynb",
-      "toc_visible": true
-    },
-    "kernelspec": {
-      "display_name": "Python 3",
-      "name": "python3"
-    }
-  },
-  "nbformat": 4,
-  "nbformat_minor": 0
-}

notebooks/official/model_monitoring/README.md

@@ -15,128 +15,6 @@ The steps performed include:
    Learn more about [Vertex AI Model Monitoring for batch predictions](https://cloud.google.com/vertex-ai/docs/model-monitoring/model-monitoring-batch-predictions).
 
 
-[Vertex AI Model Monitoring for AutoML tabular models](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model_monitoring/get_started_with_model_monitoring_automl.ipynb)
-
-```
-Learn to use the `Vertex AI Model Monitoring` service to detect feature skew and drift in the input predict requests, for AutoML tabular models.
-
-The steps performed include:
-
-- Train an `AutoML` model.
-- Deploy the `Model` resource to the `Endpoint` resource.
-- Configure the `Endpoint` resource for model monitoring.
-- Generate synthetic prediction requests for skew.
-- Wait for email alert notification.
-- Generate synthetic prediction requests for drift.
-- Wait for email alert notification.
-
-```
-
-   Learn more about [Vertex AI Model Monitoring](https://cloud.google.com/vertex-ai/docs/model-monitoring).
-
-
-[Vertex AI Model Monitoring for batch prediction in AutoML image models](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model_monitoring/get_started_with_model_monitoring_automl_image_batch.ipynb)
-
-```
-Learn how to use `Vertex AI Model Monitoring` with `Vertex AI Batch Prediction` with an AutoML image classification model to detect an out of distribution image.
-
-The steps performed include:
-
-1. Train an AutoML image classification model.
-2. Submit a batch prediction containing both in and out of distribution images.
-3. Use Model Monitoring to calculate anomaly score on each image.
-4. Identify the images in the batch prediction request that are out of distribution.
-
-```
-
-   Learn more about [Vertex AI Model Monitoring](https://cloud.google.com/vertex-ai/docs/model-monitoring).
-
-
-[Vertex AI Model Monitoring for custom tabular models](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model_monitoring/get_started_with_model_monitoring_custom.ipynb)
-
-```
-Learn to use the `Vertex AI Model Monitoring` service to detect feature skew and drift in the input predict requests, for custom tabular models.
-
-The steps performed include:
-
-- Download a pre-trained custom tabular model.
-- Upload the pre-trained model as a `Model` resource.
-- Deploy the `Model` resource to the `Endpoint` resource.
-- Configure the `Endpoint` resource for model monitoring.
-- Generate synthetic prediction requests for skew.
-- Wait for email alert notification.
-- Generate synthetic prediction requests for drift.
-- Wait for email alert notification.
-
-```
-
-   Learn more about [Vertex AI Model Monitoring](https://cloud.google.com/vertex-ai/docs/model-monitoring).
-
-
-[Vertex AI Model Monitoring for custom tabular models with TensorFlow Serving container](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model_monitoring/get_started_with_model_monitoring_custom_tf_serving.ipynb)
-
-```
-Learn to use the `Vertex AI Model Monitoring` service to detect feature skew and drift in the input predict requests, for custom tabular models, using a custom deployment container.
-
-The steps performed include:
-
-- Download a pre-trained custom tabular model.
-- Upload the pre-trained model as a `Model` resource.
-- Deploying the `Model` resource to an `Endpoint` resource with `TensorFlow Serving` serving binary.
-- Configure the `Endpoint` resource for model monitoring.
-- Generate synthetic prediction requests for skew.
-- Wait for email alert notification.
-- Generate synthetic prediction requests for drift.
-- Wait for email alert notification.
-
-```
-
-   Learn more about [Vertex AI Model Monitoring](https://cloud.google.com/vertex-ai/docs/model-monitoring).
-
-
-[Vertex AI Model Monitoring for setup for tabular models](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model_monitoring/get_started_with_model_monitoring_setup.ipynb)
-
-```
-Learn to setup the `Vertex AI Model Monitoring` service to detect feature skew and drift in the input predict requests.
-
-The steps performed include:
-
-- Download a pre-trained custom tabular model.
-- Upload the pre-trained model as a `Model` resource.
-- Deploy the `Model` resource to the `Endpoint` resource.
-- Configure the `Endpoint` resource for model monitoring.
-    - Skew and drift detection for feature inputs.
-    - Skew and drift detection for feature attributions.
-- Automatic generation of the `input schema` by sending 1000 prediction request.
-- List, pause, resume and delete monitoring jobs.
-- Restart monitoring job with predefined `input schema`.
-- View logged monitored data.
-
-```
-
-   Learn more about [Vertex AI Model Monitoring](https://cloud.google.com/vertex-ai/docs/model-monitoring).
-
-
-[Vertex AI Model Monitoring for XGBoost models](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model_monitoring/get_started_with_model_monitoring_xgboost.ipynb)
-
-```
-Learn to use the `Vertex AI Model Monitoring` service to detect feature skew and drift in the input predict requests for XGBoost models.
-
-The steps performed include:
-
-- Download a pre-trained XGBoost model.
-- Upload the pre-trained model as a `Model` resource.
-- Deploy the `Model` resource to the `Endpoint` resource.
-- Configure the `Endpoint` resource for model monitoring:
-  - drift detection only -- no access to training data.
-  - predefine the input schema to map feature alias names to the unnamed array input to the model.
-- Generate synthetic prediction requests for drift.
-
-```
-
-   Learn more about [Vertex AI Model Monitoring](https://cloud.google.com/vertex-ai/docs/model-monitoring).
-
-
 [Vertex AI Model Monitoring with Explainable AI Feature Attributions](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model_monitoring/model_monitoring.ipynb)
 
 ```

notebooks/official/model_registry/README.md

@@ -17,5 +17,5 @@ The steps performed include:
 
    Learn more about [Vertex AI Model Registry](https://cloud.google.com/vertex-ai/docs/model-registry/introduction).
 
-   Learn more about [BigQuery ML](https://cloud.google.com/vertex-ai/docs/beginner/bqml).
+   Learn more about [BigQuery ML](https://cloud.google.com/bigquery-ml/docs/introduction).
 

notebooks/official/pipelines/README.md

@@ -285,48 +285,3 @@ The steps performed include:
 
    Learn more about [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction).
 
-
-[BQML and AutoML - Experimenting with Vertex AI](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/rapid_prototyping_bqml_automl.ipynb)
-
-```
-Learn how to use `Vertex AI Predictions` for rapid prototyping a model.
-
-The steps performed include:
-
-- Creating a BigQuery and Vertex AI training dataset.
-- Training a BigQuery ML and AutoML model.
-- Extracting evaluation metrics from the BigQueryML and AutoML models.
-- Selecting the best trained model.
-- Deploying the best trained model.
-- Testing the deployed model infrastructure.
-
-```
-
-   Learn more about [AutoML components](https://cloud.google.com/vertex-ai/docs/pipelines/vertex-automl-component).
-
-[Train custom tabular ML models with many frameworks and import to Vertex AI using Vertex Pipelines](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/notebooks/official/pipelines/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines)
-
-Learn how to build a pipeline that does the following:
-
-*   Ingest data
-*   Transform data
-*   Clean up data
-*   Split data into train/test subsets
-*   Configure model
-*   Train model using multiple ML frameworks
-*   Import model into Vertex Model Registry
-*   [Optional] Deploy model to Vertex Endpoints for serving
-
-Included pipelines:
-
-*   Train ML model
-*   *   Tabular classification
-*   *   *   TensorFlow
-*   *   *   PyTorch
-*   *   *   XGBoost
-*   *   *   Scikit-learn
-*   *   Tabular regression
-*   *   *   TensorFlow
-*   *   *   PyTorch
-*   *   *   XGBoost
-*   *   *   Scikit-learn

notebooks/official/pipelines/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_classification_logistic_regression_model_using_Scikit_learn_and_import_to_Vertex_AI/pipeline.py

@@ -1,73 +0,0 @@
-# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
-from kfp import components
-
-# %% Loading components
-download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/storage/download/component.yaml")
-select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Select_columns/in_CSV_format/component.yaml")
-fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
-binarize_column_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Binarize_column/in_CSV_format/component.yaml")
-train_logistic_regression_model_using_scikit_learn_from_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/ML_frameworks/Scikit_learn/Train_logistic_regression_model/from_CSV/component.yaml")
-upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/1f5cf6e06409b704064b2086c0a705e4e6b4fcde/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/component.yaml")
-deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml")
-
-# %% Pipeline definition
-def train_tabular_classification_logistic_regression_model_using_Scikit_learn_pipeline():
-    dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
-    feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]  # Excluded "trip_total"
-    label_column = "tips"
-    # Deploying the model might incur additional costs over time
-    deploy_model = False
-
-    classification_label_column = "class"
-    all_columns = [label_column] + feature_columns
-
-    training_data = download_from_gcs_op(
-        gcs_path=dataset_gcs_uri
-    ).outputs["Data"]
-
-    training_data = select_columns_using_Pandas_on_CSV_data_op(
-        table=training_data,
-        column_names=all_columns,
-    ).outputs["transformed_table"]
-
-    # Cleaning the NaN values.
-    training_data = fill_all_missing_values_using_Pandas_on_CSV_data_op(
-        table=training_data,
-        replacement_value="0",
-        #replacement_type_name="float",
-    ).outputs["transformed_table"]
-
-    classification_training_data = binarize_column_using_Pandas_on_CSV_data_op(
-        table=training_data,
-        column_name=label_column,
-        predicate="> 0",
-        new_column_name=classification_label_column,
-    ).outputs["transformed_table"]
-
-    model = train_logistic_regression_model_using_scikit_learn_from_CSV_op(
-        dataset=classification_training_data,
-        label_column_name=classification_label_column,
-        # Optional:
-        #penalty="l2",
-        #solver="lbfgs",
-        #max_iterations=100,
-        #multi_class_mode="auto",
-        #random_seed=0,
-    ).outputs["model"]
-
-    vertex_model_name = upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op(
-        model=model,
-    ).outputs["model_name"]
-
-    # Deploying the model might incur additional costs over time
-    if deploy_model:
-        sklearn_vertex_endpoint_name = deploy_model_to_endpoint_op(
-            model_name=vertex_model_name,
-        ).outputs["endpoint_name"]
-
-pipeline_func = train_tabular_classification_logistic_regression_model_using_Scikit_learn_pipeline
-
-# %% Pipeline submission
-if __name__ == '__main__':
-    from google.cloud import aiplatform
-    aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()

notebooks/official/pipelines/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_classification_model_using_PyTorch_and_import_to_Vertex_AI/pipeline.py

@@ -1,95 +0,0 @@
-# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
-from kfp import components
-
-# %% Loading components
-download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/storage/download/component.yaml")
-select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Select_columns/in_CSV_format/component.yaml")
-fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
-binarize_column_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Binarize_column/in_CSV_format/component.yaml")
-create_fully_connected_pytorch_network_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Create_fully_connected_network/component.yaml")
-train_pytorch_model_from_csv_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml")
-create_pytorch_model_archive_with_base_handler_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml")
-upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/component.yaml")
-deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml")
-
-# %% Pipeline definition
-def train_tabular_classification_model_using_PyTorch_pipeline():
-    dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
-    feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]  # Excluded "trip_total"
-    label_column = "tips"
-    # Deploying the model might incur additional costs over time
-    deploy_model = False
-
-    classification_label_column = "class"
-    all_columns = [label_column] + feature_columns
-
-    training_data = download_from_gcs_op(
-        gcs_path=dataset_gcs_uri
-    ).outputs["Data"]
-
-    training_data = select_columns_using_Pandas_on_CSV_data_op(
-        table=training_data,
-        column_names=all_columns,
-    ).outputs["transformed_table"]
-
-    # Cleaning the NaN values.
-    training_data = fill_all_missing_values_using_Pandas_on_CSV_data_op(
-        table=training_data,
-        replacement_value="0",
-        #replacement_type_name="float",
-    ).outputs["transformed_table"]
-
-    classification_training_data = binarize_column_using_Pandas_on_CSV_data_op(
-        table=training_data,
-        column_name=label_column,
-        predicate=" > 0",
-        new_column_name=classification_label_column,
-    ).outputs["transformed_table"]
-
-    network = create_fully_connected_pytorch_network_op(
-        input_size=len(feature_columns),
-        # Optional:
-        hidden_layer_sizes=[10],
-        activation_name="elu",
-        output_activation_name="sigmoid",
-        # output_size=1,
-    ).outputs["model"]
-
-    model = train_pytorch_model_from_csv_op(
-        model=network,
-        training_data=classification_training_data,
-        label_column_name=classification_label_column,
-        loss_function_name="binary_cross_entropy",
-        # Optional:
-        #number_of_epochs=1,
-        #learning_rate=0.1,
-        #optimizer_name="Adadelta",
-        #optimizer_parameters={},
-        #batch_size=32,
-        #batch_log_interval=100,
-        #random_seed=0,
-    ).outputs["trained_model"]
-
-    model_archive = create_pytorch_model_archive_with_base_handler_op(
-        model=model,
-        # Optional:
-        # model_name="model",
-        # model_version="1.0",
-    ).outputs["Model archive"]
-
-    vertex_model_name = upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op(
-        model_archive=model_archive,
-    ).outputs["model_name"]
-
-    # Deploying the model might incur additional costs over time
-    if deploy_model:
-        vertex_endpoint_name = deploy_model_to_endpoint_op(
-            model_name=vertex_model_name,
-        ).outputs["endpoint_name"]
-
-pipeline_func=train_tabular_classification_model_using_PyTorch_pipeline
-
-# %% Pipeline submission
-if __name__ == '__main__':
-    from google.cloud import aiplatform
-    aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()

notebooks/official/pipelines/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_classification_model_using_TensorFlow_and_import_to_Vertex_AI/pipeline.py

@@ -1,106 +0,0 @@
-# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
-from kfp import components
-
-# %% Loading components
-download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/storage/download/component.yaml")
-select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Select_columns/in_CSV_format/component.yaml")
-fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
-binarize_column_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Binarize_column/in_CSV_format/component.yaml")
-split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
-create_fully_connected_tensorflow_network_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Create_fully_connected_network/component.yaml")
-train_model_using_Keras_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml")
-predict_with_TensorFlow_model_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Predict/on_CSV/component.yaml")
-upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/component.yaml")
-deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml")
-
-# %% Pipeline definition
-def train_tabular_classification_model_using_TensorFlow_pipeline():
-    dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
-    feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]  # Excluded "trip_total"
-    label_column = "tips"
-    training_set_fraction = 0.8
-    # Deploying the model might incur additional costs over time
-    deploy_model = False
-
-    classification_label_column = "class"
-    all_columns = [label_column] + feature_columns
-
-    dataset = download_from_gcs_op(
-        gcs_path=dataset_gcs_uri
-    ).outputs["Data"]
-
-    dataset = select_columns_using_Pandas_on_CSV_data_op(
-        table=dataset,
-        column_names=all_columns,
-    ).outputs["transformed_table"]
-
-    dataset = fill_all_missing_values_using_Pandas_on_CSV_data_op(
-        table=dataset,
-        replacement_value="0",
-        # # Optional:
-        # column_names=None,  # =[...]
-    ).outputs["transformed_table"]
-
-    classification_dataset = binarize_column_using_Pandas_on_CSV_data_op(
-        table=dataset,
-        column_name=label_column,
-        predicate=" > 0",
-        new_column_name=classification_label_column,
-    ).outputs["transformed_table"]
-
-    split_task = split_rows_into_subsets_op(
-        table=classification_dataset,
-        fraction_1=training_set_fraction,
-    )
-    classification_training_data = split_task.outputs["split_1"]
-    classification_testing_data = split_task.outputs["split_2"]
-
-    network = create_fully_connected_tensorflow_network_op(
-        input_size=len(feature_columns),
-        # Optional:
-        hidden_layer_sizes=[10],
-        activation_name="elu",
-        output_activation_name="sigmoid",
-        # output_size=1,
-    ).outputs["model"]
-
-    model = train_model_using_Keras_on_CSV_op(
-        training_data=classification_training_data,
-        model=network,
-        label_column_name=classification_label_column,
-        # Optional:
-        loss_function_name="binary_crossentropy",
-        number_of_epochs=10,
-        #learning_rate=0.1,
-        #optimizer_name="Adadelta",
-        #optimizer_parameters={},
-        #batch_size=32,
-        #metric_names=["mean_absolute_error"],
-        #random_seed=0,
-    ).outputs["trained_model"]
-
-    predictions = predict_with_TensorFlow_model_on_CSV_data_op(
-        dataset=classification_testing_data,
-        model=model,
-        # label_column_name needs to be set when doing prediction on a dataset that has labels
-        label_column_name=classification_label_column,
-        # Optional:
-        # batch_size=1000,
-    ).outputs["predictions"]
-
-    vertex_model_name = upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op(
-        model=model,
-    ).outputs["model_name"]
-
-    # Deploying the model might incur additional costs over time
-    if deploy_model:
-        vertex_endpoint_name = deploy_model_to_endpoint_op(
-            model_name=vertex_model_name,
-        ).outputs["endpoint_name"]
-
-pipeline_func = train_tabular_classification_model_using_TensorFlow_pipeline
-
-# %% Pipeline submission
-if __name__ == '__main__':
-    from google.cloud import aiplatform
-    aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()

notebooks/official/pipelines/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_classification_model_using_XGBoost_and_import_to_Vertex_AI/pipeline.py

@@ -1,94 +0,0 @@
-# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
-from kfp import components
-
-# %% Loading components
-download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/storage/download/component.yaml")
-select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Select_columns/in_CSV_format/component.yaml")
-fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
-binarize_column_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Binarize_column/in_CSV_format/component.yaml")
-split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
-train_XGBoost_model_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/XGBoost/Train/component.yaml")
-xgboost_predict_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/XGBoost/Predict/component.yaml")
-upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/component.yaml")
-deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml")
-
-# %% Pipeline definition
-def train_tabular_classification_model_using_XGBoost_pipeline():
-    dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
-    feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]  # Excluded "trip_total"
-    label_column = "tips"
-    training_set_fraction = 0.8
-    # Deploying the model might incur additional costs over time
-    deploy_model = False
-
-    classification_label_column = "class"
-    all_columns = [label_column] + feature_columns
-
-    dataset = download_from_gcs_op(
-        gcs_path=dataset_gcs_uri
-    ).outputs["Data"]
-
-    dataset = select_columns_using_Pandas_on_CSV_data_op(
-        table=dataset,
-        column_names=all_columns,
-    ).outputs["transformed_table"]
-
-    dataset = fill_all_missing_values_using_Pandas_on_CSV_data_op(
-        table=dataset,
-        replacement_value="0",
-        # # Optional:
-        # column_names=None,  # =[...]
-    ).outputs["transformed_table"]
-
-    classification_dataset = binarize_column_using_Pandas_on_CSV_data_op(
-        table=dataset,
-        column_name=label_column,
-        predicate="> 0",
-        new_column_name=classification_label_column,
-    ).outputs["transformed_table"]
-
-    split_task = split_rows_into_subsets_op(
-        table=classification_dataset,
-        fraction_1=training_set_fraction,
-    )
-    classification_training_data = split_task.outputs["split_1"]
-    classification_testing_data = split_task.outputs["split_2"]
-
-    model = train_XGBoost_model_on_CSV_op(
-        training_data=classification_training_data,
-        label_column_name=classification_label_column,
-        objective="binary:logistic",
-        # Optional:
-        #starting_model=None,
-        #num_iterations=10,
-        #booster_params={},
-        #booster="gbtree",
-        #learning_rate=0.3,
-        #min_split_loss=0,
-        #max_depth=6,
-    ).outputs["model"]
-
-    # Predicting on the testing data
-    predictions = xgboost_predict_on_CSV_op(
-        data=classification_testing_data,
-        model=model,
-        # label_column needs to be set when doing prediction on a dataset that has labels
-        label_column_name=classification_label_column,
-    ).outputs["predictions"]
-
-    vertex_model_name = upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op(
-        model=model,
-    ).outputs["model_name"]
-
-    # Deploying the model might incur additional costs over time
-    if deploy_model:
-        vertex_endpoint_name = deploy_model_to_endpoint_op(
-            model_name=vertex_model_name,
-        ).outputs["endpoint_name"]
-
-pipeline_func = train_tabular_classification_model_using_XGBoost_pipeline
-
-# %% Pipeline submission
-if __name__ == '__main__':
-    from google.cloud import aiplatform
-    aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()

notebooks/official/pipelines/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_classification_model_using_all_frameworks_and_import_to_Vertex_AI/pipeline.py

@@ -1,224 +0,0 @@
-# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
-from kfp import components
-
-# %% Loading components
-download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/storage/download/component.yaml")
-select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Select_columns/in_CSV_format/component.yaml")
-fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
-binarize_column_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Binarize_column/in_CSV_format/component.yaml")
-split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
-
-# TensorFlow
-create_fully_connected_tensorflow_network_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Create_fully_connected_network/component.yaml")
-train_model_using_Keras_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml")
-predict_with_TensorFlow_model_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Predict/on_CSV/component.yaml")
-upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/component.yaml")
-
-# PyTorch
-create_fully_connected_pytorch_network_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Create_fully_connected_network/component.yaml")
-train_pytorch_model_from_csv_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml")
-create_pytorch_model_archive_with_base_handler_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml")
-upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/component.yaml")
-
-# XGBoost
-train_XGBoost_model_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/XGBoost/Train/component.yaml")
-xgboost_predict_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/XGBoost/Predict/component.yaml")
-upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/component.yaml")
-
-# Scikit-learn
-#train_linear_regression_model_using_scikit_learn_from_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/1f5cf6e06409b704064b2086c0a705e4e6b4fcde/community-content/pipeline_components/ML_frameworks/Scikit_learn/Train_linear_regression_model/from_CSV/component.yaml")
-train_logistic_regression_model_using_scikit_learn_from_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/1f5cf6e06409b704064b2086c0a705e4e6b4fcde/community-content/pipeline_components/ML_frameworks/Scikit_learn/Train_logistic_regression_model/from_CSV/component.yaml")
-upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/component.yaml")
-
-# Vertex AI
-deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml")
-
-# %% Pipeline definition
-def train_tabular_classification_model_using_all_frameworks_pipeline():
-    dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
-    feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]  # Excluded "trip_total"
-    label_column = "tips"
-    training_set_fraction = 0.8
-    # Deploying the model might incur additional costs over time
-    deploy_model = False
-
-    classification_label_column = "class"
-    all_columns = [label_column] + feature_columns
-
-    dataset = download_from_gcs_op(
-        gcs_path=dataset_gcs_uri
-    ).outputs["Data"]
-
-    dataset = select_columns_using_Pandas_on_CSV_data_op(
-        table=dataset,
-        column_names=all_columns,
-    ).outputs["transformed_table"]
-
-    dataset = fill_all_missing_values_using_Pandas_on_CSV_data_op(
-        table=dataset,
-        replacement_value="0",
-        # # Optional:
-        # column_names=None,  # =[...]
-    ).outputs["transformed_table"]
-
-    classification_dataset = binarize_column_using_Pandas_on_CSV_data_op(
-        table=dataset,
-        column_name=label_column,
-        predicate=" > 0",
-        new_column_name=classification_label_column,
-    ).outputs["transformed_table"]
-
-    split_task = split_rows_into_subsets_op(
-        table=classification_dataset,
-        fraction_1=training_set_fraction,
-    )
-    classification_training_data = split_task.outputs["split_1"]
-    classification_testing_data = split_task.outputs["split_2"]
-
-    # TensorFlow
-    tensorflow_network = create_fully_connected_tensorflow_network_op(
-        input_size=len(feature_columns),
-        # Optional:
-        hidden_layer_sizes=[10],
-        activation_name="elu",
-        output_activation_name="sigmoid",
-        # output_size=1,
-    ).outputs["model"]
-
-    tensorflow_model = train_model_using_Keras_on_CSV_op(
-        training_data=classification_training_data,
-        model=tensorflow_network,
-        label_column_name=classification_label_column,
-        # Optional:
-        loss_function_name="binary_crossentropy",
-        number_of_epochs=10,
-        #learning_rate=0.1,
-        #optimizer_name="Adadelta",
-        #optimizer_parameters={},
-        #batch_size=32,
-        #metric_names=["mean_absolute_error"],
-        #random_seed=0,
-    ).outputs["trained_model"]
-
-    tensorflow_predictions = predict_with_TensorFlow_model_on_CSV_data_op(
-        dataset=classification_testing_data,
-        model=tensorflow_model,
-        # label_column_name needs to be set when doing prediction on a dataset that has labels
-        label_column_name=classification_label_column,
-        # Optional:
-        # batch_size=1000,
-    ).outputs["predictions"]
-
-    tensorflow_vertex_model_name = upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op(
-        model=tensorflow_model,
-    ).outputs["model_name"]
-
-    # Deploying the model might incur additional costs over time
-    if deploy_model:
-        tensorflow_vertex_endpoint_name = deploy_model_to_endpoint_op(
-            model_name=tensorflow_vertex_model_name,
-        ).outputs["endpoint_name"]
-
-    # PyTorch
-    pytorch_network = create_fully_connected_pytorch_network_op(
-        input_size=len(feature_columns),
-        # Optional:
-        hidden_layer_sizes=[10],
-        activation_name="elu",
-        output_activation_name="sigmoid",
-        # output_size=1,
-    ).outputs["model"]
-
-    pytorch_model = train_pytorch_model_from_csv_op(
-        model=pytorch_network,
-        training_data=classification_training_data,
-        label_column_name=classification_label_column,
-        loss_function_name="binary_cross_entropy",
-        # Optional:
-        #number_of_epochs=1,
-        #learning_rate=0.1,
-        #optimizer_name="Adadelta",
-        #optimizer_parameters={},
-        #batch_size=32,
-        #batch_log_interval=100,
-        #random_seed=0,
-    ).outputs["trained_model"]
-
-    pytorch_model_archive = create_pytorch_model_archive_with_base_handler_op(
-        model=pytorch_model,
-        # Optional:
-        # model_name="model",
-        # model_version="1.0",
-    ).outputs["Model archive"]
-
-    pytorch_vertex_model_name = upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op(
-        model_archive=pytorch_model_archive,
-    ).outputs["model_name"]
-
-    # Deploying the model might incur additional costs over time
-    if deploy_model:
-        pytorch_vertex_endpoint_name = deploy_model_to_endpoint_op(
-            model_name=pytorch_vertex_model_name,
-        ).outputs["endpoint_name"]
-
-    # XGBoost
-    xgboost_model = train_XGBoost_model_on_CSV_op(
-        training_data=classification_training_data,
-        label_column_name=classification_label_column,
-        objective="binary:logistic",
-        # Optional:
-        #starting_model=None,
-        #num_iterations=10,
-        #booster_params={},
-        #booster="gbtree",
-        #learning_rate=0.3,
-        #min_split_loss=0,
-        #max_depth=6,
-    ).outputs["model"]
-
-    # Predicting on the testing data
-    xgboost_predictions = xgboost_predict_on_CSV_op(
-        data=classification_testing_data,
-        model=xgboost_model,
-        # label_column needs to be set when doing prediction on a dataset that has labels
-        label_column_name=classification_label_column,
-    ).outputs["predictions"]
-
-    xgboost_vertex_model_name = upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op(
-        model=xgboost_model,
-    ).outputs["model_name"]
-
-    # Deploying the model might incur additional costs over time
-    if deploy_model:
-        xgboost_vertex_endpoint_name = deploy_model_to_endpoint_op(
-            model_name=xgboost_vertex_model_name,
-        ).outputs["endpoint_name"]
-
-    # Scikit-learn
-    sklearn_model = train_logistic_regression_model_using_scikit_learn_from_CSV_op(
-        dataset=classification_training_data,
-        label_column_name=classification_label_column,
-        # Optional:
-        #penalty="l2",
-        #solver="lbfgs",
-        #max_iterations=100,
-        #multi_class_mode="auto",
-        #random_seed=0,
-    ).outputs["model"]
-
-    sklearn_vertex_model_name = upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op(
-        model=sklearn_model,
-    ).outputs["model_name"]
-
-    # Deploying the model might incur additional costs over time
-    if deploy_model:
-        sklearn_vertex_endpoint_name = deploy_model_to_endpoint_op(
-            model_name=sklearn_vertex_model_name,
-        ).outputs["endpoint_name"]
-
-pipeline_func=train_tabular_classification_model_using_all_frameworks_pipeline
-
-# %% Pipeline submission
-if __name__ == '__main__':
-    from google.cloud import aiplatform
-    aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()

notebooks/official/pipelines/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_regression_linear_model_using_Scikit_learn_and_import_to_Vertex_AI/pipeline.py

@@ -1,57 +0,0 @@
-# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
-from kfp import components
-
-# %% Loading components
-download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/storage/download/component.yaml")
-select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Select_columns/in_CSV_format/component.yaml")
-fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
-train_linear_regression_model_using_scikit_learn_from_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/1f5cf6e06409b704064b2086c0a705e4e6b4fcde/community-content/pipeline_components/ML_frameworks/Scikit_learn/Train_linear_regression_model/from_CSV/component.yaml")
-upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/component.yaml")
-deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml")
-
-# %% Pipeline definition
-def train_tabular_regression_linear_model_using_Scikit_learn_pipeline():
-    dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
-    feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]  # Excluded "trip_total"
-    label_column = "tips"
-    all_columns = [label_column] + feature_columns
-    # Deploying the model might incur additional costs over time
-    deploy_model = False
-
-    training_data = download_from_gcs_op(
-        gcs_path=dataset_gcs_uri
-    ).outputs["Data"]
-
-    training_data = select_columns_using_Pandas_on_CSV_data_op(
-        table=training_data,
-        column_names=all_columns,
-    ).outputs["transformed_table"]
-
-    # Cleaning the NaN values.
-    training_data = fill_all_missing_values_using_Pandas_on_CSV_data_op(
-        table=training_data,
-        replacement_value="0",
-        #replacement_type_name="float",
-    ).outputs["transformed_table"]
-
-    model = train_linear_regression_model_using_scikit_learn_from_CSV_op(
-        dataset=training_data,
-        label_column_name=label_column,
-    ).outputs["model"]
-
-    vertex_model_name = upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op(
-        model=model,
-    ).outputs["model_name"]
-
-    # Deploying the model might incur additional costs over time
-    if deploy_model:
-        sklearn_vertex_endpoint_name = deploy_model_to_endpoint_op(
-            model_name=vertex_model_name,
-        ).outputs["endpoint_name"]
-
-pipeline_func = train_tabular_regression_linear_model_using_Scikit_learn_pipeline
-
-# %% Pipeline submission
-if __name__ == '__main__':
-    from google.cloud import aiplatform
-    aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()

notebooks/official/pipelines/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_regression_model_using_PyTorch_and_import_to_Vertex_AI/pipeline.py

@@ -1,85 +0,0 @@
-# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
-from kfp import components
-
-# %% Loading components
-download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/storage/download/component.yaml")
-select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Select_columns/in_CSV_format/component.yaml")
-fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
-create_fully_connected_pytorch_network_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Create_fully_connected_network/component.yaml")
-train_pytorch_model_from_csv_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml")
-create_pytorch_model_archive_with_base_handler_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml")
-upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/component.yaml")
-deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml")
-
-# %% Pipeline definition
-def train_tabular_regression_model_using_PyTorch_pipeline():
-    dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
-    feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]  # Excluded "trip_total"
-    label_column = "tips"
-    all_columns = [label_column] + feature_columns
-    # Deploying the model might incur additional costs over time
-    deploy_model = False
-
-    training_data = download_from_gcs_op(
-        gcs_path=dataset_gcs_uri
-    ).outputs["Data"]
-
-    training_data = select_columns_using_Pandas_on_CSV_data_op(
-        table=training_data,
-        column_names=all_columns,
-    ).outputs["transformed_table"]
-
-    # Cleaning the NaN values.
-    training_data = fill_all_missing_values_using_Pandas_on_CSV_data_op(
-        table=training_data,
-        replacement_value="0",
-        #replacement_type_name="float",
-    ).outputs["transformed_table"]
-
-    network = create_fully_connected_pytorch_network_op(
-        input_size=len(feature_columns),
-        # Optional:
-        hidden_layer_sizes=[10],
-        activation_name="elu",
-        # output_activation_name=None,
-        # output_size=1,
-    ).outputs["model"]
-
-    model = train_pytorch_model_from_csv_op(
-        model=network,
-        training_data=training_data,
-        label_column_name=label_column,
-        # Optional:
-        #loss_function_name="mse_loss",
-        #number_of_epochs=1,
-        #learning_rate=0.1,
-        #optimizer_name="Adadelta",
-        #optimizer_parameters={},
-        #batch_size=32,
-        #batch_log_interval=100,
-        #random_seed=0,
-    ).outputs["trained_model"]
-
-    model_archive = create_pytorch_model_archive_with_base_handler_op(
-        model=model,
-        # Optional:
-        # model_name="model",
-        # model_version="1.0",
-    ).outputs["Model archive"]
-
-    vertex_model_name = upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op(
-        model_archive=model_archive,
-    ).outputs["model_name"]
-
-    # Deploying the model might incur additional costs over time
-    if deploy_model:
-        vertex_endpoint_name = deploy_model_to_endpoint_op(
-            model_name=vertex_model_name,
-        ).outputs["endpoint_name"]
-
-pipeline_func=train_tabular_regression_model_using_PyTorch_pipeline
-
-# %% Pipeline submission
-if __name__ == '__main__':
-    from google.cloud import aiplatform
-    aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()

notebooks/official/pipelines/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_regression_model_using_TensorFlow_and_import_to_Vertex_AI/pipeline.py

@@ -1,97 +0,0 @@
-# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
-from kfp import components
-
-# %% Loading components
-download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/storage/download/component.yaml")
-select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Select_columns/in_CSV_format/component.yaml")
-fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
-split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
-create_fully_connected_tensorflow_network_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Create_fully_connected_network/component.yaml")
-train_model_using_Keras_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml")
-predict_with_TensorFlow_model_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Predict/on_CSV/component.yaml")
-upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/component.yaml")
-deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml")
-
-# %% Pipeline definition
-def train_tabular_regression_model_using_Tensorflow_pipeline():
-    dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
-    feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]  # Excluded "trip_total"
-    label_column = "tips"
-    training_set_fraction = 0.8
-    # Deploying the model might incur additional costs over time
-    deploy_model = False
-
-    all_columns = [label_column] + feature_columns
-
-    dataset = download_from_gcs_op(
-        gcs_path=dataset_gcs_uri
-    ).outputs["Data"]
-
-    dataset = select_columns_using_Pandas_on_CSV_data_op(
-        table=dataset,
-        column_names=all_columns,
-    ).outputs["transformed_table"]
-
-    dataset = fill_all_missing_values_using_Pandas_on_CSV_data_op(
-        table=dataset,
-        replacement_value="0",
-        # # Optional:
-        # column_names=None,  # =[...]
-    ).outputs["transformed_table"]
-
-    split_task = split_rows_into_subsets_op(
-        table=dataset,
-        fraction_1=training_set_fraction,
-    )
-    training_data = split_task.outputs["split_1"]
-    testing_data = split_task.outputs["split_2"]
-
-    network = create_fully_connected_tensorflow_network_op(
-        input_size=len(feature_columns),
-        # Optional:
-        hidden_layer_sizes=[10],
-        activation_name="elu",
-        # output_activation_name=None,
-        # output_size=1,
-    ).outputs["model"]
-
-    model = train_model_using_Keras_on_CSV_op(
-        training_data=training_data,
-        model=network,
-        label_column_name=label_column,
-        # Optional:
-        #loss_function_name="mean_squared_error",
-        number_of_epochs=10,
-        #learning_rate=0.1,
-        #optimizer_name="Adadelta",
-        #optimizer_parameters={},
-        #batch_size=32,
-        metric_names=["mean_absolute_error"],
-        #random_seed=0,
-    ).outputs["trained_model"]
-
-    predictions = predict_with_TensorFlow_model_on_CSV_data_op(
-        dataset=testing_data,
-        model=model,
-        # label_column_name needs to be set when doing prediction on a dataset that has labels
-        label_column_name=label_column,
-        # Optional:
-        # batch_size=1000,
-    ).outputs["predictions"]
-
-    vertex_model_name = upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op(
-        model=model,
-    ).outputs["model_name"]
-
-    # Deploying the model might incur additional costs over time
-    if deploy_model:
-        vertex_endpoint_name = deploy_model_to_endpoint_op(
-            model_name=vertex_model_name,
-        ).outputs["endpoint_name"]
-
-pipeline_func=train_tabular_regression_model_using_Tensorflow_pipeline
-
-# %% Pipeline submission
-if __name__ == '__main__':
-    from google.cloud import aiplatform
-    aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()

notebooks/official/pipelines/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_regression_model_using_XGBoost_and_import_to_Vertex_AI/pipeline.py

@@ -1,85 +0,0 @@
-# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
-from kfp import components
-
-# %% Loading components
-download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/storage/download/component.yaml")
-select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Select_columns/in_CSV_format/component.yaml")
-fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
-split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
-train_XGBoost_model_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/XGBoost/Train/component.yaml")
-xgboost_predict_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/XGBoost/Predict/component.yaml")
-upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/component.yaml")
-deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml")
-
-# %% Pipeline definition
-def train_tabular_regression_model_using_XGBoost_pipeline():
-    dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
-    feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]  # Excluded "trip_total"
-    label_column = "tips"
-    training_set_fraction = 0.8
-    # Deploying the model might incur additional costs over time
-    deploy_model = False
-
-    all_columns = [label_column] + feature_columns
-
-    dataset = download_from_gcs_op(
-        gcs_path=dataset_gcs_uri
-    ).outputs["Data"]
-
-    dataset = select_columns_using_Pandas_on_CSV_data_op(
-        table=dataset,
-        column_names=all_columns,
-    ).outputs["transformed_table"]
-
-    dataset = fill_all_missing_values_using_Pandas_on_CSV_data_op(
-        table=dataset,
-        replacement_value="0",
-        # # Optional:
-        # column_names=None,  # =[...]
-    ).outputs["transformed_table"]
-
-    split_task = split_rows_into_subsets_op(
-        table=dataset,
-        fraction_1=training_set_fraction,
-    )
-    training_data = split_task.outputs["split_1"]
-    testing_data = split_task.outputs["split_2"]
-
-    model = train_XGBoost_model_on_CSV_op(
-        training_data=training_data,
-        label_column_name=label_column,
-        # Optional:
-        #starting_model=None,
-        #num_iterations=10,
-        #booster_params={},
-        #objective="reg:squarederror",
-        #booster="gbtree",
-        #learning_rate=0.3,
-        #min_split_loss=0,
-        #max_depth=6,
-    ).outputs["model"]
-
-    # Predicting on the testing data
-    predictions = xgboost_predict_on_CSV_op(
-        data=testing_data,
-        model=model,
-        # label_column needs to be set when doing prediction on a dataset that has labels
-        label_column_name=label_column,
-    ).outputs["predictions"]
-
-    vertex_model_name = upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op(
-        model=model,
-    ).outputs["model_name"]
-
-    # Deploying the model might incur additional costs over time
-    if deploy_model:
-        vertex_endpoint_name = deploy_model_to_endpoint_op(
-            model_name=vertex_model_name,
-        ).outputs["endpoint_name"]
-
-pipeline_func = train_tabular_regression_model_using_XGBoost_pipeline
-
-# %% Pipeline submission
-if __name__ == '__main__':
-    from google.cloud import aiplatform
-    aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()

notebooks/official/pipelines/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_regression_model_using_all_frameworks_and_import_to_Vertex_AI/pipeline.py

@@ -1,208 +0,0 @@
-# python3 -m pip install "kfp<2.0.0" "google-cloud-aiplatform>=1.16.0" --upgrade --quiet
-from kfp import components
-
-# %% Loading components
-download_from_gcs_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/storage/download/component.yaml")
-select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Select_columns/in_CSV_format/component.yaml")
-fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
-split_rows_into_subsets_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/dataset_manipulation/Split_rows_into_subsets/in_CSV/component.yaml")
-
-# TensorFlow
-create_fully_connected_tensorflow_network_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Create_fully_connected_network/component.yaml")
-train_model_using_Keras_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Train_model_using_Keras/on_CSV/component.yaml")
-predict_with_TensorFlow_model_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/tensorflow/Predict/on_CSV/component.yaml")
-upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_Tensorflow_model/component.yaml")
-
-# PyTorch
-create_fully_connected_pytorch_network_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Create_fully_connected_network/component.yaml")
-train_pytorch_model_from_csv_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Train_PyTorch_model/from_CSV/component.yaml")
-create_pytorch_model_archive_with_base_handler_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/PyTorch/Create_PyTorch_Model_Archive/with_base_handler/component.yaml")
-upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/component.yaml")
-
-# XGBoost
-train_XGBoost_model_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/XGBoost/Train/component.yaml")
-xgboost_predict_on_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/XGBoost/Predict/component.yaml")
-upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_XGBoost_model/component.yaml")
-
-# Scikit-learn
-train_linear_regression_model_using_scikit_learn_from_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/1f5cf6e06409b704064b2086c0a705e4e6b4fcde/community-content/pipeline_components/ML_frameworks/Scikit_learn/Train_linear_regression_model/from_CSV/component.yaml")
-upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/component.yaml")
-
-# Vertex AI
-deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml")
-
-# %% Pipeline definition
-def train_tabular_regression_model_using_all_frameworks_pipeline():
-    dataset_gcs_uri = "gs://ml-pipeline-dataset/Chicago_taxi_trips/chicago_taxi_trips_2019-01-01_-_2019-02-01_limit=10000.csv"
-    feature_columns = ["trip_seconds", "trip_miles", "pickup_community_area", "dropoff_community_area", "fare", "tolls", "extras"]  # Excluded "trip_total"
-    label_column = "tips"
-    training_set_fraction = 0.8
-    # Deploying the model might incur additional costs over time
-    deploy_model = False
-
-    all_columns = [label_column] + feature_columns
-
-    dataset = download_from_gcs_op(
-        gcs_path=dataset_gcs_uri
-    ).outputs["Data"]
-
-    dataset = select_columns_using_Pandas_on_CSV_data_op(
-        table=dataset,
-        column_names=all_columns,
-    ).outputs["transformed_table"]
-
-    dataset = fill_all_missing_values_using_Pandas_on_CSV_data_op(
-        table=dataset,
-        replacement_value="0",
-        # # Optional:
-        # column_names=None,  # =[...]
-    ).outputs["transformed_table"]
-
-    split_task = split_rows_into_subsets_op(
-        table=dataset,
-        fraction_1=training_set_fraction,
-    )
-    training_data = split_task.outputs["split_1"]
-    testing_data = split_task.outputs["split_2"]
-
-    # TensorFlow
-    tensorflow_network = create_fully_connected_tensorflow_network_op(
-        input_size=len(feature_columns),
-        # Optional:
-        hidden_layer_sizes=[10],
-        activation_name="elu",
-        # output_activation_name=None,
-        # output_size=1,
-    ).outputs["model"]
-
-    tensorflow_model = train_model_using_Keras_on_CSV_op(
-        training_data=training_data,
-        model=tensorflow_network,
-        label_column_name=label_column,
-        # Optional:
-        #loss_function_name="mean_squared_error",
-        number_of_epochs=10,
-        #learning_rate=0.1,
-        #optimizer_name="Adadelta",
-        #optimizer_parameters={},
-        #batch_size=32,
-        metric_names=["mean_absolute_error"],
-        #random_seed=0,
-    ).outputs["trained_model"]
-
-    tensorflow_predictions = predict_with_TensorFlow_model_on_CSV_data_op(
-        dataset=testing_data,
-        model=tensorflow_model,
-        # label_column_name needs to be set when doing prediction on a dataset that has labels
-        label_column_name=label_column,
-        # Optional:
-        # batch_size=1000,
-    ).outputs["predictions"]
-
-    tensorflow_vertex_model_name = upload_Tensorflow_model_to_Google_Cloud_Vertex_AI_op(
-        model=tensorflow_model,
-    ).outputs["model_name"]
-
-    # Deploying the model might incur additional costs over time
-    if deploy_model:
-        tensorflow_vertex_endpoint_name = deploy_model_to_endpoint_op(
-            model_name=tensorflow_vertex_model_name,
-        ).outputs["endpoint_name"]
-
-    # PyTorch
-    pytorch_network = create_fully_connected_pytorch_network_op(
-        input_size=len(feature_columns),
-        # Optional:
-        hidden_layer_sizes=[10],
-        activation_name="elu",
-        # output_activation_name=None,
-        # output_size=1,
-    ).outputs["model"]
-
-    pytorch_model = train_pytorch_model_from_csv_op(
-        model=pytorch_network,
-        training_data=training_data,
-        label_column_name=label_column,
-        # Optional:
-        #loss_function_name="mse_loss",
-        #number_of_epochs=1,
-        #learning_rate=0.1,
-        #optimizer_name="Adadelta",
-        #optimizer_parameters={},
-        #batch_size=32,
-        #batch_log_interval=100,
-        #random_seed=0,
-    ).outputs["trained_model"]
-
-    pytorch_model_archive = create_pytorch_model_archive_with_base_handler_op(
-        model=pytorch_model,
-        # Optional:
-        # model_name="model",
-        # model_version="1.0",
-    ).outputs["Model archive"]
-
-    pytorch_vertex_model_name = upload_PyTorch_model_archive_to_Google_Cloud_Vertex_AI_op(
-        model_archive=pytorch_model_archive,
-    ).outputs["model_name"]
-
-    # Deploying the model might incur additional costs over time
-    if deploy_model:
-        pytorch_vertex_endpoint_name = deploy_model_to_endpoint_op(
-            model_name=pytorch_vertex_model_name,
-        ).outputs["endpoint_name"]
-
-    # XGBoost
-    xgboost_model = train_XGBoost_model_on_CSV_op(
-        training_data=training_data,
-        label_column_name=label_column,
-        # Optional:
-        #starting_model=None,
-        #num_iterations=10,
-        #booster_params={},
-        #objective="reg:squarederror",
-        #booster="gbtree",
-        #learning_rate=0.3,
-        #min_split_loss=0,
-        #max_depth=6,
-    ).outputs["model"]
-
-    # Predicting on the testing data
-    xgboost_predictions = xgboost_predict_on_CSV_op(
-        data=testing_data,
-        model=xgboost_model,
-        # label_column needs to be set when doing prediction on a dataset that has labels
-        label_column_name=label_column,
-    ).outputs["predictions"]
-
-    xgboost_vertex_model_name = upload_XGBoost_model_to_Google_Cloud_Vertex_AI_op(
-        model=xgboost_model,
-    ).outputs["model_name"]
-
-    # Deploying the model might incur additional costs over time
-    if deploy_model:
-        xgboost_vertex_endpoint_name = deploy_model_to_endpoint_op(
-            model_name=xgboost_vertex_model_name,
-        ).outputs["endpoint_name"]
-
-    # Scikit-learn
-    sklearn_model = train_linear_regression_model_using_scikit_learn_from_CSV_op(
-        dataset=training_data,
-        label_column_name=label_column,
-    ).outputs["model"]
-
-    sklearn_vertex_model_name = upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op(
-        model=sklearn_model,
-    ).outputs["model_name"]
-
-    # Deploying the model might incur additional costs over time
-    if deploy_model:
-        sklearn_vertex_endpoint_name = deploy_model_to_endpoint_op(
-            model_name=sklearn_vertex_model_name,
-        ).outputs["endpoint_name"]
-
-pipeline_func=train_tabular_regression_model_using_all_frameworks_pipeline
-
-# %% Pipeline submission
-if __name__ == '__main__':
-    from google.cloud import aiplatform
-    aiplatform.PipelineJob.from_pipeline_func(pipeline_func=pipeline_func).submit()

notebooks/official/pipelines/automl_tabular_classification_beans.ipynb

@@ -68,7 +68,7 @@
         "\n",
         "<a href=\"https://storage.googleapis.com/amy-jo/images/mp/beans.png\" target=\"_blank\"><img src=\"https://storage.googleapis.com/amy-jo/images/mp/beans.png\" width=\"95%\"/></a>\n",
         "\n",
-        "Learn more about [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction) and [AutoML components](https://cloud.google.com/vertex-ai/docs/pipelines/vertex-automl-component). Learn more about [Classification for tabular data](https://cloud.google.com/vertex-ai/docs/tabular-data/classification-regression/overview)."
+        "Learn more about [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction) and [AutoML components](https://cloud.google.com/vertex-ai/docs/pipelines/vertex-automl-component)."
       ]
     },
     {

notebooks/official/pipelines/custom_model_training_and_batch_prediction.ipynb

@@ -65,7 +65,7 @@
         "\n",
         "This tutorial demonstrates how to use Vertex AI Pipelines with pre-built Google Cloud Pipeline Components for custom training.\n",
         "\n",
-        "Learn more about [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction) and [Custom training components](https://cloud.google.com/vertex-ai/docs/training/create-training-pipeline)."
+        "Learn more about [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction) and [Vertex AI Training components](https://cloud.google.com/vertex-ai/docs/training/create-training-pipeline)."
       ]
     },
     {

notebooks/official/pipelines/google_cloud_pipeline_components_automl_tabular.ipynb

@@ -63,7 +63,7 @@
         "\n",
         "This notebook shows how to use the components defined in [`google_cloud_pipeline_components`](https://github.com/kubeflow/pipelines/tree/master/components/google-cloud) to build an AutoML tabular regression workflow on Vertex AI Pipelines.\n",
         "\n",
-        "Learn more about [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction) and [AutoML components](https://cloud.google.com/vertex-ai/docs/pipelines/vertex-automl-component). Learn more about [Regression for tabular data](https://cloud.google.com/vertex-ai/docs/tabular-data/classification-regression/overview)."
+        "Learn more about [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction) and [AutoML components](https://cloud.google.com/vertex-ai/docs/pipelines/vertex-automl-component)."
       ]
     },
     {

notebooks/official/pipelines/google_cloud_pipeline_components_dataproc_tabular.ipynb

@@ -784,7 +784,7 @@
       "outputs": [],
       "source": [
         "# Setup\n",
-        "DATAPROC_RUNTIME_VERSION = \"1.1.3\"\n",
+        "SPARK_VERSION = \"3.1.2\"\n",
         "SRC = path(\"src\")\n",
         "BUILD_PATH = path(\"build\")\n",
         "DELIVERABLES = path(\"deliverables\")\n",
@@ -856,7 +856,7 @@
         "    HPT_BUNDLE_URI,\n",
         "]\n",
         "HPT_RUNTIME_PROPERTIES = {\n",
-        "    \"spark.jars.packages\": \"ml.combust.mleap:mleap-spark-base_2.12:0.21.1,ml.combust.mleap:mleap-spark_2.12:0.21.1\"\n",
+        "    \"spark.jars.packages\": \"ml.combust.mleap:mleap-spark-base_2.12:0.20.0,ml.combust.mleap:mleap-spark_2.12:0.20.0\"\n",
         "}\n",
         "\n",
         "# Experiment\n",
@@ -2465,7 +2465,6 @@
         "    deploy_model: bool = DEPLOY_MODEL,\n",
         "    artifact_uri: str = ARTIFACT_URI,\n",
         "    serving_image_uri: str = SERVING_IMAGE_URI,\n",
-        "    dataproc_runtime_version: str = DATAPROC_RUNTIME_VERSION,\n",
         "):\n",
         "    from google_cloud_pipeline_components.v1.dataproc import \\\n",
         "        DataprocPySparkBatchOp\n",
@@ -2486,7 +2485,6 @@
         "        main_python_file_uri=preprocessing_main_python_file_uri,\n",
         "        args=build_preprocessing_args_op.output,\n",
         "        subnetwork_uri=subnetwork_uri,\n",
-        "        runtime_config_version=dataproc_runtime_version,\n",
         "    ).after(build_preprocessing_args_op)\n",
         "\n",
         "    # build training data args\n",
@@ -2504,7 +2502,6 @@
         "        main_python_file_uri=training_main_python_file_uri,\n",
         "        args=build_training_args_op.output,\n",
         "        subnetwork_uri=subnetwork_uri,\n",
-        "        runtime_config_version=dataproc_runtime_version,\n",
         "    ).after(build_training_args_op)\n",
         "\n",
         "    evaluate_model_op = evaluate_model(metrics_uri=metrics_path).after(\n",
@@ -2533,8 +2530,6 @@
         "            args=build_hpt_args_op.output,\n",
         "            runtime_config_properties=HPT_RUNTIME_PROPERTIES,\n",
         "            subnetwork_uri=subnetwork_uri,\n",
-        "            # TODO: change to Dataproc Serverless Runtime 1.1.x image when MLeap supports Spark 3.3\n",
-        "            runtime_config_version=\"1.0.29\",\n",
         "        ).after(model_training_op)\n",
         "\n",
         "        # evaluate condition to upload and deploy model to Vertex AI\n",

notebooks/official/pipelines/google_cloud_pipeline_components_model_train_upload_deploy.ipynb

@@ -63,7 +63,7 @@
         "\n",
         "This notebook shows how to use the components defined in [`google_cloud_pipeline_components`](https://github.com/kubeflow/pipelines/tree/master/components/google-cloud) to build a [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines) workflow that trains a [custom model](https://cloud.google.com/vertex-ai/docs/training/containers-overview), uploads the model as a `Model` resource, creates an `Endpoint` resource, and deploys the `Model` resource to the `Endpoint` resource.\n",
         "\n",
-        "Learn more about [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction) and [Custom training components](https://cloud.google.com/vertex-ai/docs/pipelines/customjob-component)."
+        "Learn more about [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction) and [Vertex AI Training components](https://cloud.google.com/vertex-ai/docs/pipelines/customjob-component)."
       ]
     },
     {
@@ -94,7 +94,7 @@
         "- Compile the KFP pipeline.\n",
         "- Execute the KFP pipeline using `Vertex AI Pipelines`\n",
         "\n",
-        "The components are [documented here](https://google-cloud-pipeline-components.readthedocs.io/en/google-cloud-pipeline-components-1.0.39/google_cloud_pipeline_components.aiplatform.html).\n",
+        "The components are [documented here](https://google-cloud-pipeline-components.readthedocs.io/en/latest/google_cloud_pipeline_components.aiplatform.html#module-google_cloud_pipeline_components.aiplatform).\n",
         "(From that page, see also the `CustomPythonPackageTrainingJobRunOp` and `CustomContainerTrainingJobRunOp` components, which similarly run 'custom' training, but as with the related `google.cloud.aiplatform.CustomContainerTrainingJob` and `google.cloud.aiplatform.CustomPythonPackageTrainingJob` methods from the [Vertex AI SDK](https://googleapis.dev/python/aiplatform/latest/aiplatform.html), also upload the trained model)."
       ]
     },

notebooks/official/pipelines/pipelines_intro_kfp.ipynb

@@ -207,7 +207,7 @@
       },
       "outputs": [],
       "source": [
-        "PROJECT_ID = \"andy-1234-221921\"  # @param {type:\"string\"}\n",
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}\n",
         "\n",
         "# Set the project id\n",
         "! gcloud config set project {PROJECT_ID}"
@@ -590,9 +590,8 @@
       "outputs": [],
       "source": [
         "@component\n",
-        "def consumer(text1: str, text2: str, text3: str) -> str:\n",
-        "    print(f\"text1: {text1}; text2: {text2}; text3: {text3}\")\n",
-        "    return f\"text1: {text1}; text2: {text2}; text3: {text3}\""
+        "def consumer(text1: str, text2: str, text3: str):\n",
+        "    print(f\"text1: {text1}; text2: {text2}; text3: {text3}\")"
       ]
     },
     {

notebooks/official/pytorch/README.md

@@ -0,0 +1,20 @@
+
+[Training, tuning and deploying a PyTorch text sentiment classification model on Vertex AI](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pytorch/pytorch-text-sentiment-classification-custom-train-deploy.ipynb)
+
+```
+Learn to build, train, tune and deploy a PyTorch model on [Vertex AI](https://cloud.
+
+The steps performed include:
+
+- Create training package for the text classification model.
+- Train the model with custom training on Vertex AI.
+- Check the created model artifacts.
+- Create a custom container for predictions.
+- Deploy the trained model to a Vertex AI Endpoint using the custom container for predictions.
+- Send online prediction requests to the deployed model and validate.
+- Clean up the resources created in this notebook.
+
+```
+
+   Learn more about [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training).
+

notebooks/official/pytorch/pytorch-text-sentiment-classification-custom-train-deploy.ipynb

@@ -33,18 +33,18 @@
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/training/pytorch-text-sentiment-classification-custom-train-deploy.ipynb\">\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pytorch/pytorch-text-sentiment-classification-custom-train-deploy.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/training/pytorch-text-sentiment-classification-custom-train-deploy.ipynb\">\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pytorch/pytorch-text-sentiment-classification-custom-train-deploy.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "<a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/training/pytorch-text-sentiment-classification-custom-train-deploy.ipynb\" target='_blank'>\n",
+        "<a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/pytorch/pytorch-text-sentiment-classification-custom-train-deploy.ipynb\" target='_blank'>\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "     </a>\n",
@@ -65,7 +65,7 @@
         "\n",
         "You can find more details about the model at [Hugging Face Hub](https://huggingface.co/bert-base-cased). For more notebooks with the state of the art PyTorch/Tensorflow/JAX, you can explore [Hugging FaceNotebooks](https://huggingface.co/transformers/notebooks.html).\n",
         "\n",
-        "Learn more about [Custom training](https://cloud.google.com/vertex-ai/docs/training/custom-training)."
+        "Learn more about [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training)."
       ]
     },
     {
@@ -1301,26 +1301,12 @@
         "- `machine_type`: Mahcine type on which the job needs to run.\n",
         "- `accelerator_type`: Hardware accelerator type for running the job. One of _ACCELERATOR_TYPE_UNSPECIFIED_,\n",
         "        _NVIDIA_TESLA_K80_, _NVIDIA_TESLA_P100_, _NVIDIA_TESLA_V100_, _NVIDIA_TESLA_P4_,\n",
-        "        _NVIDIA_TESLA_T4_, _NVIDIA_TELSA_A100_\n",
+        "        _NVIDIA_TESLA_T4_.\n",
         "- `accelerator_count`: The number of accelerators to attach to a worker replica.\n",
         "- `replica_count`: The number of worker replicas.\n",
         "- `args`: Command line arguments to be passed to the Python script.\n",
         "\n",
-        "Learn more about Vertex AI's [Custom Python-Package Trainining](https://cloud.google.com/python/docs/reference/aiplatform/latest/google.cloud.aiplatform.CustomPythonPackageTrainingJob).\n",
-        "\n",
-        "*Note*: This training job may take over 24 hours."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "b3709beb73e2"
-      },
-      "outputs": [],
-      "source": [
-        "if os.getenv(\"IS_TESTING\"):\n",
-        "    sys.exit(0)"
+        "Learn more about Vertex AI's [Custom Python-Package Trainining](https://cloud.google.com/python/docs/reference/aiplatform/latest/google.cloud.aiplatform.CustomPythonPackageTrainingJob)."
       ]
     },
     {

notebooks/official/sdk/SDK_AutoML_Video_Classification.ipynb

@@ -64,7 +64,7 @@
         "\n",
         "Note: you may incur charges for training, prediction, storage or usage of other GCP products in connection with testing this SDK.\n",
         "\n",
-        "Learn more about [Classification for video data](https://cloud.google.com/vertex-ai/docs/training-overview#classification_for_videos)."
+        "Learn more about [AutoML Video](https://cloud.google.com/vertex-ai/docs/tutorials/video-classification-automl/training)."
       ]
     },
     {
@@ -773,19 +773,7 @@
         "\n",
         "The `run` method when completed returns the `Model` resource.\n",
         "\n",
-        "The execution of the training pipeline can take over 24 hours to complete."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "8f135100f5d9"
-      },
-      "outputs": [],
-      "source": [
-        "if os.getenv(\"IS_TESTING\"):\n",
-        "    sys.exit(0)"
+        "The execution of the training pipeline can take over 2 hours to complete."
       ]
     },
     {

notebooks/official/sdk/SDK_Custom_Training_Python_Package_Managed_Text_Dataset_Tensorflow_Serving_Container.ipynb

@@ -64,7 +64,7 @@
         "\n",
         "Note: You may incur charges for training, prediction, storage or usage of other GCP products in connection with testing this SDK.\n",
         "\n",
-        "Learn more about [Custom training](https://cloud.google.com/vertex-ai/docs/training/custom-training)."
+        "Learn more about [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training)."
       ]
     },
     {

notebooks/official/tabnet/README.md

@@ -12,7 +12,9 @@ The steps performed are:
 
 ```
 
-   Learn more about [Tabular Workflow for TabNet](https://cloud.google.com/vertex-ai/docs/tabular-data/tabular-workflows/tabnet).
+   Learn more about [Vertex AI TabNet](https://cloud.google.com/vertex-ai/docs/tabular-data/tabular-workflows/tabnet).
+
+   Learn more about [Vertex Explainable AI](https://cloud.google.com/vertex-ai/docs/explainable-ai/overview).
 
 
 [Vertex AI TabNet](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/tabnet/tabnet_vertex_tutorial.ipynb)
@@ -30,5 +32,7 @@ The steps performed are:
 
 ```
 
-   Learn more about [Tabular Workflow for TabNet](https://cloud.google.com/vertex-ai/docs/tabular-data/tabular-workflows/tabnet).
+   Learn more about [Vertex AI TabNet](https://cloud.google.com/vertex-ai/docs/tabular-data/tabular-workflows/tabnet).
+
+   Learn more about [Vertex AI Hyperparameter Tuning](https://cloud.google.com/vertex-ai/docs/training/hyperparameter-tuning-overview).
 

notebooks/official/tabnet/ai-explanations-tabnet-algorithm.ipynb

@@ -29,7 +29,7 @@
         "id": "JAPoU8Sm5E6e"
       },
       "source": [
-        "# Vertex AI Explanations with TabNet models\n",
+        "# Vertex AI Explainations with TabNet models\n",
         "\n",
         "<table align=\"left\">\n",
         "\n",
@@ -65,7 +65,7 @@
         "\n",
         "The goal of the tutorial is to provide a sample plotting tool to visualize the output of TabNet, which is helpful in explaining the algorithm.\n",
         "\n",
-        "Learn more about [Tabular Workflow for TabNet](https://cloud.google.com/vertex-ai/docs/tabular-data/tabular-workflows/tabnet)."
+        "Learn more about [Vertex AI TabNet](https://cloud.google.com/vertex-ai/docs/tabular-data/tabular-workflows/tabnet) and [Vertex Explainable AI](https://cloud.google.com/vertex-ai/docs/explainable-ai/overview)."
       ]
     },
     {
@@ -86,7 +86,7 @@
         "- TabNet builtin algorithm\n",
         "\n",
         "The steps performed are:\n",
-        "* Setup the project.\n",
+        "* Setup the the project.\n",
         "* Download the prediction data of pretrain model onf Syn2 data.\n",
         "* Visualize and understand the feature importance based on the masks output.\n",
         "* Clean up the resource created by this tutorial."