Merge branch 'main' into ivanmkc--add-default-kernel

* upgrade: notebook standard

* upgrade: notebook standard

* fix: formatting

.cloud-build/execute_notebook_helper.py

@@ -58,6 +58,7 @@ def execute_notebook(
             output_path=notebook_source,
             progress_bar=should_log_output,
             request_save_on_cell_execute=should_log_output,
+            kernel_name="python3",
             log_output=should_log_output,
             stdout_file=sys.stdout if should_log_output else None,
             stderr_file=sys.stderr if should_log_output else None,

notebooks/community/ml_ops/stage1/README.md

@@ -88,6 +88,17 @@ The steps performed include:
 - Cancel a data labeling job.
 ```
 
+[Get Started with Vision API and Vertex AI Datasets](get_started_with_visionapi_and_vertex_datasets.ipynb)
+
+```
+The steps performed include:
+
+- Using Vision API to perform Optical Character Recognition (OCR) to extract text from PDF files.
+- Processing the results and saving them to text files.
+- Generating a Vertex AI Dataset import file.
+- Creating a new unlabelled text entity extraction Vertex AI Dataset resource in Vertex AI.
+```
+
 ### E2E Stage Example
 
 [Stage 1: Data Management](mlops_data_management.ipynb)

notebooks/official/automl/automl-text-classification.ipynb

@@ -43,7 +43,7 @@
         "    </a>\n",
         "  </td>\n",
         " <td>\n",
-        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/main/notebooks/official/automl/automl-text-classification.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/ai/platform/notebooks/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/automl/automl-text-classification.ipynb\">\n",
         "        <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
@@ -61,7 +61,7 @@
         "\n",
         "## Overview\n",
         "\n",
-        "This notebook walks you through the major phases of building and using a text classification model on [Vertex AI](https://cloud.google.com/vertex-ai/docs/). \n",
+        "This notebook walks you through the major phases of building and using an AutoML text classification model on [Vertex AI](https://cloud.google.com/vertex-ai/docs/). \n",
         "\n",
         "### Dataset\n",
         "\n",
@@ -74,7 +74,7 @@
         "This tutorial uses the following Google Cloud ML services:\n",
         "\n",
         "- `AutoML Training`\n",
-        "- `Vertex AI Model resource`\n",
+        "- `Vertex AI Model` resource\n",
         "\n",
         "The steps performed include:\n",
         "\n",
@@ -192,7 +192,7 @@
         "if IS_WORKBENCH_NOTEBOOK:\n",
         "    USER_FLAG = \"--user\"\n",
         "\n",
-        "! pip3 install {USER_FLAG} --upgrade google-cloud-aiplatform google-cloud-storage jsonlines"
+        "! pip3 install {USER_FLAG} --upgrade google-cloud-aiplatform google-cloud-storage jsonlines -q"
       ]
     },
     {
@@ -536,7 +536,7 @@
         "id": "32c971919605"
       },
       "source": [
-        "## Create a dataset and import your data\n",
+        "## Create a `Dataset` resource and import your data\n",
         "\n",
         "The notebook uses the 'Happy Moments' dataset for demonstration purposes. You can change it to another text classification dataset that [conforms to the data preparation requirements](https://cloud.google.com/vertex-ai/docs/datasets/prepare-text#classification).\n",
         "\n",
@@ -584,7 +584,7 @@
       "source": [
         "## Train your text classification model\n",
         "\n",
-        "Once your dataset has finished importing data, you are ready to train your model. To do this, you first need the full resource name of your dataset, where the full name has the format `projects/[YOUR_PROJECT]/locations/us-central1/datasets/[YOUR_DATASET_ID]`. If you don't have the resource name handy, you can list all of the datasets in your project using `TextDataset.list()`. \n",
+        "Once your dataset has finished importing data, you are ready to train your model. To do this, you first need the full resource name of your dataset, where the full name has the format `projects/[YOUR_PROJECT]/locations/[YOUR_REGIO)N]/datasets/[YOUR_DATASET_ID]`. If you don't have the resource name handy, you can list all of the datasets in your project using `TextDataset.list()`. \n",
         "\n",
         "As shown in the following code block, you can pass in the display name of your dataset in the call to `list()` to filter the results.\n"
       ]

notebooks/official/pipelines/pipelines_intro_kfp.ipynb

@@ -33,18 +33,18 @@
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/official/pipelines/pipelines_intro_kfp.ipynb\">\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/pipelines_intro_kfp.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/official/pipelines/pipelines_intro_kfp.ipynb\">\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/pipelines_intro_kfp.ipynb\">\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/official/pipelines/pipelines_intro_kfp.ipynb\">\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/pipelines_intro_kfp.ipynb\">\n",
         "        <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
@@ -72,9 +72,15 @@
       "source": [
         "### Objective\n",
         "\n",
+        "In this tutorial, you use the KFP SDK to build pipelines that generate evaluation metrics.\n",
+        "\n",
+        "This tutorial uses the following Google Cloud ML services:\n",
+        "\n",
+        "- `Vertex AI Pipelines`\n",
+        "\n",
         "The steps performed include:\n",
         "\n",
-        "- Define and compile a pipeline.\n",
+        "- Define and compile a `Vertex AI` pipeline.\n",
         "- Schedule a recurring pipeline run.\n",
         "- Specify which service account to use for a pipeline run."
       ]
@@ -111,7 +117,7 @@
       "source": [
         "### Set up your local development environment\n",
         "\n",
-        "If you are using Colab or Google Cloud Notebook, your environment already meets all the requirements to run this notebook. You can skip this step.\n",
+        "If you are using Colab or Vertex AI Workbench Notebook, your environment already meets all the requirements to run this notebook. You can skip this step.\n",
         "\n",
         "Otherwise, make sure your environment meets this notebook's requirements. You need the following:\n",
         "\n",
@@ -144,7 +150,7 @@
       "source": [
         "## Installation\n",
         "\n",
-        "Install the latest version of Vertex AI SDK for Python."
+        "Install the packages required for executing this notebook."
       ]
     },
     {
@@ -157,53 +163,20 @@
       "source": [
         "import os\n",
         "\n",
-        "# Google Cloud Notebook\n",
-        "if os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
-        "    USER_FLAG = \"--user\"\n",
-        "else:\n",
-        "    USER_FLAG = \"\"\n",
+        "# The Vertex AI Workbench Notebook product has specific requirements\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
+        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
+        "    \"/opt/deeplearning/metadata/env_version\"\n",
+        ")\n",
         "\n",
-        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "install_storage"
-      },
-      "source": [
-        "Install the latest GA version of *google-cloud-storage* library as well."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "Q9cY6x132Ouw"
-      },
-      "outputs": [],
-      "source": [
-        "! pip3 install -U google-cloud-storage $USER_FLAG"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "install_kfp"
-      },
-      "source": [
-        "Install the latest GA version of *KFP SDK* library as well."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "nbULuPjF2Oux"
-      },
-      "outputs": [],
-      "source": [
-        "! pip3 install $USER kfp --upgrade"
+        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
+        "USER_FLAG = \"\"\n",
+        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\"\n",
+        "\n",
+        "! pip3 install --upgrade google-cloud-aiplatform {USER_FLAG} -q\n",
+        "! pip3 install -U google-cloud-storage {USER_FLAG} -q\n",
+        "! pip3 install {USER_FLAG} kfp google-cloud-pipeline-components --upgrade -q"
       ]
     },
     {
@@ -397,23 +370,30 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Google Cloud Notebook**, your environment is already authenticated. Skip this step.\n",
+        "**If you are using Vertex AI Workbench Notebook**, your environment is already authenticated. Skip this step.\n",
         "\n",
-        "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
+        "**If you are using Colab**, run the cell below and follow the instructions\n",
+        "when prompted to authenticate your account via oAuth.\n",
         "\n",
         "**Otherwise**, follow these steps:\n",
         "\n",
-        "In the Cloud Console, go to the [Create service account key](https://console.cloud.google.com/apis/credentials/serviceaccountkey) page.\n",
+        "1. In the Cloud Console, go to the [**Create service account key**\n",
+        "   page](https://console.cloud.google.com/apis/credentials/serviceaccountkey).\n",
         "\n",
-        "**Click Create service account**.\n",
+        "2. Click **Create service account**.\n",
         "\n",
-        "In the **Service account name** field, enter a name, and click **Create**.\n",
+        "3. In the **Service account name** field, enter a name, and\n",
+        "   click **Create**.\n",
         "\n",
-        "In the **Grant this service account access to project** section, click the Role drop-down list. Type \"Vertex\" into the filter box, and select **Vertex Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
+        "4. In the **Grant this service account access to project** section, click the **Role** drop-down list. Type \"Vertex AI\"\n",
+        "into the filter box, and select\n",
+        "   **Vertex AI Administrator**. Type \"Storage Object Admin\" into the filter box, and select **Storage Object Admin**.\n",
         "\n",
-        "Click Create. A JSON file that contains your key downloads to your local environment.\n",
+        "5. Click *Create*. A JSON file that contains your key downloads to your\n",
+        "local environment.\n",
         "\n",
-        "Enter the path to your service account key as the GOOGLE_APPLICATION_CREDENTIALS variable in the cell below and run the cell."
+        "6. Enter the path to your service account key as the\n",
+        "`GOOGLE_APPLICATION_CREDENTIALS` variable in the cell below and run the cell."
       ]
     },
     {
@@ -432,8 +412,11 @@
         "import os\n",
         "import sys\n",
         "\n",
-        "# If on Google Cloud Notebook, then don't execute this code\n",
-        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
+        "# If on Vertex AI Workbench, then don't execute this code\n",
+        "IS_COLAB = \"google.colab\" in sys.modules\n",
+        "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\") and not os.getenv(\n",
+        "    \"DL_ANACONDA_HOME\"\n",
+        "):\n",
         "    if \"google.colab\" in sys.modules:\n",
         "        from google.colab import auth as google_auth\n",
         "\n",
@@ -481,8 +464,9 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
+        "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
       ]
     },
     {
@@ -560,12 +544,15 @@
         "    or SERVICE_ACCOUNT is None\n",
         "    or SERVICE_ACCOUNT == \"[your-service-account]\"\n",
         "):\n",
+        "    # Get your service account from gcloud\n",
+        "    if not IS_COLAB:\n",
+        "        shell_output = !gcloud auth list 2>/dev/null\n",
+        "        SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
         "\n",
-        "    # Get your GCP project id from gcloud\n",
-        "\n",
-        "    shell_output = !gcloud auth list 2>/dev/null\n",
-        "\n",
-        "    SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
+        "    if IS_COLAB:\n",
+        "        shell_output = ! gcloud projects describe  $PROJECT_ID\n",
+        "        project_number = shell_output[-1].split(\":\")[1].strip().replace(\"'\", \"\")\n",
+        "        SERVICE_ACCOUNT = f\"{project_number}-compute@developer.gserviceaccount.com\"\n",
         "\n",
         "    print(\"Service Account:\", SERVICE_ACCOUNT)"
       ]
@@ -614,7 +601,12 @@
       },
       "outputs": [],
       "source": [
-        "import google.cloud.aiplatform as aip"
+        "from typing import NamedTuple\n",
+        "\n",
+        "import google.cloud.aiplatform as aip\n",
+        "from kfp import dsl\n",
+        "from kfp.v2 import compiler\n",
+        "from kfp.v2.dsl import component"
       ]
     },
     {
@@ -664,30 +656,6 @@
         "PIPELINE_ROOT = \"{}/pipeline_root/intro\".format(BUCKET_URI)"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "additional_imports"
-      },
-      "source": [
-        "Additional imports."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "import_kfp:namedtuple"
-      },
-      "outputs": [],
-      "source": [
-        "from typing import NamedTuple\n",
-        "\n",
-        "from kfp import dsl\n",
-        "from kfp.v2 import compiler\n",
-        "from kfp.v2.dsl import component"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -1267,7 +1235,7 @@
         "except Exception as e:\n",
         "    print(e)\n",
         "\n",
-        "if delete_bucket and \"BUCKET_URI\" in globals():\n",
+        "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
         "    ! gsutil rm -r $BUCKET_URI"
       ]
     }