feat: replace with Pipeline.run()

2026-09-26 14:42:04 +00:00 · 2021-09-01 15:20:50 +00:00 · 2021-09-01 15:17:36 +00:00 · 2021-09-01 03:52:58 +00:00 · 2021-09-01 03:50:28 +00:00 · 2021-09-01 03:48:06 +00:00
9 changed files with 144 additions and 131 deletions
@@ -686,8 +686,7 @@
        "from google_cloud_pipeline_components import aiplatform as gcc_aip\n",
        "from kfp.v2 import dsl\n",
        "from kfp.v2.dsl import (ClassificationMetrics, Input, Metrics, Model, Output,\n",
-        "                        component)\n",
-        "from kfp.v2.google.client import AIPlatformClient"
+        "                        component)"
      ]
    },
    {
@@ -991,15 +990,16 @@
      },
      "outputs": [],
      "source": [
-        "from kfp.v2.google.client import AIPlatformClient  # noqa: F811\n",
+        "DISPLAY_NAME = \"beans_\" + TIMESTAMP\n",
        "\n",
-        "api_client = AIPlatformClient(project_id=PROJECT_ID, region=REGION)\n",
-        "\n",
-        "response = api_client.create_run_from_job_spec(\n",
-        "    job_spec_path=\"tabular classification_pipeline.json\".replace(\" \", \"_\"),\n",
+        "job = aip.PipelineJob(\n",
+        "    display_name=DISPLAY_NAME,\n",
+        "    template_path=\"tabular classification_pipeline.json\".replace(\" \", \"_\"),\n",
        "    pipeline_root=PIPELINE_ROOT,\n",
        "    parameter_values={\"project\": PROJECT_ID, \"display_name\": DISPLAY_NAME},\n",
-        ")"
+        ")\n",
+        "\n",
+        "job.run()"
      ]
    },
    {
@@ -127,6 +127,56 @@
        "6. Open this notebook in the Jupyter Notebook Dashboard.\n"
      ]
    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "install_aip:mbsdk"
+      },
+      "source": [
+        "## Installation\n",
+        "\n",
+        "Install the latest version of Vertex SDK for Python."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "install_aip:mbsdk"
+      },
+      "outputs": [],
+      "source": [
+        "import os\n",
+        "\n",
+        "# Google Cloud Notebook\n",
+        "if os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n",
+        "    USER_FLAG = \"--user\"\n",
+        "else:\n",
+        "    USER_FLAG = \"\"\n",
+        "\n",
+        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "install_storage"
+      },
+      "source": [
+        "Install the latest GA version of *google-cloud-storage* library as well."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "install_storage"
+      },
+      "outputs": [],
+      "source": [
+        "! pip3 install -U google-cloud-storage $USER_FLAG"
+      ]
+    },
    {
      "cell_type": "markdown",
      "metadata": {
@@ -584,8 +634,7 @@
        "\n",
        "from kfp import dsl\n",
        "from kfp.v2 import compiler\n",
-        "from kfp.v2.dsl import component\n",
-        "from kfp.v2.google.client import AIPlatformClient"
+        "from kfp.v2.dsl import component"
      ]
    },
    {
@@ -783,13 +832,15 @@
      },
      "outputs": [],
      "source": [
-        "from kfp.v2.google.client import AIPlatformClient  # noqa: F811\n",
+        "DISPLAY_NAME = \"control_\" + TIMESTAMP\n",
        "\n",
-        "api_client = AIPlatformClient(project_id=PROJECT_ID, region=REGION)\n",
+        "job = aip.PipelineJob(\n",
+        "    display_name=DISPLAY_NAME,\n",
+        "    template_path=\"control_pipeline.json\".replace(\" \", \"_\"),\n",
+        "    pipeline_root=PIPELINE_ROOT,\n",
+        ")\n",
        "\n",
-        "response = api_client.create_run_from_job_spec(\n",
-        "    job_spec_path=\"control_pipeline.json\".replace(\" \", \"_\"), pipeline_root=PIPELINE_ROOT\n",
-        ")"
+        "job.run()"
      ]
    },
    {
@@ -672,13 +672,12 @@
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
-        "id": "import_pipelines:gcpc"
+        "id": "import_pipelines:min"
      },
      "outputs": [],
      "source": [
        "import kfp\n",
-        "from google_cloud_pipeline_components import aiplatform as gcc_aip\n",
-        "from kfp.v2.google.client import AIPlatformClient"
+        "from google_cloud_pipeline_components import aiplatform as gcc_aip"
      ]
    },
    {
@@ -797,14 +796,15 @@
      },
      "outputs": [],
      "source": [
-        "from kfp.v2.google.client import AIPlatformClient  # noqa: F811\n",
+        "DISPLAY_NAME = \"flowers_\" + TIMESTAMP\n",
        "\n",
-        "api_client = AIPlatformClient(project_id=PROJECT_ID, region=REGION)\n",
-        "\n",
-        "response = api_client.create_run_from_job_spec(\n",
-        "    job_spec_path=\"image classification_pipeline.json\".replace(\" \", \"_\"),\n",
+        "job = aip.PipelineJob(\n",
+        "    display_name=DISPLAY_NAME,\n",
+        "    template_path=\"image classification_pipeline.json\".replace(\" \", \"_\"),\n",
        "    pipeline_root=PIPELINE_ROOT,\n",
-        ")"
+        ")\n",
+        "\n",
+        "job.run()"
      ]
    },
    {
@@ -679,8 +679,7 @@
      "outputs": [],
      "source": [
        "import kfp\n",
-        "from google_cloud_pipeline_components import aiplatform as gcc_aip\n",
-        "from kfp.v2.google.client import AIPlatformClient"
+        "from google_cloud_pipeline_components import aiplatform as gcc_aip"
      ]
    },
    {
@@ -812,14 +811,15 @@
      },
      "outputs": [],
      "source": [
-        "from kfp.v2.google.client import AIPlatformClient  # noqa: F811\n",
+        "DISPLAY_NAME = \"cal_housing_\" + TIMESTAMP\n",
        "\n",
-        "api_client = AIPlatformClient(project_id=PROJECT_ID, region=REGION)\n",
-        "\n",
-        "response = api_client.create_run_from_job_spec(\n",
-        "    job_spec_path=\"tabular regression_pipeline.json\".replace(\" \", \"_\"),\n",
+        "job = aip.PipelineJob(\n",
+        "    display_name=DISPLAY_NAME,\n",
+        "    template_path=\"tabular regression_pipeline.json\".replace(\" \", \"_\"),\n",
        "    pipeline_root=PIPELINE_ROOT,\n",
-        ")"
+        ")\n",
+        "\n",
+        "job.run()"
      ]
    },
    {
@@ -677,8 +677,7 @@
      "outputs": [],
      "source": [
        "import kfp\n",
-        "from google_cloud_pipeline_components import aiplatform as gcc_aip\n",
-        "from kfp.v2.google.client import AIPlatformClient"
+        "from google_cloud_pipeline_components import aiplatform as gcc_aip"
      ]
    },
    {
@@ -800,14 +799,15 @@
      },
      "outputs": [],
      "source": [
-        "from kfp.v2.google.client import AIPlatformClient  # noqa: F811\n",
+        "DISPLAY_NAME = \"happydb_\" + TIMESTAMP\n",
        "\n",
-        "api_client = AIPlatformClient(project_id=PROJECT_ID, region=REGION)\n",
-        "\n",
-        "response = api_client.create_run_from_job_spec(\n",
-        "    job_spec_path=\"text classification_pipeline.json\".replace(\" \", \"_\"),\n",
+        "job = aip.PipelineJob(\n",
+        "    display_name=DISPLAY_NAME,\n",
+        "    template_path=\"text classification_pipeline.json\".replace(\" \", \"_\"),\n",
        "    pipeline_root=PIPELINE_ROOT,\n",
-        ")"
+        ")\n",
+        "\n",
+        "job.run()"
      ]
    },
    {
@@ -617,31 +617,6 @@
        "import google.cloud.aiplatform as aip"
      ]
    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "aip_constants:endpoint"
-      },
-      "source": [
-        "#### Vertex AI constants\n",
-        "\n",
-        "Setup up the following constants for Vertex AI:\n",
-        "\n",
-        "- `API_ENDPOINT`: The Vertex AI API service endpoint for `Dataset`, `Model`, `Job`, `Pipeline` and `Endpoint` services."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "aip_constants:endpoint"
-      },
-      "outputs": [],
-      "source": [
-        "# API service endpoint\n",
-        "API_ENDPOINT = \"{}-aiplatform.googleapis.com\".format(REGION)"
-      ]
-    },
    {
      "cell_type": "markdown",
      "metadata": {
@@ -677,15 +652,14 @@
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
-        "id": "import_pipelines:gcpc"
+        "id": "import_pipelines:min"
      },
      "outputs": [],
      "source": [
        "import kfp\n",
        "from google_cloud_pipeline_components import aiplatform as gcc_aip\n",
        "from kfp.v2.dsl import component\n",
-        "from kfp.v2.google import experimental\n",
-        "from kfp.v2.google.client import AIPlatformClient"
+        "from kfp.v2.google import experimental"
      ]
    },
    {
@@ -868,14 +842,15 @@
      },
      "outputs": [],
      "source": [
-        "from kfp.v2.google.client import AIPlatformClient  # noqa: F811\n",
+        "DISPLAY_NAME = \"bikes_weather_\" + TIMESTAMP\n",
        "\n",
-        "api_client = AIPlatformClient(project_id=PROJECT_ID, region=REGION)\n",
-        "\n",
-        "response = api_client.create_run_from_job_spec(\n",
-        "    job_spec_path=\"tabular regression_pipeline.json\".replace(\" \", \"_\"),\n",
+        "job = aip.PipelineJob(\n",
+        "    display_name=DISPLAY_NAME,\n",
+        "    template_path=\"tabular regression_pipeline.json\".replace(\" \", \"_\"),\n",
        "    pipeline_root=PIPELINE_ROOT,\n",
-        ")"
+        ")\n",
+        "\n",
+        "job.run()"
      ]
    },
    {
@@ -624,31 +624,6 @@
        "import google.cloud.aiplatform as aip"
      ]
    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "aip_constants:endpoint"
-      },
-      "source": [
-        "#### Vertex AI constants\n",
-        "\n",
-        "Setup up the following constants for Vertex AI:\n",
-        "\n",
-        "- `API_ENDPOINT`: The Vertex AI API service endpoint for `Dataset`, `Model`, `Job`, `Pipeline` and `Endpoint` services."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "aip_constants:endpoint"
-      },
-      "outputs": [],
-      "source": [
-        "# API service endpoint\n",
-        "API_ENDPOINT = \"{}-aiplatform.googleapis.com\".format(REGION)"
-      ]
-    },
    {
      "cell_type": "markdown",
      "metadata": {
@@ -693,8 +668,7 @@
        "import kfp\n",
        "from kfp.v2 import dsl\n",
        "from kfp.v2.dsl import (Artifact, Dataset, Input, InputPath, Model, Output,\n",
-        "                        OutputPath, component)\n",
-        "from kfp.v2.google.client import AIPlatformClient"
+        "                        OutputPath, component)"
      ]
    },
    {
@@ -991,15 +965,16 @@
      },
      "outputs": [],
      "source": [
-        "from kfp.v2.google.client import AIPlatformClient  # noqa: F811\n",
+        "DISPLAY_NAME = \"shakespeare_\" + TIMESTAMP\n",
        "\n",
-        "api_client = AIPlatformClient(project_id=PROJECT_ID, region=REGION)\n",
-        "\n",
-        "response = api_client.create_run_from_job_spec(\n",
-        "    job_spec_path=\"lightweight_pipeline.json\".replace(\" \", \"_\"),\n",
+        "job = aip.PipelineJob(\n",
+        "    display_name=DISPLAY_NAME,\n",
+        "    template_path=\"lightweight_pipeline.json\".replace(\" \", \"_\"),\n",
        "    pipeline_root=PIPELINE_ROOT,\n",
        "    parameter_values={\"message\": \"Hello, World\"},\n",
-        ")"
+        ")\n",
+        "\n",
+        "job.run()"
      ]
    },
    {
@@ -672,9 +672,8 @@
      },
      "outputs": [],
      "source": [
-        "from kfp import dsl\n",
-        "from kfp.v2.dsl import ClassificationMetrics, Metrics, Output, component\n",
-        "from kfp.v2.google.client import AIPlatformClient"
+        "from kfp.v2 import dsl\n",
+        "from kfp.v2.dsl import ClassificationMetrics, Metrics, Output, component"
      ]
    },
    {
@@ -947,16 +946,17 @@
      },
      "outputs": [],
      "source": [
-        "from kfp.v2.google.client import AIPlatformClient  # noqa: F811\n",
+        "DISPLAY_NAME = \"iris_\" + TIMESTAMP\n",
        "\n",
-        "api_client = AIPlatformClient(project_id=PROJECT_ID, region=REGION)\n",
-        "\n",
-        "response = api_client.create_run_from_job_spec(\n",
-        "    job_spec_path=\"tabular classification_pipeline.json\".replace(\" \", \"_\"),\n",
+        "job = aip.PipelineJob(\n",
+        "    display_name=DISPLAY_NAME,\n",
+        "    template_path=\"tabular classification_pipeline.json\".replace(\" \", \"_\"),\n",
        "    job_id=f\"tabular classification-v2{TIMESTAMP}-1\".replace(\" \", \"\"),\n",
        "    pipeline_root=PIPELINE_ROOT,\n",
        "    parameter_values={\"seed\": 7, \"splits\": 10},\n",
-        ")"
+        ")\n",
+        "\n",
+        "job.run()"
      ]
    },
    {
@@ -995,12 +995,15 @@
      },
      "outputs": [],
      "source": [
-        "response = api_client.create_run_from_job_spec(\n",
-        "    job_spec_path=\"tabular classification_pipeline.json\".replace(\" \", \"_\"),\n",
+        "job = aip.PipelineJob(\n",
+        "    display_name=\"iris_\" + TIMESTAMP,\n",
+        "    template_path=\"tabular classification_pipeline.json\".replace(\" \", \"_\"),\n",
        "    job_id=f\"tabular classification-pipeline-v2{TIMESTAMP}-2\".replace(\" \", \"\"),\n",
        "    pipeline_root=PIPELINE_ROOT,\n",
        "    parameter_values={\"seed\": 5, \"splits\": 7},\n",
-        ")"
+        ")\n",
+        "\n",
+        "job.run()"
      ]
    },
    {
@@ -659,7 +659,7 @@
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
-        "id": "import_kfp"
+        "id": "import_kfp:namedtuple"
      },
      "outputs": [],
      "source": [
@@ -667,8 +667,7 @@
        "\n",
        "from kfp import dsl\n",
        "from kfp.v2 import compiler\n",
-        "from kfp.v2.dsl import component\n",
-        "from kfp.v2.google.client import AIPlatformClient"
+        "from kfp.v2.dsl import component"
      ]
    },
    {
@@ -893,13 +892,15 @@
      },
      "outputs": [],
      "source": [
-        "from kfp.v2.google.client import AIPlatformClient  # noqa: F811\n",
+        "DISPLAY_NAME = \"intro_\" + TIMESTAMP\n",
        "\n",
-        "api_client = AIPlatformClient(project_id=PROJECT_ID, region=REGION)\n",
+        "job = aip.PipelineJob(\n",
+        "    display_name=DISPLAY_NAME,\n",
+        "    template_path=\"intro_pipeline.json\".replace(\" \", \"_\"),\n",
+        "    pipeline_root=PIPELINE_ROOT,\n",
+        ")\n",
        "\n",
-        "response = api_client.create_run_from_job_spec(\n",
-        "    job_spec_path=\"intro_pipeline.json\".replace(\" \", \"_\"), pipeline_root=PIPELINE_ROOT\n",
-        ")"
+        "job.run()"
      ]
    },
    {
@@ -949,6 +950,10 @@
      },
      "outputs": [],
      "source": [
+        "from kfp.v2.google.client import AIPlatformClient  # noqa: F811\n",
+        "\n",
+        "api_client = AIPlatformClient(project_id=PROJECT_ID, region=REGION)\n",
+        "\n",
        "# adjust time zone and cron schedule as necessary\n",
        "response = api_client.create_schedule_from_job_spec(\n",
        "    job_spec_path=\"intro_pipeline.json\",\n",
@@ -1023,7 +1028,7 @@
        "\n",
        "By default, pipeline step caching is enabled. This means that the results of previous step executions are reused when possible.\n",
        "\n",
-        "if you want to disable caching for a pipeline run, you can pass the `enable_caching=False` argument to the `create_run_from_job_spec` function when you submit the pipeline job, as shown below.  Try submitting the example pipeline job again, first with and then without caching enabled."
+        "if you want to disable caching for a pipeline run, you can pass the `enable_caching=False` argument to the `PipelineJob` constructor when you submit the pipeline job, as shown below.  Try submitting the example pipeline job again, first with and then without caching enabled."
      ]
    },
    {
@@ -1034,9 +1039,13 @@
      },
      "outputs": [],
      "source": [
-        "response = api_client.create_run_from_job_spec(\n",
-        "    job_spec_path=\"intro_pipeline.json\", enable_caching=False\n",
-        ")"
+        "job = aip.PipelineJob(\n",
+        "    display_name=\"intro_\" + TIMESTAMP,\n",
+        "    template_path=\"intro_pipeline.json\",\n",
+        "    enable_caching=False,\n",
+        ")\n",
+        "\n",
+        "job.run()"
      ]
    },
    {
Author	SHA1	Message	Date
Andrew Ferlitsch	6f8f6aeb6a	feat: replace with Pipeline.run()	2021-09-01 15:20:50 +00:00
Andrew Ferlitsch	afd1fb560f	feat: replace with Pipeline.run()	2021-09-01 15:17:36 +00:00
Andrew Ferlitsch	98365faf61	feat: replace with Pipeline.run()	2021-09-01 03:52:58 +00:00
Andrew Ferlitsch	1ef43428eb	feat: replace with Pipeline.run()	2021-09-01 03:50:28 +00:00
Andrew Ferlitsch	12699abcba	feat: replace with Pipeline.run()	2021-09-01 03:48:06 +00:00
Andrew Ferlitsch	98265375ec	feat: update to use Vertex Pipeline run	2021-08-31 23:50:01 +00:00
Andrew Ferlitsch	4028efedfd	feat: update to use Vertex Pipeline run	2021-08-31 23:47:50 +00:00
Andrew Ferlitsch	08798d37b8	feat: update to use Vertex Pipeline run	2021-08-31 23:33:18 +00:00
Andrew Ferlitsch	b65ff1f6c7	feat: update to use Vertex Pipeline run	2021-08-31 23:29:54 +00:00
Andrew Ferlitsch	9582ce3a76	feat: update to use Vertex Pipeline run	2021-08-31 22:04:26 +00:00
Andrew Ferlitsch	de48feb3f3	feat: update to use Vertex Pipeline run	2021-08-31 22:00:38 +00:00
Andrew Ferlitsch	8de84bfc43	feat: update to use Vertex Pipeline run	2021-08-31 21:57:50 +00:00
Andrew Ferlitsch	cbcbbf5246	feat: update to use Vertex Pipeline run	2021-08-31 21:55:17 +00:00
Andrew Ferlitsch	6328215e53	feat: update to use Vertex Pipeline run	2021-08-31 21:38:41 +00:00
Andrew Ferlitsch	9a9789d915	feat: update to use Vertex Pipeline run	2021-08-31 21:22:17 +00:00
Andrew Ferlitsch	d6d59b597c	feat: update to use Vertex Pipeline run	2021-08-31 21:20:43 +00:00
Andrew Ferlitsch	d119c25fdc	feat: update to use Vertex Pipeline run	2021-08-31 21:20:06 +00:00
Andrew Ferlitsch	3f89d7ae8f	feat: update to use Vertex Pipeline run	2021-08-31 20:49:50 +00:00
Andrew Ferlitsch	cf6a77aa30	feat: update to use Vertex Pipeline run	2021-08-31 20:49:15 +00:00
Andrew Ferlitsch	034e2f9027	feat: update to use Vertex Pipeline run	2021-08-31 20:47:01 +00:00
Andrew Ferlitsch	f97b26cf0a	feat: update to use Vertex Pipeline run	2021-08-31 20:46:06 +00:00