port: KFP2

notebooks/official/pipelines/pipelines_intro_kfp.ipynb

@@ -127,9 +127,9 @@
       "source": [
         "# Install the packages\n",
         "! pip3 install --upgrade google-cloud-aiplatform \\\n",
-        "                        google-cloud-storage \\\n",
-        "                        'kfp<2' \\\n",
-        "                        'google-cloud-pipeline-components<2'"
+        "                         google-cloud-storage \\\n",
+        "                         kfp \\\n",
+        "                         google-cloud-pipeline-components"
       ]
     },
     {
@@ -326,7 +326,7 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_URI = \"gs://your-bucket-name-unique\"  # @param {type:\"string\"}"
+        "BUCKET_URI = f\"gs://your-bucket-name-{PROJECT_ID}-unique\"  # @param {type:\"string\"}"
       ]
     },
     {
@@ -364,11 +364,40 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "b556df542518"
+        "id": "77b01a1fdbb4"
       },
       "outputs": [],
       "source": [
-        "SERVICE_ACCOUNT = \"[your-service-account@developer.gserviceaccount.com]\""
+        "SERVICE_ACCOUNT = \"[your-service-account]\"  # @param {type:\"string\"}"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "f936bebda2d4"
+      },
+      "outputs": [],
+      "source": [
+        "import sys\n",
+        "\n",
+        "IS_COLAB = \"google.colab\" in sys.modules\n",
+        "if (\n",
+        "    SERVICE_ACCOUNT == \"\"\n",
+        "    or SERVICE_ACCOUNT is None\n",
+        "    or SERVICE_ACCOUNT == \"[your-service-account]\"\n",
+        "):\n",
+        "    # Get your service account from gcloud\n",
+        "    if not IS_COLAB:\n",
+        "        shell_output = !gcloud auth list 2>/dev/null\n",
+        "        SERVICE_ACCOUNT = shell_output[2].replace(\"*\", \"\").strip()\n",
+        "\n",
+        "    else:  # IS_COLAB:\n",
+        "        shell_output = ! gcloud projects describe  $PROJECT_ID\n",
+        "        project_number = shell_output[-1].split(\":\")[1].strip().replace(\"'\", \"\")\n",
+        "        SERVICE_ACCOUNT = f\"{project_number}-compute@developer.gserviceaccount.com\"\n",
+        "\n",
+        "    print(\"Service Account:\", SERVICE_ACCOUNT)"
       ]
     },
     {
@@ -407,9 +436,8 @@
         "from typing import NamedTuple\n",
         "\n",
         "import google.cloud.aiplatform as aip\n",
-        "from kfp import dsl\n",
-        "from kfp.v2 import compiler\n",
-        "from kfp.v2.dsl import component"
+        "from kfp import compiler, dsl\n",
+        "from kfp.dsl import component"
       ]
     },
     {
@@ -506,10 +534,13 @@
       },
       "outputs": [],
       "source": [
-        "@component(output_component_file=\"hw.yaml\", base_image=\"python:3.9\")\n",
+        "@component(base_image=\"python:3.9\")\n",
         "def hello_world(text: str) -> str:\n",
         "    print(text)\n",
-        "    return text"
+        "    return text\n",
+        "\n",
+        "\n",
+        "compiler.Compiler().compile(hello_world, \"hw.yaml\")"
       ]
     },
     {
@@ -628,12 +659,12 @@
         "    pipeline_root=PIPELINE_ROOT,\n",
         ")\n",
         "def pipeline(text: str = \"hi there\"):\n",
-        "    hw_task = hello_world(text)\n",
-        "    two_outputs_task = two_outputs(text)\n",
+        "    hw_task = hello_world(text=text)\n",
+        "    two_outputs_task = two_outputs(text=text)\n",
         "    consumer_task = consumer(  # noqa: F841\n",
-        "        hw_task.output,\n",
-        "        two_outputs_task.outputs[\"output_one\"],\n",
-        "        two_outputs_task.outputs[\"output_two\"],\n",
+        "        text1=hw_task.output,\n",
+        "        text2=two_outputs_task.outputs[\"output_one\"],\n",
+        "        text3=two_outputs_task.outputs[\"output_two\"],\n",
         "    )"
       ]
     },
@@ -656,9 +687,7 @@
       },
       "outputs": [],
       "source": [
-        "from kfp.v2 import compiler  # noqa: F811\n",
-        "\n",
-        "compiler.Compiler().compile(pipeline_func=pipeline, package_path=\"intro_pipeline.json\")"
+        "compiler.Compiler().compile(pipeline_func=pipeline, package_path=\"intro_pipeline.yaml\")"
       ]
     },
     {
@@ -684,7 +713,7 @@
         "\n",
         "job = aip.PipelineJob(\n",
         "    display_name=DISPLAY_NAME,\n",
-        "    template_path=\"intro_pipeline.json\",\n",
+        "    template_path=\"intro_pipeline.yaml\",\n",
         "    pipeline_root=PIPELINE_ROOT,\n",
         ")\n",
         "\n",
@@ -762,7 +791,7 @@
         "\n",
         "if not os.getenv(\"IS_TESTING\"):\n",
         "    response = api_client.create_run_from_job_spec(\n",
-        "        job_spec_path=\"intro_pipeline.json\",\n",
+        "        job_spec_path=\"intro_pipeline.yaml\",\n",
         "        pipeline_root=PIPELINE_ROOT,\n",
         "        service_account=SERVICE_ACCOUNT,  # <-- CHANGE to use non-default service account\n",
         "    )"
@@ -791,7 +820,7 @@
       "source": [
         "job = aip.PipelineJob(\n",
         "    display_name=\"intro_pipeline_job_cached_unique\",\n",
-        "    template_path=\"intro_pipeline.json\",\n",
+        "    template_path=\"intro_pipeline.yaml\",\n",
         "    enable_caching=False,\n",
         ")\n",
         "\n",
@@ -854,7 +883,7 @@
       },
       "outputs": [],
       "source": [
-        "! curl -X GET -H \"Authorization: Bearer $(gcloud auth print-access-token)\"   -H \"Content-Type: application/json\"   https://{API_ENDPOINT}/v1beta1/projects/{PROJECT_ID}/locations/{REGION}/pipelineJobs"
+        "! curl -X GET -H \"Authorization: Bearer $(gcloud auth print-access-token)\"   -H \"Content-Type: application/yaml\"   https://{API_ENDPOINT}/v1beta1/projects/{PROJECT_ID}/locations/{REGION}/pipelineJobs"
       ]
     },
     {
@@ -876,7 +905,7 @@
       },
       "outputs": [],
       "source": [
-        "output = ! curl -X POST  -H \"Authorization: Bearer $(gcloud auth print-access-token)\" -H \"Content-Type: application/json; charset=utf-8\"   https://{API_ENDPOINT}/v1beta1/projects/{PROJECT_ID}/locations/{REGION}/pipelineJobs  --data \"@intro_pipeline.json\"\n",
+        "output = ! curl -X POST  -H \"Authorization: Bearer $(gcloud auth print-access-token)\" -H \"Content-Type: application/yaml; charset=utf-8\"   https://{API_ENDPOINT}/v1beta1/projects/{PROJECT_ID}/locations/{REGION}/pipelineJobs  --data \"@intro_pipeline.yaml\"\n",
         "\n",
         "PIPELINE_RUN_ID = output[5].split(\"/\")[-1].split('\"')[0]\n",
         "print(output)"
@@ -899,7 +928,7 @@
       },
       "outputs": [],
       "source": [
-        "! curl -X GET -H \"Authorization: Bearer $(gcloud auth print-access-token)\"   -H \"Content-Type: application/json\"   https://{API_ENDPOINT}/v1beta1/projects/{PROJECT_ID}/locations/{REGION}/pipelineJobs/{PIPELINE_RUN_ID}"
+        "! curl -X GET -H \"Authorization: Bearer $(gcloud auth print-access-token)\"   -H \"Content-Type: application/yaml\"   https://{API_ENDPOINT}/v1beta1/projects/{PROJECT_ID}/locations/{REGION}/pipelineJobs/{PIPELINE_RUN_ID}"
       ]
     },
     {
@@ -919,7 +948,7 @@
       },
       "outputs": [],
       "source": [
-        "! curl -X POST -H \"Authorization: Bearer $(gcloud auth print-access-token)\"   -H \"Content-Type: application/json\"   https://{API_ENDPOINT}/v1beta1/projects/{PROJECT_ID}/locations/{REGION}/pipelineJobs/{PIPELINE_RUN_ID}:cancel"
+        "! curl -X POST -H \"Authorization: Bearer $(gcloud auth print-access-token)\"   -H \"Content-Type: application/yaml\"   https://{API_ENDPOINT}/v1beta1/projects/{PROJECT_ID}/locations/{REGION}/pipelineJobs/{PIPELINE_RUN_ID}:cancel"
       ]
     },
     {
@@ -939,7 +968,7 @@
       },
       "outputs": [],
       "source": [
-        "! curl -X DELETE -H \"Authorization: Bearer $(gcloud auth print-access-token)\"   -H \"Content-Type: application/json\"   https://{API_ENDPOINT}/v1beta1/projects/{PROJECT_ID}/locations/{REGION}/pipelineJobs/{PIPELINE_RUN_ID}"
+        "! curl -X DELETE -H \"Authorization: Bearer $(gcloud auth print-access-token)\"   -H \"Content-Type: application/yaml\"   https://{API_ENDPOINT}/v1beta1/projects/{PROJECT_ID}/locations/{REGION}/pipelineJobs/{PIPELINE_RUN_ID}"
       ]
     },
     {