feat: update to v1

notebooks/community/ml_ops/stage3/mlops_formalization.ipynb

@@ -8,7 +8,7 @@
       },
       "outputs": [],
       "source": [
-        "# Copyright 2021 Google LLC\n",
+        "# Copyright 2022 Google LLC\n",
         "#\n",
         "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
         "# you may not use this file except in compliance with the License.\n",
@@ -197,7 +197,9 @@
         "    ! pip3 install --upgrade cloudml-hypertune $USER_FLAG\n",
         "    ! pip3 install --upgrade kfp $USER_FLAG\n",
         "    ! pip3 install --upgrade torchvision $USER_FLAG\n",
-        "    ! pip3 install --upgrade rpy2 $USER_FLAG"
+        "    ! pip3 install --upgrade rpy2 $USER_FLAG\n",
+        "    ! pip3 install --upgrade python-tabulate $USER_FLAG\n",
+        "    ! pip3 install -U opencv-python-headless==4.5.2.52 $USER_FLAG"
       ]
     },
     {
@@ -543,9 +545,8 @@
       },
       "outputs": [],
       "source": [
-        "from google_cloud_pipeline_components.experimental.dataflow import \\\n",
-        "    DataflowPythonJobOp\n",
-        "from google_cloud_pipeline_components.experimental.wait_gcp_resources import \\\n",
+        "from google_cloud_pipeline_components.v1.dataflow import DataflowPythonJobOp\n",
+        "from google_cloud_pipeline_components.v1.wait_gcp_resources import \\\n",
         "    WaitGcpResourcesOp"
       ]
     },
@@ -1508,7 +1509,8 @@
         "%%writefile requirements.txt\n",
         "apache-beam\n",
         "tensorflow-transform==1.2.0\n",
-        "tensorflow-data-validation==1.2"
+        "tensorflow-data-validation==1.2\n",
+        "future"
       ]
     },
     {
@@ -1535,7 +1537,8 @@
         "\n",
         "REQUIRED_PACKAGES = [\n",
         "    'tensorflow-transform==1.2.0',\n",
-        "    'tensorflow-data-validation==1.2'\n",
+        "    'tensorflow-data-validation==1.2',\n",
+        "    'future'\n",
         "]\n",
         "PACKAGE_NAME = 'my_package'\n",
         "PACKAGE_VERSION = '0.0.1'\n",
@@ -1600,7 +1603,7 @@
       },
       "outputs": [],
       "source": [
-        "@component(packages_to_install=[\"tensorflow\", \"tensorflow-transform\"])\n",
+        "@component(packages_to_install=[\"tensorflow\", \"tensorflow-transform==1.2.0\", \"future\"])\n",
         "def transformed_data_analysis(\n",
         "    metadata_location: str,\n",
         "    transformed_data_prefix: str,\n",
@@ -1683,7 +1686,7 @@
         "    staging_dir: str,\n",
         "    data_bucket: str,\n",
         "    metadata_location: str,\n",
-        "    dataset_labels: str,\n",
+        "    dataset_labels: dict,\n",
         "    year: int,\n",
         "    limit: int,\n",
         "    project: str = PROJECT_ID,\n",
@@ -1714,9 +1717,6 @@
         "        label=label_column,\n",
         "    )\n",
         "\n",
-        "    DataflowPythonJobOp.component_spec.implementation.container.image = (\n",
-        "        \"gcr.io/ml-pipeline/google-cloud-pipeline-components:v0.2.0_dataflow_logs_fix\"\n",
-        "    )\n",
         "    dataflow_python_op = DataflowPythonJobOp(\n",
         "        project=project,\n",
         "        location=region,\n",
@@ -1809,7 +1809,7 @@
         "        \"staging_dir\": PIPELINE_ROOT,\n",
         "        \"data_bucket\": BUCKET_NAME,\n",
         "        \"metadata_location\": BUCKET_NAME + \"/metadata.jsonl\",\n",
-        "        \"dataset_labels\": str({\"user_metadata\": BUCKET_NAME[5:]}).replace(\"'\", '\"'),\n",
+        "        \"dataset_labels\": {\"user_metadata\": BUCKET_NAME[5:]},\n",
         "        \"year\": 2020,\n",
         "        \"limit\": 300000,\n",
         "        \"project\": PROJECT_ID,\n",
@@ -1900,8 +1900,8 @@
         "print(\"transformed-data-analysis\")\n",
         "artifacts = print_pipeline_output(pipeline, \"transformed-data-analysis\")\n",
         "print(\"\\n\")\n",
-        "print(\"tabulardataset-create\")\n",
-        "artifacts = print_pipeline_output(pipeline, \"tabulardataset-create\")\n",
+        "print(\"tabular-dataset-create\")\n",
+        "artifacts = print_pipeline_output(pipeline, \"tabular-dataset-create\")\n",
         "print(\"\\n\")\n",
         "\n",
         "output = !gsutil cat $artifacts\n",
@@ -1960,7 +1960,7 @@
       },
       "outputs": [],
       "source": [
-        "@component(packages_to_install=[\"tensorflow==2.5\", \"tensorflow-transform\"])\n",
+        "@component(packages_to_install=[\"tensorflow==2.5\", \"tensorflow-transform\", \"future\"])\n",
         "def build_model(\n",
         "    dataset_id: str, display_name: str, deploy_image: str, bucket: str, project: str\n",
         ") -> str:\n",
@@ -3049,7 +3049,7 @@
         "):\n",
         "    from google_cloud_pipeline_components import aiplatform as gcc_aip\n",
         "\n",
-        "    with dsl.Condition(warmup == \"True\", name=\"train-model\"):\n",
+        "    with dsl.Condition(warmup == \"True\", name=\"warmup-model\"):\n",
         "\n",
         "        warmup_op = gcc_aip.CustomPythonPackageTrainingJobRunOp(\n",
         "            project=project,\n",
@@ -3066,7 +3066,7 @@
         "            accelerator_count=accelerator_count,\n",
         "        )\n",
         "\n",
-        "    with dsl.Condition(warmup == \"False\", name=\"warmup-model\"):\n",
+        "    with dsl.Condition(warmup == \"False\", name=\"train-model\"):\n",
         "\n",
         "        training_op = gcc_aip.CustomPythonPackageTrainingJobRunOp(\n",
         "            project=project,\n",

notebooks/official/pipelines/custom_model_training_and_batch_prediction.ipynb

@@ -222,7 +222,7 @@
       "outputs": [],
       "source": [
         "if os.getenv(\"IS_TESTING\"):\n",
-        "    ! pip3 install --upgrade --force-reinstall $USER_FLAG tensorflow==2.5 kfp google-cloud-aiplatform google-cloud-storage google-cloud-pipeline-components "
+        "    ! pip3 install --upgrade --force-reinstall $USER_FLAG tensorflow==2.5 kfp google-cloud-aiplatform google-cloud-storage google-cloud-pipeline-components"
       ]
     },
     {
@@ -639,7 +639,6 @@
       "outputs": [],
       "source": [
         "import tensorflow as tf\n",
-        "from google_cloud_pipeline_components import aiplatform as gcc_aip\n",
         "from google_cloud_pipeline_components.experimental.custom_job import utils\n",
         "from kfp.v2 import compiler, dsl\n",
         "from kfp.v2.dsl import component"
@@ -1172,6 +1171,10 @@
         "    steps: int = 200,\n",
         "    distribute: str = \"single\",\n",
         "):\n",
+        "    from google_cloud_pipeline_components.v1.batch_predict_job import \\\n",
+        "        ModelBatchPredictOp\n",
+        "    from google_cloud_pipeline_components.v1.model import ModelUploadOp\n",
+        "\n",
         "    custom_producer_task = custom_job_distributed_training_op(\n",
         "        model_dir=model_dir,\n",
         "        lr=lr,\n",
@@ -1183,7 +1186,7 @@
         "        base_output_directory=PIPELINE_ROOT,\n",
         "    )\n",
         "\n",
-        "    model_upload_op = gcc_aip.ModelUploadOp(\n",
+        "    model_upload_op = ModelUploadOp(\n",
         "        project=PROJECT_ID,\n",
         "        display_name=\"model_display_name\",\n",
         "        artifact_uri=model_dir,\n",
@@ -1191,7 +1194,7 @@
         "    )\n",
         "    model_upload_op.after(custom_producer_task)\n",
         "\n",
-        "    batch_predict_op = gcc_aip.ModelBatchPredictOp(\n",
+        "    batch_predict_op = ModelBatchPredictOp(\n",
         "        project=PROJECT_ID,\n",
         "        job_display_name=\"batch_predict_job\",\n",
         "        model=model_upload_op.outputs[\"model\"],\n",

notebooks/official/pipelines/google_cloud_pipeline_components_automl_images.ipynb

@@ -612,31 +612,6 @@
         "import google.cloud.aiplatform as aip"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "aip_constants:endpoint"
-      },
-      "source": [
-        "#### Vertex AI constants\n",
-        "\n",
-        "Setup up the following constants for Vertex AI:\n",
-        "\n",
-        "- `API_ENDPOINT`: The Vertex AI API service endpoint for `Dataset`, `Model`, `Job`, `Pipeline` and `Endpoint` services."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "aip_constants:endpoint"
-      },
-      "outputs": [],
-      "source": [
-        "# API service endpoint\n",
-        "API_ENDPOINT = \"{}-aiplatform.googleapis.com\".format(REGION)"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -676,8 +651,7 @@
       },
       "outputs": [],
       "source": [
-        "import kfp\n",
-        "from google_cloud_pipeline_components import aiplatform as gcc_aip"
+        "import kfp"
       ]
     },
     {
@@ -725,6 +699,10 @@
       "source": [
         "@kfp.dsl.pipeline(name=\"automl-image-training-v2\")\n",
         "def pipeline(project: str = PROJECT_ID, region: str = REGION):\n",
+        "    from google_cloud_pipeline_components import aiplatform as gcc_aip\n",
+        "    from google_cloud_pipeline_components.v1.endpoint import (EndpointCreateOp,\n",
+        "                                                              ModelDeployOp)\n",
+        "\n",
         "    ds_op = gcc_aip.ImageDatasetCreateOp(\n",
         "        project=project,\n",
         "        display_name=\"flowers\",\n",
@@ -745,13 +723,13 @@
         "        budget_milli_node_hours=8000,\n",
         "    )\n",
         "\n",
-        "    endpoint_op = gcc_aip.EndpointCreateOp(\n",
+        "    endpoint_op = EndpointCreateOp(\n",
         "        project=project,\n",
         "        location=region,\n",
         "        display_name=\"train-automl-flowers\",\n",
         "    )\n",
         "\n",
-        "    gcc_aip.ModelDeployOp(\n",
+        "    ModelDeployOp(\n",
         "        model=training_job_run_op.outputs[\"model\"],\n",
         "        endpoint=endpoint_op.outputs[\"endpoint\"],\n",
         "        automatic_resources_min_replica_count=1,\n",

notebooks/official/pipelines/google_cloud_pipeline_components_automl_tabular.ipynb

@@ -614,31 +614,6 @@
         "import google.cloud.aiplatform as aip"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "aip_constants:endpoint"
-      },
-      "source": [
-        "#### Vertex AI constants\n",
-        "\n",
-        "Setup up the following constants for Vertex AI:\n",
-        "\n",
-        "- `API_ENDPOINT`: The Vertex AI API service endpoint for `Dataset`, `Model`, `Job`, `Pipeline` and `Endpoint` services."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "aip_constants:endpoint"
-      },
-      "outputs": [],
-      "source": [
-        "# API service endpoint\n",
-        "API_ENDPOINT = \"{}-aiplatform.googleapis.com\".format(REGION)"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -678,8 +653,7 @@
       },
       "outputs": [],
       "source": [
-        "import kfp\n",
-        "from google_cloud_pipeline_components import aiplatform as gcc_aip"
+        "import kfp"
       ]
     },
     {
@@ -733,6 +707,9 @@
         "\n",
         "@kfp.dsl.pipeline(name=\"automl-tab-training-v2\")\n",
         "def pipeline(project: str = PROJECT_ID, region: str = REGION):\n",
+        "    from google_cloud_pipeline_components import aiplatform as gcc_aip\n",
+        "    from google_cloud_pipeline_components.v1.endpoint import (EndpointCreateOp,\n",
+        "                                                              ModelDeployOp)\n",
         "\n",
         "    dataset_create_op = gcc_aip.TabularDatasetCreateOp(\n",
         "        project=project, display_name=\"housing\", gcs_source=gcs_csv_path\n",
@@ -758,13 +735,13 @@
         "        target_column=\"median_house_value\",\n",
         "    )\n",
         "\n",
-        "    endpoint_op = gcc_aip.EndpointCreateOp(\n",
+        "    endpoint_op = EndpointCreateOp(\n",
         "        project=project,\n",
         "        location=region,\n",
         "        display_name=\"train-automl-flowers\",\n",
         "    )\n",
         "\n",
-        "    gcc_aip.ModelDeployOp(\n",
+        "    ModelDeployOp(\n",
         "        model=training_op.outputs[\"model\"],\n",
         "        endpoint=endpoint_op.outputs[\"endpoint\"],\n",
         "        dedicated_resources_machine_type=\"n1-standard-4\",\n",

notebooks/official/pipelines/google_cloud_pipeline_components_automl_text.ipynb

@@ -612,31 +612,6 @@
         "import google.cloud.aiplatform as aip"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "aip_constants:endpoint"
-      },
-      "source": [
-        "#### Vertex AI constants\n",
-        "\n",
-        "Setup up the following constants for Vertex AI:\n",
-        "\n",
-        "- `API_ENDPOINT`: The Vertex AI API service endpoint for `Dataset`, `Model`, `Job`, `Pipeline` and `Endpoint` services."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "aip_constants:endpoint"
-      },
-      "outputs": [],
-      "source": [
-        "# API service endpoint\n",
-        "API_ENDPOINT = \"{}-aiplatform.googleapis.com\".format(REGION)"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -676,8 +651,7 @@
       },
       "outputs": [],
       "source": [
-        "import kfp\n",
-        "from google_cloud_pipeline_components import aiplatform as gcc_aip"
+        "import kfp"
       ]
     },
     {
@@ -730,6 +704,9 @@
         "def pipeline(\n",
         "    project: str = PROJECT_ID, region: str = REGION, import_file: str = IMPORT_FILE\n",
         "):\n",
+        "    from google_cloud_pipeline_components import aiplatform as gcc_aip\n",
+        "    from google_cloud_pipeline_components.v1.endpoint import (EndpointCreateOp,\n",
+        "                                                              ModelDeployOp)\n",
         "\n",
         "    dataset_create_task = gcc_aip.TextDatasetCreateOp(\n",
         "        display_name=\"train-automl-happydb\",\n",
@@ -750,13 +727,13 @@
         "        project=project,\n",
         "    )\n",
         "\n",
-        "    endpoint_op = gcc_aip.EndpointCreateOp(\n",
+        "    endpoint_op = EndpointCreateOp(\n",
         "        project=project,\n",
         "        location=region,\n",
         "        display_name=\"train-automl-flowers\",\n",
         "    )\n",
         "\n",
-        "    gcc_aip.ModelDeployOp(\n",
+        "    ModelDeployOp(\n",
         "        model=training_run_task.outputs[\"model\"],\n",
         "        endpoint=endpoint_op.outputs[\"endpoint\"],\n",
         "        automatic_resources_min_replica_count=1,\n",

notebooks/official/pipelines/google_cloud_pipeline_components_model_train_upload_deploy.ipynb

@@ -657,9 +657,7 @@
       "outputs": [],
       "source": [
         "import kfp\n",
-        "from google_cloud_pipeline_components import aiplatform as gcc_aip\n",
-        "from kfp.v2.dsl import component\n",
-        "from kfp.v2.google import experimental"
+        "from kfp.v2.dsl import component"
       ]
     },
     {
@@ -752,6 +750,10 @@
         "    model_display_name: str = MODEL_DISPLAY_NAME,\n",
         "    serving_container_image_uri: str = \"us-docker.pkg.dev/cloud-aiplatform/prediction/tf2-cpu.2-3:latest\",\n",
         "):\n",
+        "    from google_cloud_pipeline_components.v1.endpoint import (EndpointCreateOp,\n",
+        "                                                              ModelDeployOp)\n",
+        "    from google_cloud_pipeline_components.v1.model import ModelUploadOp\n",
+        "    from kfp.v2.google import experimental\n",
         "\n",
         "    train_task = print_op(\"model training\")\n",
         "    experimental.run_as_aiplatform_custom_job(\n",
@@ -773,7 +775,7 @@
         "        ],\n",
         "    )\n",
         "\n",
-        "    model_upload_op = gcc_aip.ModelUploadOp(\n",
+        "    model_upload_op = ModelUploadOp(\n",
         "        project=project,\n",
         "        display_name=model_display_name,\n",
         "        artifact_uri=WORKING_DIR,\n",
@@ -782,12 +784,12 @@
         "    )\n",
         "    model_upload_op.after(train_task)\n",
         "\n",
-        "    endpoint_create_op = gcc_aip.EndpointCreateOp(\n",
+        "    endpoint_create_op = EndpointCreateOp(\n",
         "        project=project,\n",
         "        display_name=\"pipelines-created-endpoint\",\n",
         "    )\n",
         "\n",
-        "    gcc_aip.ModelDeployOp(\n",
+        "    ModelDeployOp(\n",
         "        endpoint=endpoint_create_op.outputs[\"endpoint\"],\n",
         "        model=model_upload_op.outputs[\"model\"],\n",
         "        deployed_model_display_name=model_display_name,\n",