Merge branch 'main' into mlops_6v7

notebooks/community/ml_ops/stage3/get_started_with_bqml_pipeline_components.ipynb

@@ -293,7 +293,8 @@
       },
       "outputs": [],
       "source": [
-        "BUCKET_NAME = \"gs://[your-bucket-name]\"  # @param {type:\"string\"}"
+        "BUCKET_NAME = \"[your-bucket-name]\"  # @param {type:\"string\"}\n",
+        "BUCKET_URI = f\"gs://{BUCKET_NAME}"
       ]
     },
     {
@@ -304,8 +305,8 @@
       },
       "outputs": [],
       "source": [
-        "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_NAME = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
+        "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
+        "    BUCKET_URI = \"gs://\" + PROJECT_ID + \"aip-\" + TIMESTAMP"
       ]
     },
     {
@@ -325,7 +326,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_NAME"
+        "! gsutil mb -l $REGION $BUCKET_URI"
       ]
     },
     {
@@ -345,7 +346,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil ls -al $BUCKET_NAME"
+        "! gsutil ls -al $BUCKET_URI"
       ]
     },
     {
@@ -356,7 +357,9 @@
       "source": [
         "#### Service Account\n",
         "\n",
-        "**If you don't know your service account**, try to get your service account using `gcloud` command by executing the second cell below."
+        "**If you don't know your service account**, try to get your service account using `gcloud` command by executing the second cell below.\n",
+        "\n",
+        "*Note:* The code for automatically finding your service account works on a user-managed Workbench AI noteboook. If you are using a fully-managed notebook, you will need to manually enter your service account."
       ]
     },
     {
@@ -408,9 +411,9 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_NAME\n",
+        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
         "\n",
-        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_NAME"
+        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
       ]
     },
     {
@@ -672,89 +675,72 @@
       },
       "outputs": [],
       "source": [
-        "PIPELINE_ROOT = f\"{BUCKET_NAME}/bq_query\"\n",
+        "PIPELINE_ROOT = f\"{BUCKET_URI}/bq_query\"\n",
         "\n",
-        "@dsl.pipeline(\n",
-        "    name=\"bq-hello-world\",\n",
-        "    pipeline_root=PIPELINE_ROOT)\n",
-        "def pipeline(bq_table: str,\n",
-        "PIPELINE_ROOT = f\"{BUCKET_NAME}/bq_query\"\n",
-        "MODEL_DIR = BUCKET_NAME + '/bqmodel'\n",
         "\n",
-        "@dsl.pipeline(\n",
-        "    name=\"bq-hello-world\",\n",
-        "    pipeline_root=PIPELINE_ROOT)\n",
-        "def pipeline(bq_table: str,\n",
-        "             label: str,\n",
-        "             dataset: str,\n",
-        "             model: str,\n",
-        "             artifact_uri: str,\n",
-        "             num_trials: int,\n",
-        "             deploy_image: str,\n",
-        "             machine_type: str,\n",
-        "             min_replica_count: int,\n",
-        "             max_replica_count: int,\n",
-        "             display_name: str,\n",
-        "             accelerator_type: str = \"\",\n",
-        "             accelerator_count: int = 0,\n",
-        "             project: str = PROJECT_ID,\n",
-        "             location: str = \"US\",\n",
-        "             region: str = \"us-central1\"):\n",
-        "    from google_cloud_pipeline_components.v1.bigquery import (\n",
-        "        BigqueryQueryJobOp,\n",
-        "        BigqueryCreateModelJobOp,\n",
-        "        BigqueryEvaluateModelJobOp,\n",
-        "        BigqueryPredictModelJobOp,\n",
-        "        BigqueryExportModelJobOp\n",
-        "    )\n",
-        "    from google_cloud_pipeline_components.v1.model import (\n",
-        "        ModelUploadOp\n",
-        "    )\n",
-        "    from google_cloud_pipeline_components.v1.endpoint import (\n",
-        "        EndpointCreateOp,\n",
-        "        ModelDeployOp\n",
-        "    )\n",
-        "    from kfp.v2.components import importer_node\n",
+        "@dsl.pipeline(name=\"bq-hello-world\", pipeline_root=PIPELINE_ROOT)\n",
+        "def pipeline(\n",
+        "    bq_table: str,\n",
+        "    label: str,\n",
+        "    dataset: str,\n",
+        "    model: str,\n",
+        "    artifact_uri: str,\n",
+        "    num_trials: int,\n",
+        "    deploy_image: str,\n",
+        "    machine_type: str,\n",
+        "    min_replica_count: int,\n",
+        "    max_replica_count: int,\n",
+        "    display_name: str,\n",
+        "    accelerator_type: str = \"\",\n",
+        "    accelerator_count: int = 0,\n",
+        "    project: str = PROJECT_ID,\n",
+        "    location: str = \"US\",\n",
+        "    region: str = \"us-central1\",\n",
+        "):\n",
         "    from google_cloud_pipeline_components.types import artifact_types\n",
+        "    from google_cloud_pipeline_components.v1.bigquery import (\n",
+        "        BigqueryCreateModelJobOp, BigqueryEvaluateModelJobOp,\n",
+        "        BigqueryExportModelJobOp, BigqueryPredictModelJobOp,\n",
+        "        BigqueryQueryJobOp)\n",
+        "    from google_cloud_pipeline_components.v1.endpoint import (EndpointCreateOp,\n",
+        "                                                              ModelDeployOp)\n",
+        "    from google_cloud_pipeline_components.v1.model import ModelUploadOp\n",
+        "    from kfp.v2.components import importer_node\n",
         "\n",
         "    bq_dataset = BigqueryQueryJobOp(\n",
-        "          project=project,\n",
-        "          location=\"US\",\n",
-        "          query=f\"CREATE SCHEMA {dataset}\"\n",
+        "        project=project, location=\"US\", query=f\"CREATE SCHEMA {dataset}\"\n",
         "    )\n",
         "\n",
         "    bq_model = BigqueryCreateModelJobOp(\n",
-        "          project=project,\n",
-        "          location=location,\n",
-        "          query=f\"CREATE OR REPLACE MODEL {dataset}.{model} OPTIONS (model_type='dnn_classifier', labels=['{label}'], num_trials={num_trials}) AS SELECT * FROM `{bq_table}` WHERE body_mass_g IS NOT NULL AND sex IS NOT NULL\"\n",
+        "        project=project,\n",
+        "        location=location,\n",
+        "        query=f\"CREATE OR REPLACE MODEL {dataset}.{model} OPTIONS (model_type='dnn_classifier', labels=['{label}'], num_trials={num_trials}) AS SELECT * FROM `{bq_table}` WHERE body_mass_g IS NOT NULL AND sex IS NOT NULL\",\n",
         "    ).after(bq_dataset)\n",
         "\n",
         "    bq_eval = BigqueryEvaluateModelJobOp(\n",
-        "          project=PROJECT_ID,\n",
-        "          location=\"US\",\n",
-        "          model=bq_model.outputs[\"model\"]\n",
+        "        project=PROJECT_ID, location=\"US\", model=bq_model.outputs[\"model\"]\n",
         "    ).after(bq_model)\n",
         "\n",
         "    bq_predict = BigqueryPredictModelJobOp(\n",
-        "          project=project,\n",
-        "          location=location,\n",
-        "          model=bq_model.outputs[\"model\"],\n",
-        "          table_name=f\"`{bq_table}`\",\n",
-        "          #query_statement=f\"SELECT * EXCEPT ({label}) FROM {bq_table} WHERE body_mass_g IS NOT NULL AND sex IS NOT NULL\"\n",
-        "          job_configuration_query={\n",
-        "            'destinationTable': {\n",
+        "        project=project,\n",
+        "        location=location,\n",
+        "        model=bq_model.outputs[\"model\"],\n",
+        "        table_name=f\"`{bq_table}`\",\n",
+        "        # query_statement=f\"SELECT * EXCEPT ({label}) FROM {bq_table} WHERE body_mass_g IS NOT NULL AND sex IS NOT NULL\"\n",
+        "        job_configuration_query={\n",
+        "            \"destinationTable\": {\n",
         "                \"projectId\": PROJECT_ID,\n",
         "                \"datasetId\": \"bqml_tutorial\",\n",
-        "                \"tableId\": \"results_1\"\n",
+        "                \"tableId\": \"results_1\",\n",
         "            }\n",
-        "          }\n",
+        "        },\n",
         "    ).after(bq_model)\n",
         "\n",
         "    bq_export = BigqueryExportModelJobOp(\n",
-        "          project=project,\n",
-        "          location=location,\n",
-        "          model=bq_model.outputs[\"model\"],\n",
-        "          model_destination_path=artifact_uri\n",
+        "        project=project,\n",
+        "        location=location,\n",
+        "        model=bq_model.outputs[\"model\"],\n",
+        "        model_destination_path=artifact_uri,\n",
         "    ).after(bq_model)\n",
         "\n",
         "    import_unmanaged_model_task = importer_node.importer(\n",
@@ -787,7 +773,7 @@
         "        dedicated_resources_machine_type=machine_type,\n",
         "        dedicated_resources_accelerator_type=accelerator_type,\n",
         "        dedicated_resources_accelerator_count=accelerator_count,\n",
-        "        traffic_split={\"0\": 100}\n",
+        "        traffic_split={\"0\": 100},\n",
         "    )"
       ]
     },
@@ -1147,17 +1133,7 @@
         "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
         "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
         "\n",
-        "Otherwise, you can delete the individual resources you created in this tutorial:\n",
-        "\n",
-        "- Dataset\n",
-        "- Pipeline\n",
-        "- Model\n",
-        "- Endpoint\n",
-        "- AutoML Training Job\n",
-        "- Batch Job\n",
-        "- Custom Job\n",
-        "- Hyperparameter Tuning Job\n",
-        "- Cloud Storage Bucket"
+        "Otherwise, you can delete the individual resources you created in this tutorial:"
       ]
     },
     {
@@ -1171,57 +1147,9 @@
         "delete_all = True\n",
         "\n",
         "if delete_all:\n",
-        "    # Delete the dataset using the Vertex dataset object\n",
-        "    try:\n",
-        "        if \"dataset\" in globals():\n",
-        "            dataset.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
+        "    # (DEVELOPER TODO) Find generated resources from pipeline and delete\n",
         "\n",
-        "    # Delete the model using the Vertex model object\n",
-        "    try:\n",
-        "        if \"model\" in globals():\n",
-        "            model.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the endpoint using the Vertex endpoint object\n",
-        "    try:\n",
-        "        if \"endpoint\" in globals():\n",
-        "            endpoint.undeploy_all()\n",
-        "            endpoint.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the AutoML or Pipeline training job\n",
-        "    try:\n",
-        "        if \"dag\" in globals():\n",
-        "            dag.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the custom training job\n",
-        "    try:\n",
-        "        if \"job\" in globals():\n",
-        "            job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the batch prediction job using the Vertex batch prediction object\n",
-        "    try:\n",
-        "        if \"batch_predict_job\" in globals():\n",
-        "            batch_predict_job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    # Delete the hyperparameter tuning job using the Vertex hyperparameter tuning object\n",
-        "    try:\n",
-        "        if \"hpt_job\" in globals():\n",
-        "            hpt_job.delete()\n",
-        "    except Exception as e:\n",
-        "        print(e)\n",
-        "\n",
-        "    if \"BUCKET_NAME\" in globals():\n",
+        "    if \"BUCKET_URI\" in globals():\n",
         "        ! gsutil rm -r $BUCKET_NAME"
       ]
     }