custom-tabular-bq-managed-dataset.ipynb: Addressed comments regarding evaluation, normalization and dataset creation (#1313)

* Address comments

* Added back BQ dataset creation

notebooks/official/custom/custom-tabular-bq-managed-dataset.ipynb

@@ -413,18 +413,6 @@
         "- Split train and test data"
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "b5dfe6890137"
-      },
-      "outputs": [],
-      "source": [
-        "# Define the BigQuery source dataset\n",
-        "BQ_SOURCE = \"bigquery-public-data.ml_datasets.penguins\""
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -451,10 +439,6 @@
         "# Drop unusable rows\n",
         "df = df.replace(to_replace=NA_VALUES, value=np.NaN).dropna()\n",
         "\n",
-        "df_numeric = df.select_dtypes(include=\"number\").astype(\"float32\")\n",
-        "df_numeric = (df_numeric - df_numeric.mean()) / df_numeric.std()\n",
-        "df[df_numeric.columns] = df_numeric\n",
-        "\n",
         "# Convert categorical columns to numeric\n",
         "df[\"island\"], _ = pd.factorize(df[\"island\"])\n",
         "df[\"species\"], _ = pd.factorize(df[\"species\"])\n",
@@ -465,45 +449,6 @@
         "df_holdout = df[~df.index.isin(df_train.index)]"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "4e6a4fd28bab"
-      },
-      "source": [
-        "### Write the training dataset to BigQuery\n",
-        "Use the BigQuery SDK to create a dataset and write your training dataframe to it."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "8b41bbd2380a"
-      },
-      "outputs": [],
-      "source": [
-        "# Write training dataset to BigQuery\n",
-        "\n",
-        "# Create BigQuery dataset\n",
-        "dataset_id = \"dataset_id_unique\"\n",
-        "bq_dataset = bigquery.Dataset(f\"{PROJECT_ID}.{dataset_id}\")\n",
-        "bq_dataset = bq_client.create_dataset(bq_dataset, exists_ok=True)\n",
-        "\n",
-        "# Reference: https://cloud.google.com/bigquery/docs/samples/bigquery-load-table-dataframe\n",
-        "table_id = \"table_id_unique\"\n",
-        "job = bq_client.load_table_from_dataframe(\n",
-        "    dataframe=df_train,\n",
-        "    destination=f\"{PROJECT_ID}.{dataset_id}.{table_id}\",\n",
-        ")\n",
-        "\n",
-        "job.result()\n",
-        "\n",
-        "BQ_TRAIN_URI = str(job.destination)\n",
-        "\n",
-        "BQ_TRAIN_URI"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -517,6 +462,20 @@
         "See more info here: https://cloud.google.com/vertex-ai/docs/training/using-managed-datasets"
       ]
     },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "7fa452ee5c75"
+      },
+      "outputs": [],
+      "source": [
+        "# Create BigQuery dataset\n",
+        "bq_dataset_id = f\"{PROJECT_ID}.dataset_id_unique\"\n",
+        "bq_dataset = bigquery.Dataset(bq_dataset_id)\n",
+        "bq_client.create_dataset(bq_dataset, exists_ok=True)"
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
@@ -525,8 +484,10 @@
       },
       "outputs": [],
       "source": [
-        "dataset = aiplatform.TabularDataset.create(\n",
+        "dataset = aiplatform.TabularDataset.create_from_dataframe(\n",
-        "    display_name=\"sample-penguins\", bq_source=f\"bq://{BQ_TRAIN_URI}\"\n",
+        "    df_source=df_train,\n",
+        "    staging_path=f\"bq://{bq_dataset_id}.table-unique\",\n",
+        "    display_name=\"sample-penguins\",\n",
         ")"
       ]
     },
@@ -542,7 +503,9 @@
         "\n",
         "- **Use a Vertex AI pre-built container**. If you use a pre-built training container, you must additionally specify a Python package to install into the container image. This Python package contains your training code.\n",
         "\n",
-        "- **Use your own custom container image**. If you use your own container, the container image must contain your training code."
+        "- **Use your own custom container image**. If you use your own container, the container image must contain your training code.\n",
+        "\n",
+        "You will use a pre-built container for this demo."
       ]
     },
     {
@@ -631,15 +594,9 @@
         "\n",
         "# Read args\n",
         "parser = argparse.ArgumentParser()\n",
-        "parser.add_argument('--label_column', dest='label_column',\n",
+        "parser.add_argument('--label_column', required=True, type=str)\n",
-        "                    required=True, type=str,\n",
+        "parser.add_argument('--epochs', default=10, type=int)\n",
-        "                    help='Label column.')\n",
+        "parser.add_argument('--batch_size', default=10, type=int)\n",
-        "parser.add_argument('--epochs', dest='epochs',\n",
-        "                    default=10, type=int,\n",
-        "                    help='Number of epochs.')\n",
-        "parser.add_argument('--batch_size', dest='batch_size',\n",
-        "                    default=10, type=int,\n",
-        "                    help='Batch size.')\n",
         "args = parser.parse_args()\n",
         "\n",
         "# Set up training variables\n",
@@ -904,8 +861,7 @@
         "\n",
         "You can then run a quick evaluation on the prediction results:\n",
         "1. `np.argmax`: Convert each list of confidence levels to a label\n",
-        "2. Compare the predicted labels to the actual labels\n",
+        "2. Print predictions"
-        "3. Calculate `accuracy` as `correct/total`"
       ]
     },
     {
@@ -919,11 +875,7 @@
         "predictions = endpoint.predict(instances=holdout_x)\n",
         "y_predicted = np.argmax(predictions.predictions, axis=1)\n",
         "\n",
-        "correct = sum(y_predicted == np.array(holdout_y))\n",
+        "y_predicted"
-        "accuracy = len(y_predicted)\n",
-        "print(\n",
-        "    f\"Correct predictions = {correct}, Total predictions = {accuracy}, Accuracy = {correct/accuracy}\"\n",
-        ")"
       ]
     },
     {