Merge branch 'main' into xai_batch

notebooks/official/explainable_ai/sdk_custom_tabular_regression_batch_explain.ipynb

@@ -188,9 +188,9 @@
         "if IS_WORKBENCH_NOTEBOOK:\n",
         "    USER_FLAG = \"--user\"\n",
         "\n",
-        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG -q\n",
-        "! pip3 install {USER_FLAG} --upgrade google-cloud-storage -q\n",
-        "! pip3 install --upgrade tensorflow $USER_FLAG -q"
+        "! pip3 install --upgrade --quiet {USER_FLAG} google-cloud-aiplatform \\\n",
+        "                                             google-cloud-storage \\\n",
+        "                                             tensorflow==2.5"
       ]
     },
     {
@@ -317,7 +317,10 @@
       },
       "outputs": [],
       "source": [
-        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+        "REGION = \"[your-region]\"  # @param {type: \"string\"}\n",
+        "\n",
+        "if REGION == \"[your-region]\":\n",
+        "    REGION = \"us-central1\""
       ]
     },
     {
@@ -326,9 +329,9 @@
         "id": "timestamp"
       },
       "source": [
-        "#### Timestamp\n",
+        "#### UUID\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
       ]
     },
     {
@@ -339,9 +342,16 @@
       },
       "outputs": [],
       "source": [
-        "from datetime import datetime\n",
+        "import random\n",
+        "import string\n",
         "\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
+        "\n",
+        "# Generate a uuid of length 8\n",
+        "def generate_uuid():\n",
+        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=8))\n",
+        "\n",
+        "\n",
+        "UUID = generate_uuid()"
       ]
     },
     {
@@ -353,7 +363,7 @@
         "### Authenticate your Google Cloud account\n",
         "\n",
         "**If you are using Workbench AI Notebooks**, your environment is already\n",
-        "authenticated. Skip this step.\n",
+        "authenticated.  \n",
         "\n",
         "**If you are using Colab**, run the cell below and follow the instructions\n",
         "when prompted to authenticate your account via oAuth.\n",
@@ -448,7 +458,7 @@
       "outputs": [],
       "source": [
         "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
         "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
       ]
     },
@@ -948,7 +958,7 @@
       "outputs": [],
       "source": [
         "job = aip.CustomTrainingJob(\n",
-        "    display_name=\"boston_\" + TIMESTAMP,\n",
+        "    display_name=\"boston_\" + UUID,\n",
         "    script_path=\"custom/trainer/task.py\",\n",
         "    container_uri=TRAIN_IMAGE,\n",
         "    requirements=[\"gcsfs==0.7.1\", \"tensorflow-datasets==4.4\"],\n",
@@ -983,7 +993,7 @@
       },
       "outputs": [],
       "source": [
-        "MODEL_DIR = \"{}/{}\".format(BUCKET_URI, TIMESTAMP)\n",
+        "MODEL_DIR = \"{}/{}\".format(BUCKET_URI, UUID)\n",
         "\n",
         "EPOCHS = 20\n",
         "STEPS = 100\n",
@@ -1404,7 +1414,7 @@
       "outputs": [],
       "source": [
         "model = aip.Model.upload(\n",
-        "    display_name=\"boston_\" + TIMESTAMP,\n",
+        "    display_name=\"boston_\" + UUID,\n",
         "    artifact_uri=MODEL_DIR,\n",
         "    serving_container_image_uri=DEPLOY_IMAGE,\n",
         "    explanation_parameters=parameters,\n",
@@ -1434,7 +1444,7 @@
       "source": [
         "### Make test items\n",
         "\n",
-        "You will use synthetic data as a test data items. Don't be concerned that we are using synthetic data -- we just want to demonstrate how to make a prediction."
+        "You use a portion of the preprocessed evaluation data (x_test) for your batch request."
       ]
     },
     {
@@ -1460,14 +1470,16 @@
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "make_batch_file:automl,tabular,alt"
+        "id": "622926573681"
       },
       "outputs": [],
       "source": [
-        "! gsutil cat $IMPORT_FILE | head -n 1 > tmp.csv\n",
-        "! gsutil cat $IMPORT_FILE | tail -n 10 >> tmp.csv\n",
-        "\n",
-        "! cut -d, -f1-16 tmp.csv > batch.csv\n",
+        "with open(\"batch.csv\", \"w\") as f:\n",
+        "    f.write(\"crim, zn, indus, chas, nox, rm, age, dis, rad, tax, ptratio, b, lstat\\n\")\n",
+        "    f.write(str(x_test[0].tolist()).replace(\"[\", \"\").replace(\"]\", \"\"))\n",
+        "    f.write(\"\\n\")\n",
+        "    f.write(str(x_test[1].tolist()))\n",
+        "    f.write(\"\\n\")\n",
         "\n",
         "gcs_input_uri = BUCKET_URI + \"/test.csv\"\n",
         "\n",
@@ -1505,7 +1517,7 @@
         "MAX_NODES = 1\n",
         "\n",
         "batch_predict_job = model.batch_predict(\n",
-        "    job_display_name=\"boston_\" + TIMESTAMP,\n",
+        "    job_display_name=\"boston_\" + UUID,\n",
         "    gcs_source=gcs_input_uri,\n",
         "    gcs_destination_prefix=BUCKET_URI,\n",
         "    instances_format=\"csv\",\n",
@@ -1539,8 +1551,7 @@
       },
       "outputs": [],
       "source": [
-        "if not os.getenv(\"IS_TESTING\"):\n",
-        "    batch_predict_job.wait()"
+        "batch_predict_job.wait()"
       ]
     },
     {
@@ -1567,22 +1578,19 @@
       },
       "outputs": [],
       "source": [
-        "if not os.getenv(\"IS_TESTING\"):\n",
-        "    import tensorflow as tf\n",
+        "bp_iter_outputs = batch_predict_job.iter_outputs()\n",
         "\n",
-        "    bp_iter_outputs = batch_predict_job.iter_outputs()\n",
+        "explanation_results = list()\n",
+        "for blob in bp_iter_outputs:\n",
+        "    if blob.name.split(\"/\")[-1].startswith(\"explanation\"):\n",
+        "        explanation_results.append(blob.name)\n",
         "\n",
-        "    explanation_results = list()\n",
-        "    for blob in bp_iter_outputs:\n",
-        "        if blob.name.split(\"/\")[-1].startswith(\"explanation\"):\n",
-        "            explanation_results.append(blob.name)\n",
-        "\n",
-        "    tags = list()\n",
-        "    for explanation_result in explanation_results:\n",
-        "        gfile_name = f\"gs://{bp_iter_outputs.bucket.name}/{explanation_result}\"\n",
-        "        with tf.io.gfile.GFile(name=gfile_name, mode=\"r\") as gfile:\n",
-        "            for line in gfile.readlines():\n",
-        "                print(line)"
+        "tags = list()\n",
+        "for explanation_result in explanation_results:\n",
+        "    gfile_name = f\"gs://{bp_iter_outputs.bucket.name}/{explanation_result}\"\n",
+        "    with tf.io.gfile.GFile(name=gfile_name, mode=\"r\") as gfile:\n",
+        "        for line in gfile.readlines():\n",
+        "            print(line)"
       ]
     },
     {
@@ -1616,7 +1624,9 @@
         "    print(e)\n",
         "\n",
         "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
-        "    ! gsutil rm -r $BUCKET_URI"
+        "    ! gsutil rm -r $BUCKET_URI\n",
+        "\n",
+        "! rm -rf batch.csv custom.tar.gz custom"
       ]
     }
   ],