Merge branch 'main' into ml_ops_5

notebooks/community/ml_ops/stage2/get_started_vertex_training_pytorch.ipynb

@@ -89,6 +89,7 @@
         "\n",
         "- Single node training using a Python package.\n",
         "- Report accuracy when hyperparameter tuning.\n",
+        "- Save the model artifacts to Cloud Storage using GCSFuse.\n",
         "- Create a `Vertex AI Model` resource."
       ]
     },
@@ -125,7 +126,8 @@
         "    ! pip3 install --upgrade pyarrow $USER_FLAG\n",
         "    ! pip3 install --upgrade cloudml-hypertune $USER_FLAG\n",
         "    ! pip3 install --upgrade kfp $USER_FLAG\n",
-        "    ! pip3 install --upgrade torchvision $USER_FLAG"
+        "    ! pip3 install --upgrade torchvision $USER_FLAG\n",
+        "    ! pip3 install --upgrade rpy2 $USER_FLAG"
       ]
     },
     {
@@ -613,7 +615,6 @@
         "import os\n",
         "import argparse\n",
         "import logging\n",
-        "import subprocess\n",
         "import hypertune\n",
         "\n",
         "import torch\n",
@@ -831,8 +832,18 @@
         "train_model(model, loss, optimizer, train_dataset, test_dataset, True, device)\n",
         "\n",
         "logging.info('start saving')\n",
-        "torch.save(model.state_dict(), 'model.pth')\n",
-        "subprocess.check_call(['gsutil', 'cp', 'model.pth', os.path.join(args.model_dir, 'model.pth')], stderr=sys.stdout)\n",
+        "# export model to gcs using GCSFuse\n",
+        "logging.info(\"Exporting model artifacts ...\")\n",
+        "gs_prefix = 'gs://'\n",
+        "gcsfuse_prefix = '/gcs/'\n",
+        "if args.model_dir.startswith(gs_prefix):\n",
+        "    args.model_dir = args.model_dir.replace(gs_prefix, gcsfuse_prefix)\n",
+        "    dirpath = os.path.split(args.model_dir)[0]\n",
+        "    if not os.path.isdir(dirpath):\n",
+        "        os.makedirs(dirpath)\n",
+        "\n",
+        "gcs_model_path = os.path.join(os.path.join(args.model_dir, 'model.pth'))\n",
+        "torch.save(model.state_dict(), gcs_model_path)\n",
         "logging.info(f'Model is saved to {args.model_dir}')"
       ]
     },
@@ -891,7 +902,7 @@
       "source": [
         "### Make Pytorch container for prediction\n",
         "\n",
-        "Currently, Vertex AI does not have a prefined container for making predictions with a deployed Pytorch model. No problem, you can assemble your own custom container. Typically, one would base the container on the `Torch Server`. We demonstrations purpose, you build a placeholder container (not complete) that includes the latest `Torch Server` image, and push it to the `Container Registry`."
+        "Currently, Vertex AI does not have a prefined container for making predictions with a deployed Pytorch model. No problem, you can assemble your own custom container. Typically, one would base the container on the `Torch Server`. For demonstration purpose, you build a placeholder container (not complete) that includes the latest `Torch Server` image, and push it to the `Container Registry`."
       ]
     },
     {

notebooks/community/ml_ops/stage2/get_started_vertex_training_r.ipynb

@@ -87,13 +87,15 @@
         "\n",
         "The steps performed include:\n",
         "\n",
-        "- Locally training an R model in a notebook using %%R magic commands\n",
-        "- Creating a deployment image with trained R model and serving functions.\n",
-        "- Testing the deployment image locally.\n",
+        "- Locally train an R model in a notebook using %%R magic commands\n",
+        "- Create a deployment image with trained R model and serving functions.\n",
+        "- Test the deployment image locally.\n",
+        "- Create a `Vertex AI Model` resource for the deployment image with embedded R model.\n",
+        "- Deploy the deployment image with embedded R model to a `Vertex AI Endpoint` resource.\n",
+        "- Test the deployment image with embedded R model.\n",
         "- Create a R-to-Python training package.\n",
         "- Create a training image for training the model.\n",
-        "- Train a R model using `Vertex AI Trainingh` service with the R-to-Python training package.\n",
-        "- Create a `Vertex AI Model` resource."
+        "- Train a R model using `Vertex AI Trainingh` service with the R-to-Python training package."
       ]
     },
     {
@@ -1058,7 +1060,8 @@
         "\n",
         "\n",
         "- Model artifact saving:\n",
-        "    - Saves the model artifacts at the Cloud Storage location specified by `model-dir`."
+        "    - Saves the model artifacts at the Cloud Storage location specified by `model-dir`.\n",
+        "    - *Note*: GCSFuse (`/gcs`) is used to do filesystem operations on Cloud Storage buckets."
       ]
     },
     {
@@ -1076,7 +1079,6 @@
         "import rpy2\n",
         "import argparse\n",
         "import logging\n",
-        "from google.cloud import storage\n",
         "\n",
         "# import rpy2's package module\n",
         "import rpy2.robjects.packages as rpackages\n",
@@ -1102,13 +1104,22 @@
         ")\n",
         "logging.info(\"Model training completed ...\")\n",
         "\n",
+        "# GCSFuse conversion\n",
+        "gs_prefix = 'gs://'\n",
+        "gcsfuse_prefix = '/gcs/'\n",
+        "if args.model_dir.startswith(gs_prefix):\n",
+        "    args.model_dir = args.model_dir.replace(gs_prefix, gcsfuse_prefix)\n",
+        "    dirpath = os.path.split(args.model_dir)[0]\n",
+        "    if not os.path.isdir(dirpath):\n",
+        "        os.makedirs(dirpath)\n",
         "\n",
         "# Upload the saved model file to Cloud Storage\n",
         "gcs_model_path = os.path.join(args.model_dir, 'model.RData')\n",
         "logging.info(\"Saving model artifacts to {}\". format(gcs_model_path))\n",
-        "storage_path = os.path.join(gcs_model_path, 'model.RData')\n",
-        "blob = storage.blob.Blob.from_string(storage_path, client=storage.Client())\n",
-        "blob.upload_from_filename('model.RData')"
+        "with open(\"model.RData\", \"rb\") as f:\n",
+        "    data = f.read()\n",
+        "with open(gcs_model_path, \"wb\") as f:\n",
+        "    f.write(data)"
       ]
     },
     {
@@ -1258,17 +1269,6 @@
         "job.delete()"
       ]
     },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "rm_deploy_predict"
-      },
-      "outputs": [],
-      "source": [
-        "! rm -r deploy predict custom"
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {

notebooks/community/ml_ops/stage2/get_started_vertex_training_sklearn.ipynb

@@ -89,6 +89,7 @@
         "\n",
         "- Training using a Python package.\n",
         "- Report accuracy when hyperparameter tuning.\n",
+        "- Save the model artifacts to Cloud Storage using GCSFuse.\n",
         "- Create a `Vertex AI Model` resource."
       ]
     },
@@ -124,7 +125,9 @@
         "    ! pip3 install --upgrade apache-beam[gcp] $USER_FLAG\n",
         "    ! pip3 install --upgrade pyarrow $USER_FLAG\n",
         "    ! pip3 install --upgrade cloudml-hypertune $USER_FLAG\n",
-        "    ! pip3 install --upgrade kfp $USER_FLAG"
+        "    ! pip3 install --upgrade kfp $USER_FLAG\n",
+        "    ! pip3 install --upgrade torchvision $USER_FLAG\n",
+        "    ! pip3 install --upgrade rpy2 $USER_FLAG"
       ]
     },
     {
@@ -437,12 +440,13 @@
       "source": [
         "#### Set pre-built containers\n",
         "\n",
-        "Set the pre-built Docker container image for training.\n",
-        "\n",
-        "- Set the variable `TF` to the TensorFlow version of the container image. For example, `2-1` would be version 2.1, and `1-15` would be version 1.15. The following list shows some of the pre-built images available:\n",
+        "Set the pre-built Docker container image for training and prediction.\n",
         "\n",
         "\n",
-        "For the latest list, see [Pre-built containers for training](https://cloud.google.com/ai-platform-unified/docs/training/pre-built-containers)."
+        "For the latest list, see [Pre-built containers for training](https://cloud.google.com/ai-platform-unified/docs/training/pre-built-containers).\n",
+        "\n",
+        "\n",
+        "For the latest list, see [Pre-built containers for prediction](https://cloud.google.com/ai-platform-unified/docs/predictions/pre-built-containers)."
       ]
     },
     {
@@ -606,7 +610,8 @@
         "    - Evaluates the model.\n",
         "    - If hyperparameter tuning, reports the metric for accuracy.\n",
         "- Model artifact saving\n",
-        "    - Saves the model artifacts and evaluation metrics where the Cloud Storage location specified by `model-dir`."
+        "    - Saves the model artifacts and evaluation metrics where the Cloud Storage location specified by `model-dir`.\n",
+        "    - *Note*: GCSFuse (`/gcs`) is used to do filesystem operations on Cloud Storage buckets."
       ]
     },
     {
@@ -627,7 +632,6 @@
         "\n",
         "import pandas as pd\n",
         "import wget\n",
-        "from google.cloud import storage\n",
         "from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer\n",
         "from sklearn.model_selection import train_test_split\n",
         "from sklearn.naive_bayes import MultinomialNB\n",
@@ -697,6 +701,7 @@
         "    )\n",
         "    return score\n",
         "\n",
+        "\n",
         "def export_model_to_gcs(fitted_pipeline: Pipeline, gcs_uri: str) -> str:\n",
         "    \"\"\"Exports trained pipeline to GCS\n",
         "    Parameters:\n",
@@ -707,19 +712,14 @@
         "    Returns:\n",
         "            export_path (str): Model GCS location\n",
         "    \"\"\"\n",
-        "\n",
+        "    # Upload model artifact to Cloud Storage\n",
         "    artifact_filename = 'model.pkl'\n",
+        "    storage_path = os.path.join(gcs_uri, artifact_filename)\n",
         "\n",
         "    # Save model artifact to local filesystem (doesn't persist)\n",
-        "    local_path = artifact_filename\n",
-        "    with open(local_path, 'wb') as model_file:\n",
+        "    with open(storage_path, 'wb') as model_file:\n",
         "        pickle.dump(fitted_pipeline, model_file)\n",
         "\n",
-        "    # Upload model artifact to Cloud Storage\n",
-        "    storage_path = os.path.join(gcs_uri, artifact_filename)\n",
-        "    blob = storage.blob.Blob.from_string(storage_path, client=storage.Client())\n",
-        "    blob.upload_from_filename(local_path)\n",
-        "\n",
         "\n",
         "def export_evaluation_report_to_gcs(report: str, gcs_uri: str) -> None:\n",
         "    \"\"\"\n",
@@ -730,12 +730,13 @@
         "            i.e gs://example_bucket/training-job\n",
         "    \"\"\"\n",
         "\n",
-        "    artifact_filename = 'report.txt'\n",
-        "\n",
         "    # Upload model artifact to Cloud Storage\n",
+        "    artifact_filename = 'report.txt'\n",
         "    storage_path = os.path.join(gcs_uri, artifact_filename)\n",
-        "    blob = storage.blob.Blob.from_string(storage_path, client=storage.Client())\n",
-        "    blob.upload_from_string(report)\n",
+        "\n",
+        "    # Save model artifact to local filesystem (doesn't persist)\n",
+        "    with open(storage_path, 'w') as report_file:\n",
+        "        report_file.write(report)\n",
         "\n",
         "\n",
         "logging.info(\"Starting custom training job.\")\n",
@@ -745,8 +746,16 @@
         "model = train_model(model, *data)\n",
         "score = evaluate_model(model, *data)\n",
         "\n",
-        "# export model to gcs\n",
+        "# export model to gcs using GCSFuse\n",
         "logging.info(\"Exporting model artifacts ...\")\n",
+        "gs_prefix = 'gs://'\n",
+        "gcsfuse_prefix = '/gcs/'\n",
+        "if args.model_dir.startswith(gs_prefix):\n",
+        "    args.model_dir = args.model_dir.replace(gs_prefix, gcsfuse_prefix)\n",
+        "    dirpath = os.path.split(args.model_dir)[0]\n",
+        "    if not os.path.isdir(dirpath):\n",
+        "        os.makedirs(dirpath)\n",
+        "\n",
         "export_model_to_gcs(model, args.model_dir)\n",
         "export_evaluation_report_to_gcs(str(score), args.model_dir)\n",
         "logging.info(f\"Exported model artifacts to GCS bucket: {args.model_dir}\")"

notebooks/community/ml_ops/stage2/get_started_vertex_training_xgboost.ipynb

@@ -89,6 +89,7 @@
         "\n",
         "- Training using a Python package.\n",
         "- Report accuracy when hyperparameter tuning.\n",
+        "- Save the model artifacts to Cloud Storage using GCSFuse.\n",
         "- Create a `Vertex AI Model` resource."
       ]
     },
@@ -124,7 +125,9 @@
         "    ! pip3 install --upgrade apache-beam[gcp] $USER_FLAG\n",
         "    ! pip3 install --upgrade pyarrow $USER_FLAG\n",
         "    ! pip3 install --upgrade cloudml-hypertune $USER_FLAG\n",
-        "    ! pip3 install --upgrade kfp $USER_FLAG"
+        "    ! pip3 install --upgrade kfp $USER_FLAG\n",
+        "    ! pip3 install --upgrade torchvision $USER_FLAG\n",
+        "    ! pip3 install --upgrade rpy2 $USER_FLAG"
       ]
     },
     {
@@ -693,19 +696,24 @@
         "model = train_model(dtrain)\n",
         "accuracy = evaluate_model(model, test_data, test_labels)\n",
         "\n",
+        "# GCSFuse conversion\n",
+        "gs_prefix = 'gs://'\n",
+        "gcsfuse_prefix = '/gcs/'\n",
+        "if args.model_dir.startswith(gs_prefix):\n",
+        "    args.model_dir = args.model_dir.replace(gs_prefix, gcsfuse_prefix)\n",
+        "    dirpath = os.path.split(args.model_dir)[0]\n",
+        "    if not os.path.isdir(dirpath):\n",
+        "        os.makedirs(dirpath)\n",
+        "\n",
         "# Export the classifier to a file\n",
-        "model_filename = 'model.bst'\n",
-        "model.save_model(model_filename)\n",
-        "\n",
-        "# Upload the saved model file to Cloud Storage\n",
-        "gcs_model_path = os.path.join(args.model_dir, model_filename)\n",
+        "gcs_model_path = os.path.join(args.model_dir, 'model.bst')\n",
         "logging.info(\"Saving model artifacts to {}\". format(gcs_model_path))\n",
-        "subprocess.check_call(['gsutil', 'cp', model_filename, gcs_model_path], stderr=sys.stdout)\n",
+        "model.save_model(gcs_model_path)\n",
         "\n",
-        "with open(\"metrics.json\", \"w\") as f:\n",
-        "    f.write(f\"{'accuracy: {accuracy}'}\")\n",
         "logging.info(\"Saving metrics to {}/metrics.json\". format(args.model_dir))\n",
-        "subprocess.check_call(['gsutil', 'cp', 'metrics.json', args.model_dir + '/metrics.json'], stderr=sys.stdout)"
+        "gcs_metrics_path = os.path.join(args.model_dir, 'metrics.json')\n",
+        "with open(gcs_metrics_path, \"w\") as f:\n",
+        "    f.write(f\"{'accuracy: {accuracy}'}\")"
       ]
     },
     {