Merge branch 'main' into mlops_10

notebooks/community/ml_ops/stage2/get_started_vertex_tensorboard.ipynb

@@ -149,8 +149,8 @@
       "source": [
         "### Set up your local development environment\n",
         "\n",
-        "**If you are using Colab or Google Cloud Notebooks**, your environment already meets\n",
-        "all the requirements to run this notebook. You can skip this step.\n",
+        "**If you are using Colab or Vertex AI Workbench Notebooks**, your environment already meets\n",
+        "all the requirements to run this notebook.  \n",
         "\n",
         "**Otherwise**, make sure your environment meets this notebook's requirements.\n",
         "You need the following:\n",

notebooks/community/ml_ops/stage2/get_started_vertex_training_lightgbm.ipynb

@@ -138,7 +138,7 @@
       "source": [
         "### Set up your local development environment\n",
         "\n",
-        "If you are using Colab or Google Cloud Notebooks, your environment already meets all the requirements to run this notebook. You can skip this step.\n",
+        "If you are using Colab or Vertex AI Workbench Notebooks, your environment already meets all the requirements to run this notebook. You can skip this step.\n",
         "\n",
         "Otherwise, make sure your environment meets this notebook's requirements. You need the following:\n",
         "\n",

notebooks/community/ml_ops/stage2/get_started_vertex_training_pytorch.ipynb

@@ -128,7 +128,7 @@
       "source": [
         "### Set up your local development environment\n",
         "\n",
-        "**If you are using Colab or Google Cloud Notebooks**, your environment already meets\n",
+        "**If you are using Colab or Vertex AI Workbench Notebooks**, your environment already meets\n",
         "all the requirements to run this notebook. You can skip this step."
       ]
     },

notebooks/community/ml_ops/stage2/get_started_vertex_training_sklearn.ipynb

@@ -127,7 +127,7 @@
       "source": [
         "### Set up your local development environment\n",
         "\n",
-        "**If you are using Colab or Google Cloud Notebooks**, your environment already meets\n",
+        "**If you are using Colab or Vertex AI Workbench Notebooks**, your environment already meets\n",
         "all the requirements to run this notebook. You can skip this step.\n",
         "\n",
         "**Otherwise**, make sure your environment meets this notebook's requirements.\n",

notebooks/community/ml_ops/stage2/get_started_vertex_vizier.ipynb

@@ -62,7 +62,7 @@
         "## Overview\n",
         "\n",
         "\n",
-        "This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 2 : experimentation: get started with Vertex Vizier."
+        "This tutorial demonstrates how to use Vertex AI for E2E MLOps on Google Cloud in production. This tutorial covers stage 2 : experimentation: get started with Vertex AI Vizier."
       ]
     },
     {

notebooks/community/ml_ops/stage2/get_started_with_visionapi_and_automl.ipynb

@@ -151,7 +151,7 @@
       "source": [
         "### Set up your local development environment\n",
         "\n",
-        "If you are using Colab or Google Cloud Notebooks, your environment already meets all the requirements to run this notebook. You can skip this step.\n",
+        "If you are using Colab or Vertex AI Workbench Notebooks, your environment already meets all the requirements to run this notebook.  \n",
         "\n",
         "Otherwise, make sure your environment meets this notebook's requirements. You need the following:\n",
         "\n",

notebooks/community/ml_ops/stage3/get_started_with_rapid_prototyping_bqml_automl.ipynb

@@ -204,7 +204,7 @@
       "source": [
         "### Set up your local development environment\n",
         "\n",
-        "If you are using Colab or Google Cloud Notebooks, your environment already meets all the requirements to run this notebook. You can skip this step.\n",
+        "If you are using Colab or Vertex AI Workbench Notebooks, your environment already meets all the requirements to run this notebook.  \n",
         "\n",
         "Otherwise, make sure your environment meets this notebook's requirements. You need the following:\n",
         "\n",

notebooks/community/ml_ops/stage6/get_started_with_cpr.ipynb

@@ -169,15 +169,6 @@
         "scikit-learn~=0.24"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "faf22f3af1ce"
-      },
-      "source": [
-        "**The model you deploy will have a different set of dependencies pre-installed than your notebook environment has. You should not assume that because things work in the notebook, they will work in the model. Instead, you will be very explicit about the dependencies for the model by listing them in requirements.txt and then use `pip install` to install the exact same dependencies in the notebook. Please note, of course, that there is a chance that a dependency is missed in requirements.txt that already exists in the notebook. If that's the case, things will run in the notebook, but not in the model. To guard against that, you will test the model locally before deploying to the cloud.**"
-      ]
-    },
     {
       "cell_type": "code",
       "execution_count": null,

notebooks/community/ml_ops/stage6/get_started_with_matching_engine_swivel.ipynb

@@ -143,7 +143,7 @@
       "source": [
         "### Set up your local development environment\n",
         "\n",
-        "**If you are using Colab or Google Cloud Notebooks**, your environment already meets\n",
+        "**If you are using Colab or Vertex AI Workbench Notebooks**, your environment already meets\n",
         "all the requirements to run this notebook. You can skip this step."
       ]
     },
@@ -424,7 +424,7 @@
         "### Authenticate your Google Cloud account\n",
         "\n",
         "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
-        "authenticated. Skip this step."
+        "authenticated. "
       ]
     },
     {
@@ -1355,8 +1355,6 @@
       "source": [
         "QUERY_EMBEDDING_PATH = f\"{BUCKET_URI}/embeddings/train.jsonl\"\n",
         "\n",
-        "import tensorflow as tf\n",
-        "\n",
         "with tf.io.gfile.GFile(QUERY_EMBEDDING_PATH, \"w\") as f:\n",
         "    for i in range(1, 200001):\n",
         "        query = str(i)\n",
@@ -1417,7 +1415,7 @@
         "MAX_NODES = 4\n",
         "\n",
         "batch_predict_job = model.batch_predict(\n",
-        "    job_display_name=f\"batch_predict_swivel\",\n",
+        "    job_display_name=\"batch_predict_swivel\",\n",
         "    gcs_source=[QUERY_EMBEDDING_PATH],\n",
         "    gcs_destination_prefix=f\"{BUCKET_URI}/embeddings/output\",\n",
         "    machine_type=DEPLOY_COMPUTE,\n",

notebooks/community/ml_ops/stage6/get_started_with_matching_engine_twotowers.ipynb

@@ -142,7 +142,7 @@
       "source": [
         "### Set up your local development environment\n",
         "\n",
-        "**If you are using Colab or Google Cloud Notebooks**, your environment already meets\n",
+        "**If you are using Colab or Vertex AI Workbench Notebooks**, your environment already meets\n",
         "all the requirements to run this notebook. You can skip this step."
       ]
     },
@@ -422,7 +422,7 @@
         "### Authenticate your Google Cloud account\n",
         "\n",
         "**If you are using Vertex AI Workbench Notebooks**, your environment is already\n",
-        "authenticated. Skip this step."
+        "authenticated. "
       ]
     },
     {

notebooks/community/ml_ops/stage6/get_started_with_xai_and_custom_server.ipynb

@@ -134,7 +134,7 @@
       "source": [
         "### Set up your local development environment\n",
         "\n",
-        "**If you are using Colab or Google Cloud Notebooks**, your environment already meets\n",
+        "**If you are using Colab or Vertex AI Workbench Notebooks**, your environment already meets\n",
         "all the requirements to run this notebook. You can skip this step."
       ]
     },
@@ -395,7 +395,7 @@
       "source": [
         "### Authenticate your Google Cloud account\n",
         "\n",
-        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. Skip this step.\n",
+        "**If you are using Vertex AI Workbench Notebooks**, your environment is already authenticated. \n",
         "\n",
         "**If you are using Colab**, run the cell below and follow the instructions when prompted to authenticate your account via oAuth.\n",
         "\n",

notebooks/notebook_template_review.py

@@ -25,7 +25,11 @@ args = parser.parse_args()
 
 if args.errors_codes:
     args.errors_codes = args.errors_codes.split(',')
-    
+    args.errors = True
+
+if args.errors_csv:
+    args.errors = True
+
 def parse_dir(directory):
     entries = os.scandir(directory)
     for entry in entries:
@@ -38,7 +42,7 @@ def parse_dir(directory):
             parse_dir(entry.path)
         elif entry.name.endswith('.ipynb'):
             parse_notebook(entry.path)
-            
+
 def parse_notebook(path):
     with open(path, 'r') as f:
         try:
@@ -158,6 +162,11 @@ def parse_notebook(path):
             if cell['cell_type'] != 'code':
                 report_error(path, 22, "Installation code section not found")
             else:
+                if cell['source'][0].startswith('! mkdir'):
+                    cell, nth = get_cell(path, cells, nth)
+                if 'requirements.txt' in cell['source'][0]:
+                    cell, nth = get_cell(path, cells, nth)
+                    
                 text = ''
                 for line in cell['source']:
                     text += line
@@ -238,7 +247,7 @@ def parse_notebook(path):
         if cell['source'][0].startswith("### Region"): 
             report_error(path, 34, "Region section not found")
         '''
-                
+
 
 def get_cell(path, cells, nth):
     while empty_cell(path, cells, nth):
@@ -249,15 +258,38 @@ def get_cell(path, cells, nth):
         check_text_cell(path, cell)
     return cell, nth + 1
 
-                
+
 def empty_cell(path, cells, nth):
     if len(cells[nth]['source']) == 0:
         report_error(path, 10, f'empty cell: cell #{nth}')
         return True
     else:
         return False
-    
+
 def check_text_cell(path, cell):
+    
+    branding = {
+        'Vertex SDK': 'Vertex AI SDK',
+        'Vertex Training': 'Vertex AI Training',
+        'Vertex Prediction': 'Vertex AI Prediction',
+        'Vertex Batch Prediction': 'Vertex AI Batch Prediction',
+        'Vertex XAI': 'Vertex Explainable AI',
+        'Vertex Experiments': 'Vertex AI Experiments',
+        'Vertex TensorBoard': 'Vertex AI TensorBoard',
+        'Vertex Pipelines': 'Vertex AI Pipelines',
+        'Vertex Hyperparameter Tuning': 'Vertex AI Hyperparameter Tuning',
+        'Vertex Metadata': 'Vertex ML Metadata',
+        'Vertex AI Metadata': 'Vertex ML Metadata',
+        'Vertex Vizier': 'Vertex AI Vizier',
+        'Vertex Dataset': 'Vertex AI Dataset',
+        'Vertex Model': 'Vertex AI Model',
+        'Vertex Endpoint': 'Vertex AI Endpoint',
+        'Vertex Private Endpoint': 'Vertex AI Private Endpoint',
+        'Tensorflow': 'TensorFlow',
+        'Tensorboard': 'TensorBoard',
+        'Google Cloud Notebooks': 'Vertex AI Workbench Notebooks'
+    }
+    
     for line in cell['source']:
         if 'TODO' in line:
             report_error(path, 14, f'TODO in cell: {line}')
@@ -266,28 +298,9 @@ def check_text_cell(path, cell):
         if 'will' in line.lower() or 'would' in line.lower():
             report_error(path, 16, f'Do not use future tense (e.g., will), replace with present tense: {line}')
             
-        if 'Vertex SDK' in line:
-            report_error(path, 27, f"Branding: Vertex AI SDK: {line}")
-        if 'Vertex Training' in line:
-            report_error(path, 27, f"Branding: Vertex AI Training: {line}")
-        if 'Vertex Prediction' in line:
-            report_error(path, 27, f"Branding: Vertex AI Prediction: {line}")
-        if 'Vertex Batch Prediction' in line:
-            report_error(path, 27, f"Branding: Vertex AI Batch Prediction {line}")
-        if 'Vertex XAI' in line:
-            report_error(path, 27, f"Branding: Vertex Explainable AI: {line}")
-        if 'Vertex Experiments' in line:
-            report_error(path, 27, f"Branding: Vertex AI Experiments: {line}")
-        if 'Vertex TensorBoard' in line:
-            report_error(path, 27, f"Branding: Vertex AI TensorBoard: {line}")
-        if 'Vertex Pipelines' in line:
-            report_error(path, 27, f"Branding: Vertex AI Pipelines: {line}")
-        if 'Vertex Hyperparameter Tuning' in line:
-            report_error(path, 27, f"Branding: Vertex AI Hyperparameter Tuning: {line}")
-        if 'Tensorflow' in line:
-            report_error(path, 27, f"Branding: TensorFlow: {line}")
-        if 'Tensorboard' in line:
-            report_error(path, 27, f"Branding: TensorBoard: {line}")
+        for mistake, brand in branding.items():
+            if mistake in line:
+                report_error(path, 27, f"Branding {brand}: {line}")
 
 
 def check_sentence_case(path, heading):
@@ -298,12 +311,12 @@ def check_sentence_case(path, heading):
     for word in words[1:]:
         word = word.replace(':', '').replace('(', '').replace(')', '')
         if word in ['E2E', 'Vertex', 'AutoML', 'ML', 'AI', 'GCP', 'API', 'R', 'CMEK', 'TFX', 'TFDV', 'SDK',
-                    'VM', 'CPR', 'NVIDIA', 'ID']:
+                    'VM', 'CPR', 'NVIDIA', 'ID', 'DASK']:
             continue
         if word.isupper():
             report_error(path, 3, f"heading is not sentence case: {word}")
-            
-            
+
+
 def report_error(notebook, code, msg):
     if args.errors:
         if args.errors_codes:
@@ -314,7 +327,7 @@ def report_error(notebook, code, msg):
             print(notebook, ',', code)
         else:
             print(f"{notebook}: ERROR ({code}): {msg}")
-            
+
 def parse_objective(path, cell):
     desc = ''
     in_desc = True
@@ -392,7 +405,7 @@ def add_index(path, title, desc, uses, steps):
         
     if args.steps:
         print(steps)
-            
+
 
 if args.notebook_dir:
     if not os.path.isdir(args.notebook_dir):
@@ -406,4 +419,4 @@ elif args.notebook:
     parse_notebook(args.notebook)
 else:
         print("Error: must specify a directory or notebook")
-        exit(1)
+        exit(1)