Merge branch 'main' into autoreview_31

notebooks/notebook_template_review.py

@@ -35,6 +35,7 @@ import os
 import sys
 import urllib.request
 import csv
+from enum import Enum
 
 parser = argparse.ArgumentParser()
 parser.add_argument('--notebook-dir', dest='notebook_dir',
@@ -70,85 +71,87 @@ if args.errors_codes:
 if args.errors_csv:
     args.errors = True
 
-# Copyright cell
-#   Google copyright cell required
-ERROR_COPYRIGHT = 0
 
-# Links cell
-#   H1 heading required
-#   git, colab and workbench link required
-#   links must be valid links
-ERROR_TITLE_HEADING = 1
-ERROR_HEADING_CASE = 2
-ERROR_HEADING_CAP = 3
-ERROR_LINK_GIT_MISSING = 4
-ERROR_LINK_COLAB_MISSING = 5
-ERROR_LINK_WORKBENCH_MISSING = 6
-ERROR_LINK_GIT_BAD = 7
-ERROR_LINK_COLAB_BAD = 8
-ERROR_LINK_WORKBENCH_BAD = 9
+class ErrorCodes(Enum):
+    # Copyright cell
+    #   Google copyright cell required
+    ERROR_COPYRIGHT = 0,
 
-# Overview cells
-#   Overview cell required
-#   Objective cell required
-#   Dataset cell required
-#   Costs cell required
-#     Check for required Vertex and optional BQ and Dataflow
-ERROR_OVERVIEW_NOTFOUND = 10
-ERROR_OBJECTIVE_NOTFOUND = 11
-ERROR_OBJECTIVE_MISSING_DESC = 12
-ERROR_OBJECTIVE_MISSING_USES = 13
-ERROR_OBJECTIVE_MISSING_STEPS = 14
-ERROR_DATASET_NOTFOUND = 15
-ERROR_COSTS_NOTFOUND = 16
-ERROR_COSTS_MISSING = 17
+    # Links cell
+    #   H1 heading required
+    #   git, colab and workbench link required
+    #   links must be valid links
+    ERROR_TITLE_HEADING = 1,
+    ERROR_HEADING_CASE = 2,
+    ERROR_HEADING_CAP = 3,
+    ERROR_LINK_GIT_MISSING = 4,
+    ERROR_LINK_COLAB_MISSING = 5,
+    ERROR_LINK_WORKBENCH_MISSING = 6,
+    ERROR_LINK_GIT_BAD = 7,
+    ERROR_LINK_COLAB_BAD = 8,
+    ERROR_LINK_WORKBENCH_BAD = 9,
 
-# Installation cell
-#   Installation cell required
-#   Wrong heading for installation cell
-#   Installation code cell not found
-#   pip3 required
-#   option -q required
-#   option {USER_FLAG} required
-#   installation code cell not match template
-#   all packages must be installed as a single pip3
-ERROR_INSTALLATION_NOTFOUND = 18
-ERROR_INSTALLATION_HEADING = 19
-ERROR_INSTALLATION_CODE_NOTFOUND = 20
-ERROR_INSTALLATION_PIP3 = 21
-ERROR_INSTALLATION_QUIET = 22
-ERROR_INSTALLATION_USER_FLAG = 23
-ERROR_INSTALLATION_CODE_TEMPLATE = 24
-ERROR_INSTALLATION_SINGLE_PIP3 = 25
+    # Overview cells
+    #   Overview cell required
+    #   Objective cell required
+    #   Dataset cell required
+    #   Costs cell required
+    #     Check for required Vertex and optional BQ and Dataflow
+    ERROR_OVERVIEW_NOTFOUND = 10,
+    ERROR_OBJECTIVE_NOTFOUND = 11,
+    ERROR_OBJECTIVE_MISSING_DESC = 12,
+    ERROR_OBJECTIVE_MISSING_USES = 13,
+    ERROR_OBJECTIVE_MISSING_STEPS = 14,
+    ERROR_DATASET_NOTFOUND = 15,
+    ERROR_COSTS_NOTFOUND = 16,
+    ERROR_COSTS_MISSING = 17,
 
-# Restart kernel cell
-#    Restart code cell required
-#    Restart code cell not found
-ERROR_RESTART_NOTFOUND = 23
-ERROR_RESTART_CODE_NOTFOUND = 24
+    # Installation cell
+    #   Installation cell required
+    #   Wrong heading for installation cell
+    #   Installation code cell not found
+    #   pip3 required
+    #   option -q required
+    #   option {USER_FLAG} required
+    #   installation code cell not match template
+    #   all packages must be installed as a single pip3
+    ERROR_INSTALLATION_NOTFOUND = 18,
+    ERROR_INSTALLATION_HEADING = 19,
+    ERROR_INSTALLATION_CODE_NOTFOUND = 20,
+    ERROR_INSTALLATION_PIP3 = 21,
+    ERROR_INSTALLATION_QUIET = 22,
+    ERROR_INSTALLATION_USER_FLAG = 23,
+    ERROR_INSTALLATION_CODE_TEMPLATE = 24,
+    ERROR_INSTALLATION_SINGLE_PIP3 = 25,
 
-# Before you begin cell
-#    Before you begin cell required
-#    Before you begin cell incomplete
-ERROR_BEFOREBEGIN_NOTFOUND = 25
-ERROR_BEFOREBEGIN_INCOMPLETE = 26
+    # Restart kernel cell
+    #    Restart code cell required
+    #    Restart code cell not found
+    ERROR_RESTART_NOTFOUND = 23,
+    ERROR_RESTART_CODE_NOTFOUND = 24,
 
-# Set Project ID
-#    Set project ID cell required
-#    Set project ID code cell not found
-#    Set project ID not match template
-ERROR_PROJECTID_NOTFOUND = 27
-ERROR_PROJECTID_CODE_NOTFOUND = 28
-ERROR_PROJECTID_TEMPLATE = 29
+    # Before you begin cell
+    #    Before you begin cell required
+    #    Before you begin cell incomplete
+    ERROR_BEFOREBEGIN_NOTFOUND = 25,
+    ERROR_BEFOREBEGIN_INCOMPLETE = 26,
 
-# Technical Writer Rules
-ERROR_TWRULE_TODO = 51
-ERROR_TWRULE_FIRSTPERSON = 52
-ERROR_TWRULE_FUTURETENSE = 53
-ERROR_TWRULE_BRANDING = 54
+    # Set Project ID
+    #    Set project ID cell required
+    #    Set project ID code cell not found
+    #    Set project ID not match template
+    ERROR_PROJECTID_NOTFOUND = 27,
+    ERROR_PROJECTID_CODE_NOTFOUND = 28,
+    ERROR_PROJECTID_TEMPLATE = 29,
+
+    # Technical Writer Rules
+    ERROR_TWRULE_TODO = 51,
+    ERROR_TWRULE_FIRSTPERSON = 52,
+    ERROR_TWRULE_FUTURETENSE = 53,
+    ERROR_TWRULE_BRANDING = 54,
+
+    ERROR_EMPTY_CALL = 101
 
-ERROR_PLACEHOLDER = 100
-ERROR_EMPTY_CALL = ERROR_PLACEHOLDER + 1
 
 # globals
 num_errors = 0
@@ -214,133 +217,38 @@ def parse_notebook(path: str) -> None:
         nth = parse_recommendations(path, cells, nth)
 
         # Dataset
-        cell, nth = get_cell(path, cells, nth)
-        if not cell['source'][0].startswith("### Dataset") and not cell['source'][0].startswith("### Model") and not cell['source'][0].startswith("### Embedding"):
-            report_error(path, ERROR_DATASET_NOTFOUND, "Dataset/Model section not found")
+        nth = parse_dataset(path, cells, nth)
             
         # Costs
-        cell, nth = get_cell(path, cells, nth)
-        if not cell['source'][0].startswith("### Costs"):
-            report_error(path, ERROR_COSTS_NOTFOUND, "Costs section not found")
-        else:
-            text = ''
-            for line in cell['source']:
-                text += line
-            if 'BQ' in costs and 'BigQuery' not in text:
-                report_error(path, ERROR_COSTS_MISSING, 'Costs section missing reference to BiqQuery')
-            if 'Vertex' in costs and 'Vertex' not in text:
-                report_error(path, ERROR_COSTS_MISSING, 'Costs section missing reference to Vertex')
-            if 'Dataflow' in costs and 'Dataflow' not in text:    
-                report_error(path, ERROR_COSTS_MISSING, 'Costs section missing reference to Dataflow')
+        nth = parse_costs(path, cells, nth, costs)
+
                 
         # (optional) Setup local environment
-        cell, nth = get_cell(path, cells, nth)
-        if cell['source'][0].startswith('### Set up your local development environment'):
-            cell, nth = get_cell(path, cells, nth)
-            if cell['source'][0].startswith('**Otherwise**, make sure your environment meets'):
-                cell, nth = get_cell(path, cells, nth)
+        nth = parse_setuplocal(path, cells, nth)
                 
         # (optional) Helper functions
-        if 'helper' in cell['source'][0]:
-            cell, nth = get_cell(path, cells, nth)
-            cell, nth = get_cell(path, cells, nth)
+        nth = parse_helpers(path, cells, nth)
                 
         # Installation
-        if not cell['source'][0].startswith("## Install"):
-            if cell['source'][0].startswith("### Install"):
-                report_error(path, ERROR_INSTALLATION_HEADING, "Installation section needs to be H2 heading")
-            else:
-                report_error(path, ERROR_INSTALLATION_NOTFOUND, "Installation section not found")
-        else:
-            cell, nth = get_cell(path, cells, nth)
-            if cell['cell_type'] != 'code':
-                report_error(path, ERROR_INSTALLATION_NOTFOUND, "Installation section not found")
-            else:
-                if cell['source'][0].startswith('! mkdir'):
-                    cell, nth = get_cell(path, cells, nth)
-                if 'requirements.txt' in cell['source'][0]:
-                    cell, nth = get_cell(path, cells, nth)
-                    
-                text = ''
-                for line in cell['source']:
-                    text += line
-                    if 'pip ' in line:
-                        if 'pip3' not in line:
-                            report_error(path, ERROR_INSTALLATION_PIP3, "Installation code section: use pip3")
-                        if line.endswith('\\\n'):
-                            continue
-                        if '-q' not in line and '--quiet' not in line :
-                            report_error(path, ERROR_INSTALLATION_QUIET, "Installation code section: use -q with pip3")
-                        if 'USER_FLAG' not in line and 'sh(' not in line:
-                            report_error(path, ERROR_INSTALLATION_USER_FLAG, "Installation code section: use {USER_FLAG} with pip3")
-                if 'if IS_WORKBENCH_NOTEBOOK:' not in text:
-                    report_error(path, ERROR_INSTALLATION_CODE_TEMPLATE, "Installation code section out of date (see template)")
-            
+        nth = parse_installation(path, cells, nth)
+        
         # Restart kernel
-        while True:
-            cont = False
-            cell, nth = get_cell(path, cells, nth)
-            for line in cell['source']:
-                if 'pip' in line:
-                    report_error(path, ERROR_INSTALLATION_SINGLE_PIP3, f"All pip installations must be in a single code cell: {line}")
-                    cont = True
-                    break
-            if not cont:
-                break
-           
-        if not cell['source'][0].startswith("### Restart the kernel"):
-            report_error(path, ERROR_RESTART_NOTFOUND, "Restart the kernel section not found")
-        else:
-            cell, nth = get_cell(path, cells, nth) # code cell
-            if cell['cell_type'] != 'code':
-                report_error(path, ERROR_RESTART_CODE_NOTFOUND, "Restart the kernel code section not found")
+        nth = parse_restart(path, cells, nth)
+
                 
         # (optional) Check package versions
-        cell, nth = get_cell(path, cells, nth)
-        if cell['source'][0].startswith('#### Check package versions'):
-            cell, nth = get_cell(path, cells, nth) # code cell
-            cell, nth = get_cell(path, cells, nth) # next text cell
+        nth = parse_versions(path, cells, nth)
             
         # Before you begin
-        if not cell['source'][0].startswith("## Before you begin"):
-            report_error(path, ERROR_BEFOREBEGIN_NOTFOUND, "Before you begin section not found")
-        else:
-            # maybe one or two cells
-            if len(cell['source']) < 2:
-                cell, nth = get_cell(path, cells, nth)
-                if not cell['source'][0].startswith("### Set up your Google Cloud project"):
-                    report_error(path, ERROR_BEFOREBEGIN_INCOMPLETE, "Before you begin section incomplete")
+        nth = parse_beforebegin(path, cells, nth)
+
               
         # (optional) enable APIs
-        cell, nth = get_cell(path, cells, nth)
-        if cell['source'][0].startswith("### Enable APIs"):
-            cell, nth = get_cell(path, cells, nth) # code cell
-            cell, nth = get_cell(path, cells, nth) # next text cell
+        th = parse_enableapis(path, cells, nth)
             
         # Set project ID
-        if not cell['source'][0].startswith('#### Set your project ID'):
-            report_error(path, ERROR_PROJECTID_NOTFOUND, "Set project ID section not found")
-        else: 
-            cell, nth = get_cell(path, cells, nth)
-            if cell['cell_type'] != 'code':
-                report_error(path, ERROR_PROJECTID_CODE_NOTFOUND, "Set project ID code section not found")
-            elif not cell['source'][0].startswith('PROJECT_ID = "[your-project-id]"'):
-                report_error(path, 33, f"Set project ID not match template: {line}")
-            
-            cell, nth = get_cell(path, cells, nth)
-            if cell['cell_type'] != 'code' or 'or PROJECT_ID == "[your-project-id]":' not in cell['source'][0]:
-                report_error(path, ERROR_PROJECTID_TEMPLATE, f"Set project ID not match template: {line}")  
-            
-            cell, nth = get_cell(path, cells, nth)
-            if cell['cell_type'] != 'code' or '! gcloud config set project' not in cell['source'][0]:
-                report_error(path, ERROR_PROJECTID_TEMPLATE, f"Set project ID not match template: {line}")   
-            
-        '''
-        # Region
-        cell, nth = get_cell(path, cells, nth)
-        if cell['source'][0].startswith("### Region"): 
-            report_error(path, 34, "Region section not found")
-        '''
+        nth = parse_setproject(path, cells, nth)
+
 
 
 def parse_copyright(path: str,
@@ -357,7 +265,7 @@ def parse_copyright(path: str,
     """
     cell, nth = get_cell(path, cells, nth)
     if not 'Copyright' in cell['source'][0]:
-        report_error(path, ERROR_COPYRIGHT, "missing copyright cell")
+        report_error(path, ErrorCodes.ERROR_COPYRIGHT, "missing copyright cell")
     return nth
 
 
@@ -393,11 +301,12 @@ def parse_title(path: str,
     """
     cell, nth = get_cell(path, cells, nth)
     if not cell['source'][0].startswith('# '):
-        report_error(path, ERROR_TITLE_HEADING, "title cell must start with H1 heading")
+        report_error(path, ErrorCodes.ERROR_TITLE_HEADING, "title cell must start with H1 heading")
         title = ''
     else:
         title = cell['source'][0][2:].strip()
         check_sentence_case(path, title)
+
             
         # H1 title only
         if len(cell['source']) == 1:
@@ -434,7 +343,7 @@ def parse_links(path: str,
                 # if new notebook
                 derived_link = os.path.join('https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/', path)
                 if git_link != derived_link:
-                    report_error(path, ERROR_LINK_GIT_BAD, f"bad GitHub link: {git_link}")
+                    report_error(path, ErrorCodes.ERROR_LINK_GIT_BAD, f"bad GitHub link: {git_link}")
                     
         if '<a href="https://colab.research.google.com/' in line:
             colab_link = 'https://github.com/' + line.strip()[50:-2].replace('" target="_blank', '')
@@ -444,7 +353,7 @@ def parse_links(path: str,
                 # if new notebook
                 derived_link = os.path.join('https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks', path)
                 if colab_link != derived_link:
-                    report_error(path, ERROR_LINK_COLAB_BAD, f"bad Colab link: {colab_link}")
+                    report_error(path, ErrorCodes.ERROR_LINK_COLAB_BAD, f"bad Colab link: {colab_link}")
                     
 
         if '<a href="https://console.cloud.google.com/vertex-ai/workbench/' in line:
@@ -454,14 +363,14 @@ def parse_links(path: str,
             except Exception as e:
                 derived_link = os.path.join('https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/', path)
                 if colab_link != workbench_link:
-                    report_error(path, ERROR_LINK_WORKBENCH_BAD, f"bad Workbench link: {workbench_link}")
+                    report_error(path, ErrorCodes.ERROR_LINK_WORKBENCH_BAD, f"bad Workbench link: {workbench_link}")
 
     if 'View on GitHub' not in source or not git_link:
-        report_error(path, ERROR_LINK_GIT_MISSING, 'Missing link for GitHub')
+        report_error(path, ErrorCodes.ERROR_LINK_GIT_MISSING, 'Missing link for GitHub')
     if 'Run in Colab' not in source or not colab_link:
-        report_error(path, ERROR_LINK_COLAB_MISSING, 'Missing link for Colab')    
+        report_error(path, ErrorCodes.ERROR_LINK_COLAB_MISSING, 'Missing link for Colab')    
     if 'Open in Vertex AI Workbench' not in source or not workbench_link:
-        report_error(path, ERROR_LINK_WORKBENCH_MISSING, 'Missing link for Workbench')
+        report_error(path, ErrorCodes.ERROR_LINK_WORKBENCH_MISSING, 'Missing link for Workbench')
     return nth, git_link, colab_link, workbench_link
 
 
@@ -480,7 +389,7 @@ def parse_overview(path: str,
     """
     cell, nth = get_cell(path, cells, nth)
     if not cell['source'][0].startswith("## Overview"):
-        report_error(path, ERROR_OVERVIEW_NOTFOUND, "Overview section not found")
+        report_error(path, ErrorCodes.ERROR_OVERVIEW_NOTFOUND, "Overview section not found")
         
     return nth
 
@@ -505,7 +414,7 @@ def parse_objective(path: str,
     
     cell, nth = get_cell(path, cells, nth)
     if not cell['source'][0].startswith("### Objective"):
-        report_error(path, ERROR_OBJECTIVE_NOTFOUND, "Objective section not found")
+        report_error(path, ErrorCodes.ERROR_OBJECTIVE_NOTFOUND, "Objective section not found")
         return nth, desc, uses, steps, costs
     
     in_desc = True
@@ -549,7 +458,7 @@ def parse_objective(path: str,
                     steps += line
             
     if desc == '':
-        report_error(path, ERROR_OBJECTIVE_MISSING_DESC, "Objective section missing desc")
+        report_error(path, ErrorCodes.ERROR_OBJECTIVE_MISSING_DESC, "Objective section missing desc")
     else:
         desc = desc.lstrip()
         sentences = desc.split('.')
@@ -559,7 +468,7 @@ def parse_objective(path: str,
             desc = desc[22].upper() + desc[23:]
         
     if uses == '':
-        report_error(path, ERROR_OBJECTIVE_MISSING_USES, "Objective section missing uses services list")
+        report_error(path, ErrorCodes.ERROR_OBJECTIVE_MISSING_USES, "Objective section missing uses services list")
     else:
         if 'BigQuery' in uses:
             costs.append('BQ')
@@ -569,7 +478,7 @@ def parse_objective(path: str,
             costs.append('Dataflow')
             
     if steps == '':
-        report_error(path, ERROR_OBJECTIVE_MISSING_STEPS, "Objective section missing steps list")
+        report_error(path, ErrorCodes.ERROR_OBJECTIVE_MISSING_STEPS, "Objective section missing steps list")
             
     return nth, desc, uses, steps, costs
 
@@ -579,7 +488,7 @@ def parse_recommendations(path: str,
                           nth: int) -> int:
     
     """
-    Parse the overview cell
+    Parse the recommendations cell
     
     path: used only for reporting an error
     cells: The content cells (JSON) for the notebook
@@ -595,51 +504,280 @@ def parse_recommendations(path: str,
     return nth - 1
 
 
-def get_cell(path: str, 
-             cells: list, 
-             nth: int) -> (list, int):
-    """
-        Get the next notebook cell.
-        
-        path: used only for reporting an error
-        cells: The content cells (JSON) for the notebook
-        nth: The index of the last cell that was parsed (reviewed).
-        
-        Returns:
-        
-        cell: content of the next cell
-        nth + 1: index of subsequent cell
-    """
-    while empty_cell(path, cells, nth):
-        nth += 1
-        
-    cell = cells[nth]
-    if cell['cell_type'] == 'markdown':
-        check_text_cell(path, cell)
-    return cell, nth + 1
-
-
-def empty_cell(path: str, 
-               cells: list, 
-               nth: int) -> bool:
-    """
-        Check for empty cells
-        
-        path: used only for reporting an error
-        cells: The content cells (JSON) for the notebook
-        nth: The index of the last cell that was parsed (reviewed).
-        
-        Returns:
-        
-        bool: whether cell is empty or not
-    """
-    if len(cells[nth]['source']) == 0:
-        report_error(path, ERROR_EMPTY_CELL, f'empty cell: cell #{nth}')
-        return True
-    else:
-        return False
-
+def parse_dataset(path: str, 
+                  cells: list, 
+                  nth: int) -> int:
     
+    """
+    Parse the dataset cell
+    
+    path: used only for reporting an error
+    cells: The content cells (JSON) for the notebook
+    nth: The index of the last cell that was parsed (reviewed).
+    
+    Returns: cell index
+    """
+    # Dataset
+    cell, nth = get_cell(path, cells, nth)
+    if not cell['source'][0].startswith("### Dataset") and not cell['source'][0].startswith("### Model") and not cell['source'][0].startswith("### Embedding"):
+        report_error(path, ErrorCodes.ERROR_DATASET_NOTFOUND, "Dataset/Model section not found")
+        
+    return nth
+
+def parse_costs(path: str, 
+                cells: list, 
+                nth: int,
+                costs: list) -> int:
+    
+    """
+    Parse the costs cell
+    
+    path: used only for reporting an error
+    cells: The content cells (JSON) for the notebook
+    nth: The index of the last cell that was parsed (reviewed).
+    costs: List of resources used
+    
+    Returns: cell index
+    """
+    # Costs
+    cell, nth = get_cell(path, cells, nth)
+    if not cell['source'][0].startswith("### Costs"):
+        report_error(path, ErrorCodes.ERROR_COSTS_NOTFOUND, "Costs section not found")
+    else:
+        text = ''
+        for line in cell['source']:
+            text += line
+        if 'BQ' in costs and 'BigQuery' not in text:
+            report_error(path, ErrorCodes.ERROR_COSTS_MISSING, 'Costs section missing reference to BiqQuery')
+        if 'Vertex' in costs and 'Vertex' not in text:
+            report_error(path, ErrorCodes.ERROR_COSTS_MISSING, 'Costs section missing reference to Vertex')
+        if 'Dataflow' in costs and 'Dataflow' not in text:    
+            report_error(path, ErrorCodes.ERROR_COSTS_MISSING, 'Costs section missing reference to Dataflow')
+            
+    return nth
+
+
+def parse_setuplocal(path: str, 
+                     cells: list, 
+                     nth: int) -> int:
+    
+    """
+    Parse the (optional) setup local environment cell
+    
+    path: used only for reporting an error
+    cells: The content cells (JSON) for the notebook
+    nth: The index of the last cell that was parsed (reviewed).
+    
+    Returns: cell index
+    """
+    cell, nth = get_cell(path, cells, nth)
+    if not cell['source'][0].startswith('### Set up your local development environment'):
+        return nth - 1
+    cell, nth = get_cell(path, cells, nth)
+    if not cell['source'][0].startswith('**Otherwise**, make sure your environment meets'):
+        return nth - 1
+    return nth
+
+
+def parse_helpers(path: str, 
+                  cells: list, 
+                  nth: int) -> int:
+    
+    """
+    Parse the (optional) helpers text/code cell
+    
+    path: used only for reporting an error
+    cells: The content cells (JSON) for the notebook
+    nth: The index of the last cell that was parsed (reviewed).
+    
+    Returns: cell index
+    """
+    cell, nth = get_cell(path, cells, nth)
+    if 'helper' in cell['source'][0]:
+        return nth + 1 # text and code
+                  
+    return nth - 1
+
+
+def parse_installation(path: str, 
+                       cells: list, 
+                       nth: int) -> int:
+    
+    """
+    Parse the installation cells
+    
+    path: used only for reporting an error
+    cells: The content cells (JSON) for the notebook
+    nth: The index of the last cell that was parsed (reviewed).
+    
+    Returns: cell index
+    """
+    cell, nth = get_cell(path, cells, nth)
+    if not cell['source'][0].startswith("## Install"):
+        if cell['source'][0].startswith("### Install"):
+            report_error(path, ErrorCodes.ERROR_INSTALLATION_HEADING, "Installation section needs to be H2 heading")
+        else:
+            report_error(path, ErrorCodes.ERROR_INSTALLATION_NOTFOUND, "Installation section not found")
+    else:
+        cell, nth = get_cell(path, cells, nth)
+        if cell['cell_type'] != 'code':
+            report_error(path, ErrorCodes.ERROR_INSTALLATION_NOTFOUND, "Installation section not found")
+        else:
+            if cell['source'][0].startswith('! mkdir'):
+                cell, nth = get_cell(path, cells, nth)
+            if 'requirements.txt' in cell['source'][0]:
+                cell, nth = get_cell(path, cells, nth)
+                    
+            text = ''
+            for line in cell['source']:
+                text += line
+                if 'pip ' in line:
+                    if 'pip3' not in line:
+                        report_error(path, ErrorCodes.ERROR_INSTALLATION_PIP3, "Installation code section: use pip3")
+                    if line.endswith('\\\n'):
+                        continue
+                    if '-q' not in line and '--quiet' not in line :
+                        report_error(path, ErrorCodes.ERROR_INSTALLATION_QUIET, "Installation code section: use -q with pip3")
+                    if 'USER_FLAG' not in line and 'sh(' not in line:
+                        report_error(path, ErrorCodes.ERROR_INSTALLATION_USER_FLAG, "Installation code section: use {USER_FLAG} with pip3")
+            if 'if IS_WORKBENCH_NOTEBOOK:' not in text:
+                report_error(path, ErrorCodes.ERROR_INSTALLATION_CODE_TEMPLATE, "Installation code section out of date (see template)")
+                
+    return nth
+
+
+def parse_restart(path: str, 
+                  cells: list, 
+                  nth: int) -> int:
+    
+    """
+    Parse the restart cells
+    
+    path: used only for reporting an error
+    cells: The content cells (JSON) for the notebook
+    nth: The index of the last cell that was parsed (reviewed).
+    
+    Returns: cell index
+    """
+    # Restart kernel
+    while True:
+        cont = False
+        cell, nth = get_cell(path, cells, nth)
+        for line in cell['source']:
+            if 'pip' in line:
+                report_error(path, ErrorCodes.ERROR_INSTALLATION_SINGLE_PIP3, f"All pip installations must be in a single code cell: {line}")
+                cont = True
+                break
+        if not cont:
+            break
+           
+    if not cell['source'][0].startswith("### Restart the kernel"):
+        report_error(path, ErrorCodes.ERROR_RESTART_NOTFOUND, "Restart the kernel section not found")
+    else:
+        cell, nth = get_cell(path, cells, nth) # code cell
+        if cell['cell_type'] != 'code':
+            report_error(path, ErrorCodes.ERROR_RESTART_CODE_NOTFOUND, "Restart the kernel code section not found")
+            
+    return nth
+
+
+def parse_versions(path: str, 
+                   cells: list, 
+                   nth: int) -> int:
+    
+    """
+    Parse the (optional) package versions code/text cell
+    
+    path: used only for reporting an error
+    cells: The content cells (JSON) for the notebook
+    nth: The index of the last cell that was parsed (reviewed).
+    
+    Returns: cell index
+    """
+    cell, nth = get_cell(path, cells, nth)
+    if cell['source'][0].startswith('#### Check package versions'):
+        return nth + 1
+    return nth - 1
+
+
+def parse_beforebegin(path: str, 
+                      cells: list, 
+                      nth: int) -> int:
+    
+    """
+    Parse the before you begin cell
+    
+    path: used only for reporting an error
+    cells: The content cells (JSON) for the notebook
+    nth: The index of the last cell that was parsed (reviewed).
+    
+    Returns: cell index
+    """
+    cell, nth = get_cell(path, cells, nth)
+    if not cell['source'][0].startswith("## Before you begin"):
+        report_error(path, ErrorCodes.ERROR_BEFOREBEGIN_NOTFOUND, "Before you begin section not found")
+    else:
+        # maybe one or two cells
+        if len(cell['source']) < 2:
+            cell, nth = get_cell(path, cells, nth)
+            if not cell['source'][0].startswith("### Set up your Google Cloud project"):
+                report_error(path, ErrorCodes.ERROR_BEFOREBEGIN_INCOMPLETE, "Before you begin section incomplete")
+    return nth
+
+
+def parse_enableapis(path: str, 
+                     cells: list, 
+                     nth: int) -> int:
+    """
+    Parse the (optional) enable apis code/text cell
+    
+    path: used only for reporting an error
+    cells: The content cells (JSON) for the notebook
+    nth: The index of the last cell that was parsed (reviewed).
+    
+    Returns: cell index
+    """
+    cell, nth = get_cell(path, cells, nth)
+    if cell['source'][0].startswith("### Enable APIs"):
+        return nth + 1
+                     
+    return nth - 1
+
+
+def parse_setproject(path: str, 
+                      cells: list, 
+                      nth: int) -> int:
+    
+    """
+    Parse the set project cells
+    
+    path: used only for reporting an error
+    cells: The content cells (JSON) for the notebook
+    nth: The index of the last cell that was parsed (reviewed).
+    
+    Returns: cell index
+    """
+    cell, nth = get_cell(path, cells, nth)
+    if not cell['source'][0].startswith('#### Set your project ID'):
+        report_error(path, ErrorCodes.ERROR_PROJECTID_NOTFOUND, "Set project ID section not found")
+    else: 
+        cell, nth = get_cell(path, cells, nth)
+        if cell['cell_type'] != 'code':
+            report_error(path, ErrorCodes.ERROR_PROJECTID_CODE_NOTFOUND, "Set project ID code section not found")
+        elif not cell['source'][0].startswith('PROJECT_ID = "[your-project-id]"'):
+            report_error(path, ErrorCodes.ERROR_PROJECTID_TEMPLATE, f"Set project ID not match template")
+            
+        cell, nth = get_cell(path, cells, nth)
+        if cell['cell_type'] != 'code' or 'or PROJECT_ID == "[your-project-id]":' not in cell['source'][0]:
+            report_error(path, ErrorCodes.ERROR_PROJECTID_TEMPLATE, f"Set project ID not match template")  
+            
+        cell, nth = get_cell(path, cells, nth)
+        if cell['cell_type'] != 'code' or '! gcloud config set project' not in cell['source'][0]:
+            report_error(path, ErrorCodes.ERROR_PROJECTID_TEMPLATE, f"Set project ID not match template")
+            
+    return nth
+
+
 def check_text_cell(path: str,
                     cell: list) -> None:
     """
@@ -686,10 +824,12 @@ def check_text_cell(path: str,
         'Tensorflow': 'TensorFlow',
         'Tensorboard': 'TensorBoard',
         'Google Cloud Notebooks': 'Vertex AI Workbench Notebooks',
-        'BQ': 'BigQuery',
+        'BQ ': 'BigQuery',
+        'BQ.': 'BigQuery',
         'Bigquery': 'BigQuery',
         'BQML': 'BigQuery ML',
-        'GCS': 'Cloud Storage',
+        'GCS ': 'Cloud Storage',
+        'GCS.': 'Cloud Storage',
         'Google Cloud Storage': 'Cloud Storage',
         'Pytorch': 'PyTorch',
         'Sklearn': 'scikit-learn',
@@ -697,16 +837,20 @@ def check_text_cell(path: str,
     }
     
     for line in cell['source']:
+        # HTML code
+        if '<a ' in line:
+            continue
+            
         if 'TODO' in line or 'WIP' in line:
-            report_error(path, ERROR_TWRULE_TODO, f'TODO in cell: {line}')
+            report_error(path, ErrorCodes.ERROR_TWRULE_TODO, f'TODO in cell: {line}')
         if 'we ' in line.lower() or "let's" in line.lower() in line.lower():
-            report_error(path, ERROR_TWRULE_FIRSTPERSON, f'Do not use first person (e.g., we), replace with 2nd person (you): {line}')
+            report_error(path, ErrorCodes.ERROR_TWRULE_FIRSTPERSON, f'Do not use first person (e.g., we), replace with 2nd person (you): {line}')
         if 'will' in line.lower() or 'would' in line.lower():
-            report_error(path, ERROR_TWRULE_FUTURETENSE, f'Do not use future tense (e.g., will), replace with present tense: {line}')
+            report_error(path, ErrorCodes.ERROR_TWRULE_FUTURETENSE, f'Do not use future tense (e.g., will), replace with present tense: {line}')
             
         for mistake, brand in branding.items():
             if mistake in line:
-                report_error(path, ERROR_TWRULE_BRANDING, f"Branding {brand}: {line}")
+                report_error(path, ErrorCodes.ERROR_TWRULE_BRANDING, f"Branding {mistake} -> {brand}: {line}")
 
 
 def check_sentence_case(path: str, 
@@ -725,18 +869,63 @@ def check_sentence_case(path: str,
     
     words = heading.split(' ')
     if not words[0][0].isupper():
-        report_error(path, ERROR_HEADING_CAP, f"heading must start with capitalized word: {words[0]}")
+        report_error(path, ErrorCodes.ERROR_HEADING_CAP, f"heading must start with capitalized word: {words[0]}")
         
     for word in words[1:]:
         word = word.replace(':', '').replace('(', '').replace(')', '')
         if word in ACRONYMS:
             continue
         if word.isupper():
-            report_error(path, ERROR_HEADING_CASE, f"heading is not sentence case: {word}")
+            report_error(path, ErrorCodes.ERROR_HEADING_CASE, f"heading is not sentence case: {word}")
+
+
+def get_cell(path: str, 
+             cells: list, 
+             nth: int) -> (list, int):
+    """
+        Get the next notebook cell.
+        
+        path: used only for reporting an error
+        cells: The content cells (JSON) for the notebook
+        nth: The index of the last cell that was parsed (reviewed).
+        
+        Returns:
+        
+        cell: content of the next cell
+        nth + 1: index of subsequent cell
+    """
+    while empty_cell(path, cells, nth):
+        nth += 1
+        
+    cell = cells[nth]
+    if cell['cell_type'] == 'markdown':
+        check_text_cell(path, cell)
+    return cell, nth + 1
+
+
+def empty_cell(path: str, 
+               cells: list, 
+               nth: int) -> bool:
+    """
+        Check for empty cells
+        
+        path: used only for reporting an error
+        cells: The content cells (JSON) for the notebook
+        nth: The index of the last cell that was parsed (reviewed).
+        
+        Returns:
+        
+        bool: whether cell is empty or not
+    """
+    if len(cells[nth]['source']) == 0:
+        report_error(path, ErrorCodes.ERROR_EMPTY_CELL, f'empty cell: cell #{nth}')
+        return True
+    else:
+        return False
 
 
 def report_error(notebook: str, 
-                 code: str,
+                 code: ErrorCodes,
                  errmsg: str) -> None:
     """
     Report an error.
@@ -750,6 +939,7 @@ def report_error(notebook: str,
     global num_errors
     
     if args.errors:
+        code = code.value[0]
         if args.errors_codes:
             if str(code) not in args.errors_codes:
                 return
@@ -761,10 +951,17 @@ def report_error(notebook: str,
             num_errors += 1
 
 
-
-
-
-def add_index(path, tag, title, desc, uses, steps, git_link, colab_link, workbench_link):
+def add_index(path: str, 
+              tag: str, 
+              title : str, 
+              desc: str, 
+              uses: str, 
+              steps: str, 
+              git_link: str, 
+              colab_link: str, 
+              workbench_link: str) -> None:
+    """
+    """
     global last_tag
     
     if not args.web and not args.repo:
@@ -863,5 +1060,5 @@ else:
 
 if args.web:
     print('</table>\n')
-    
+
 exit(num_errors)