sync with ai-platform-samples (#25)

* sync with ai-platform-samples

* fix: update links

.cloud-build/CheckPythonVersion.py

@@ -1,3 +1,18 @@
+#!/usr/bin/env python
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import sys
 
 MINIMUM_MAJOR_VERSION = 3

.cloud-build/ExecuteChangedNotebooks.py

@@ -1,5 +1,5 @@
-#!/bin/bash
-# Copyright 2019 Google LLC
+#!/usr/bin/env python
+# Copyright 2021 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,21 +12,114 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
 import argparse
+import dataclasses
+import datetime
+import functools
 import pathlib
+import os
 import subprocess
 from pathlib import Path
-from typing import Dict, List, Optional
+from typing import List, Optional
+import concurrent
+from tabulate import tabulate
 
 import ExecuteNotebook
 
 
+def str2bool(v):
+    if isinstance(v, bool):
+        return v
+    if v.lower() in ("yes", "true", "t", "y", "1"):
+        return True
+    elif v.lower() in ("no", "false", "f", "n", "0"):
+        return False
+    else:
+        raise argparse.ArgumentTypeError("Boolean value expected.")
+
+
+def format_timedelta(delta: datetime.timedelta) -> str:
+    """Formats a timedelta duration to [N days] %H:%M:%S format"""
+    seconds = int(delta.total_seconds())
+
+    secs_in_a_day = 86400
+    secs_in_a_hour = 3600
+    secs_in_a_min = 60
+
+    days, seconds = divmod(seconds, secs_in_a_day)
+    hours, seconds = divmod(seconds, secs_in_a_hour)
+    minutes, seconds = divmod(seconds, secs_in_a_min)
+
+    time_fmt = f"{hours:02d}:{minutes:02d}:{seconds:02d}"
+
+    if days > 0:
+        suffix = "s" if days > 1 else ""
+        return f"{days} day{suffix} {time_fmt}"
+
+    return time_fmt
+
+
+@dataclasses.dataclass
+class NotebookExecutionResult:
+    notebook: str
+    duration: datetime.timedelta
+    is_pass: bool
+    error_message: Optional[str]
+
+
+def execute_notebook(
+    artifacts_path: str,
+    variable_project_id: str,
+    variable_region: str,
+    should_log_output: bool,
+    should_use_new_kernel: bool,
+    notebook: str,
+) -> NotebookExecutionResult:
+    print(f"Running notebook: {notebook}")
+
+    result = NotebookExecutionResult(
+        notebook=notebook,
+        duration=datetime.timedelta(seconds=0),
+        is_pass=False,
+        error_message=None,
+    )
+
+    # TODO: Handle cases where multiple notebooks have the same name
+    time_start = datetime.datetime.now()
+    try:
+        ExecuteNotebook.execute_notebook(
+            notebook_file_path=notebook,
+            output_file_folder=artifacts_path,
+            replacement_map={
+                "PROJECT_ID": variable_project_id,
+                "REGION": variable_region,
+            },
+            should_log_output=should_log_output,
+            should_use_new_kernel=should_use_new_kernel,
+        )
+        result.duration = datetime.datetime.now() - time_start
+        result.is_pass = True
+        print(f"{notebook} PASSED in {format_timedelta(result.duration)}.")
+    except Exception as error:
+        result.duration = datetime.datetime.now() - time_start
+        result.is_pass = False
+        result.error_message = str(error)
+        print(
+            f"{notebook} FAILED in {format_timedelta(result.duration)}: {result.error_message}"
+        )
+
+    return result
+
+
 def run_changed_notebooks(
     test_paths_file: str,
+    base_branch: Optional[str],
     output_folder: str,
     variable_project_id: str,
     variable_region: str,
-    base_branch: Optional[str],
+    should_parallelize: bool,
+    should_use_separate_kernels: bool,
 ):
     """
     Run the notebooks that exist under the folders defined in the test_paths_file.
@@ -49,6 +142,13 @@ def run_changed_notebooks(
             Required. The value for PROJECT_ID to inject into notebooks.
         variable_region (str):
             Required. The value for REGION to inject into notebooks.
+        should_parallelize (bool):
+            Required. Should run notebooks in parallel using a thread pool as opposed to in sequence.
+        should_use_separate_kernels (bool):
+            Note: Dependencies don't install correctly when this is set to True
+            See https://github.com/nteract/papermill/issues/625
+
+            Required. Should run each notebook in a separate and independent virtual environment.
     """
 
     test_paths = []
@@ -84,41 +184,68 @@ def run_changed_notebooks(
     artifacts_path.joinpath("success").mkdir(parents=True, exist_ok=True)
     artifacts_path.joinpath("failure").mkdir(parents=True, exist_ok=True)
 
-    passed_notebooks: List[str] = []
-    failed_notebooks: List[str] = []
+    notebook_execution_results: List[NotebookExecutionResult] = []
 
     if len(notebooks) > 0:
         print(f"Found {len(notebooks)} modified notebooks: {notebooks}")
 
-        for notebook in notebooks:
-            print(f"Running notebook: {notebook}")
-
-            # TODO: Handle cases where multiple notebooks have the same name
-            try:
-                ExecuteNotebook.execute_notebook(
-                    notebook_file_path=notebook,
-                    output_file_folder=artifacts_path,
-                    replacement_map={
-                        "PROJECT_ID": variable_project_id,
-                        "REGION": variable_region,
-                    },
+        if should_parallelize and len(notebooks) > 1:
+            print(
+                "Running notebooks in parallel, so no logs will be displayed. Please wait..."
+            )
+            with concurrent.futures.ThreadPoolExecutor(max_workers=None) as executor:
+                notebook_execution_results = list(
+                    executor.map(
+                        functools.partial(
+                            execute_notebook,
+                            artifacts_path,
+                            variable_project_id,
+                            variable_region,
+                            False,
+                            should_use_separate_kernels,
+                        ),
+                        notebooks,
+                    )
                 )
-                print(f"Notebook finished successfully.")
-                passed_notebooks.append(notebook)
-            except Exception as error:
-                print(f"Notebook finished with failure: {error}")
-                failed_notebooks.append(notebook)
+        else:
+            notebook_execution_results = [
+                execute_notebook(
+                    artifacts_path=artifacts_path,
+                    variable_project_id=variable_project_id,
+                    variable_region=variable_region,
+                    notebook=notebook,
+                    should_log_output=True,
+                    should_use_new_kernel=should_use_separate_kernels,
+                )
+                for notebook in notebooks
+            ]
     else:
         print("No notebooks modified in this pull request.")
 
-    if len(failed_notebooks) > 0:
-        print(f"{len(failed_notebooks)} notebooks failed:")
-        print(failed_notebooks)
-        print(f"{len(passed_notebooks)} notebooks passed:")
-        print(passed_notebooks)
-    elif len(passed_notebooks) > 0:
-        print("All notebooks executed successfully:")
-        print(passed_notebooks)
+    print("\n=== RESULTS ===\n")
+
+    notebooks_sorted = sorted(
+        notebook_execution_results,
+        key=lambda result: result.is_pass,
+        reverse=True,
+    )
+    # Print results
+    print(
+        tabulate(
+            [
+                [
+                    os.path.basename(os.path.normpath(result.notebook)),
+                    "PASSED" if result.is_pass else "FAILED",
+                    format_timedelta(result.duration),
+                    result.error_message or "--",
+                ]
+                for result in notebooks_sorted
+            ],
+            headers=["file", "status", "duration", "error"],
+        )
+    )
+
+    print("\n=== END RESULTS===\n")
 
 
 parser = argparse.ArgumentParser(description="Run changed notebooks.")
@@ -152,6 +279,27 @@ parser.add_argument(
     required=True,
 )
 
+# Note: Dependencies don't install correctly when this is set to True
+parser.add_argument(
+    "--should_parallelize",
+    type=str2bool,
+    nargs="?",
+    const=True,
+    default=False,
+    help="Should run notebooks in parallel.",
+)
+
+# Note: This isn't guaranteed to work correctly due to existing Papermill issue
+# See https://github.com/nteract/papermill/issues/625
+parser.add_argument(
+    "--should_use_separate_kernels",
+    type=str2bool,
+    nargs="?",
+    const=True,
+    default=False,
+    help="(Experimental) Should run each notebook in a separate and independent virtual environment.",
+)
+
 args = parser.parse_args()
 run_changed_notebooks(
     test_paths_file=args.test_paths_file,
@@ -159,4 +307,6 @@ run_changed_notebooks(
     output_folder=args.output_folder,
     variable_project_id=args.variable_project_id,
     variable_region=args.variable_region,
+    should_parallelize=args.should_parallelize,
+    should_use_separate_kernels=args.should_use_separate_kernels,
 )

.cloud-build/ExecuteNotebook.py

@@ -1,20 +1,115 @@
+#!/usr/bin/env python
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
 import sys
 import nbformat
-from nbconvert.preprocessors import ExecutePreprocessor, CellExecutionError
 import os
 import errno
 from NotebookProcessors import RemoveNoExecuteCells, UpdateVariablesPreprocessor
-from typing import Dict
+from typing import Dict, Tuple
+import papermill as pm
+import shutil
+import virtualenv
+import uuid
+from jupyter_client.kernelspecapp import KernelSpecManager
 
 # This script is used to execute a notebook and write out the output notebook.
 # The replaces calling the nbconvert via command-line, which doesn't write the output notebook correctly when there are errors during execution.
 
+STAGING_FOLDER = "staging"
+ENVIRONMENTS_PATH = "environments"
+KERNELS_SPECS_PATH = "kernel_specs"
+
+
+def create_and_install_kernel() -> Tuple[str, str]:
+    # Create environment
+    kernel_name = str(uuid.uuid4())
+    env_name = f"{ENVIRONMENTS_PATH}/{kernel_name}"
+    # venv.create(env_name, system_site_packages=True, with_pip=True)
+    virtualenv.cli_run([env_name, "--system-site-packages"])
+
+    # Create kernel spec
+    kernel_spec = {
+        "argv": [
+            f"{env_name}/bin/python",
+            "-m",
+            "ipykernel_launcher",
+            "-f",
+            "{connection_file}",
+        ],
+        "display_name": "Python 3",
+        "language": "python",
+    }
+    kernel_spec_folder = os.path.join(KERNELS_SPECS_PATH, kernel_name)
+    kernel_spec_file = os.path.join(kernel_spec_folder, "kernel.json")
+
+    # Create kernel spec folder
+    if not os.path.exists(os.path.dirname(kernel_spec_file)):
+        try:
+            os.makedirs(os.path.dirname(kernel_spec_file))
+        except OSError as exc:  # Guard against race condition
+            if exc.errno != errno.EEXIST:
+                raise
+
+    with open(kernel_spec_file, mode="w", encoding="utf-8") as f:
+        json.dump(kernel_spec, f)
+
+    # Install kernel
+    kernel_spec_manager = KernelSpecManager()
+    kernel_spec_manager.install_kernel_spec(
+        source_dir=kernel_spec_folder, kernel_name=kernel_name
+    )
+
+    return kernel_name, env_name
+
 
 def execute_notebook(
-    notebook_file_path: str, output_file_folder: str, replacement_map: Dict[str, str]
+    notebook_file_path: str,
+    output_file_folder: str,
+    replacement_map: Dict[str, str],
+    should_log_output: bool,
+    should_use_new_kernel: bool,
 ):
+    # Create staging directory if it doesn't exist
+    staging_file_path = f"{STAGING_FOLDER}/{notebook_file_path}"
+    if not os.path.exists(os.path.dirname(staging_file_path)):
+        try:
+            os.makedirs(os.path.dirname(staging_file_path))
+        except OSError as exc:  # Guard against race condition
+            if exc.errno != errno.EEXIST:
+                raise
+
     file_name = os.path.basename(os.path.normpath(notebook_file_path))
 
+    # Create environments folder
+    if not os.path.exists(ENVIRONMENTS_PATH):
+        try:
+            os.makedirs(ENVIRONMENTS_PATH)
+        except OSError as exc:  # Guard against race condition
+            if exc.errno != errno.EEXIST:
+                raise
+
+    # Create and install kernel
+    kernel_name = next(
+        iter(KernelSpecManager().find_kernel_specs().keys()), None
+    )  # Find first existing kernel and use as default
+    env_name = None
+    if should_use_new_kernel:
+        kernel_name, env_name = create_and_install_kernel()
+
     # Read notebook
     with open(notebook_file_path) as f:
         nb = nbformat.read(f, as_version=4)
@@ -28,7 +123,6 @@ def execute_notebook(
         update_variables_preprocessor = UpdateVariablesPreprocessor(
             replacement_map=replacement_map
         )
-        execute_preprocessor = ExecutePreprocessor(timeout=-1, kernel_name="python3")
 
         # Use no-execute preprocessor
         (
@@ -38,16 +132,33 @@ def execute_notebook(
 
         (nb, resources) = update_variables_preprocessor.preprocess(nb, resources)
 
-        # Execute notebook
-        out = execute_preprocessor.preprocess(nb, resources)
+        # print(f"Staging modified notebook to: {staging_file_path}")
+        with open(staging_file_path, mode="w", encoding="utf-8") as f:
+            nbformat.write(nb, f)
 
-    except Exception as error:
-        out = None
-        print(f"Error executing the notebook: {notebook_file_path}.\n\n")
+        # Execute notebook
+        pm.execute_notebook(
+            input_path=staging_file_path,
+            output_path=staging_file_path,
+            kernel_name=kernel_name,
+            progress_bar=should_log_output,
+            request_save_on_cell_execute=should_log_output,
+            log_output=should_log_output,
+            stdout_file=sys.stdout if should_log_output else None,
+            stderr_file=sys.stderr if should_log_output else None,
+        )
+    except Exception:
+        # print(f"Error executing the notebook: {notebook_file_path}.\n\n")
         has_error = True
 
         raise
+
     finally:
+        # Clear env
+        if env_name is not None:
+            shutil.rmtree(path=env_name)
+
+        # Copy execute notebook
         output_file_path = os.path.join(
             output_file_folder, "failure" if has_error else "success", file_name
         )
@@ -60,6 +171,5 @@ def execute_notebook(
                 if exc.errno != errno.EEXIST:
                     raise
 
-        print(f"Writing output to: {output_file_path}")
-        with open(output_file_path, mode="w", encoding="utf-8") as f:
-            nbformat.write(nb, f)
+        # print(f"Writing output to: {output_file_path}")
+        shutil.move(staging_file_path, output_file_path)

.cloud-build/NotebookProcessors.py

@@ -1,3 +1,18 @@
+#!/usr/bin/env python
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 from nbconvert.preprocessors import Preprocessor
 from typing import Dict
 import UpdateNotebookVariables

.cloud-build/UpdateNotebookVariables.py

@@ -1,3 +1,18 @@
+#!/usr/bin/env python
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import re
 
 """

.cloud-build/cleanup/cleanup-cloudbuild.yaml

@@ -0,0 +1,8 @@
+steps:  
+  # Install Python dependencies and run cleanup script
+  - name: ${_PYTHON_IMAGE}
+    entrypoint: /bin/sh
+    args:
+    - -c
+    - 'python3 -m pip install -U -r .cloud-build/cleanup/cleanup-requirements.txt && python3 .cloud-build/cleanup/cleanup.py'
+timeout: 86400s

.cloud-build/cleanup/cleanup-requirements.txt

@@ -0,0 +1 @@
+google-cloud-aiplatform

.cloud-build/cleanup/cleanup.py

@@ -0,0 +1,42 @@
+from typing import List
+from resource_cleanup_manager import (
+    ResourceCleanupManager,
+    DatasetResourceCleanupManager,
+    EndpointResourceCleanupManager,
+    ModelResourceCleanupManager,
+)
+
+
+def run_cleanup_managers(managers: List[ResourceCleanupManager], is_dry_run: bool):
+    for manager in managers:
+        type_name = manager.type_name
+
+        print(f"Fetching {type_name}'s...")
+        resources = manager.list()
+        print(f"Found {len(resources)} {type_name}'s")
+        for resource in resources:
+            if not manager.is_deletable(resource):
+                continue
+
+            if is_dry_run:
+                resource_name = manager.resource_name(resource)
+                print(f"Will delete '{type_name}': {resource_name}")
+            else:
+                manager.delete(resource)
+
+        print("")
+
+
+is_dry_run = False
+
+if is_dry_run:
+    print("Starting cleanup in dry run mode...")
+
+# List of all cleanup managers
+managers = [
+    DatasetResourceCleanupManager(),
+    EndpointResourceCleanupManager(),
+    ModelResourceCleanupManager(),
+]
+
+run_cleanup_managers(managers=managers, is_dry_run=is_dry_run)

.cloud-build/cleanup/resource_cleanup_manager.py

@@ -0,0 +1,87 @@
+import abc
+from google.cloud import aiplatform
+from typing import Any
+from proto.datetime_helpers import DatetimeWithNanoseconds
+from google.cloud.aiplatform import base
+
+# If a resource was updated within this number of seconds, do not delete.
+RESOURCE_UPDATE_BUFFER_IN_SECONDS = 60 * 60 * 8
+
+
+class ResourceCleanupManager(abc.ABC):
+    @property
+    @abc.abstractmethod
+    def type_name(str) -> str:
+        pass
+
+    @abc.abstractmethod
+    def list(self) -> Any:
+        pass
+
+    @abc.abstractmethod
+    def resource_name(self, resource: Any) -> str:
+        pass
+
+    @abc.abstractmethod
+    def delete(self, resource: Any):
+        pass
+
+    @abc.abstractmethod
+    def get_seconds_since_modification(self, resource: Any) -> float:
+        pass
+
+    def is_deletable(self, resource: Any) -> bool:
+        time_difference = self.get_seconds_since_modification(resource)
+
+        if self.resource_name(resource).startswith("perm"):
+            print(f"Skipping '{resource}' due to name starting with 'perm'.")
+            return False
+
+        # Check that it wasn't created too recently, to prevent race conditions
+        if time_difference <= RESOURCE_UPDATE_BUFFER_IN_SECONDS:
+            print(
+                f"Skipping '{resource}' due update_time being '{time_difference}', which is less than '{RESOURCE_UPDATE_BUFFER_IN_SECONDS}'."
+            )
+            return False
+
+        return True
+
+
+class VertexAIResourceCleanupManager(ResourceCleanupManager):
+    @property
+    @abc.abstractmethod
+    def vertex_ai_resource(self) -> base.VertexAiResourceNounWithFutureManager:
+        pass
+
+    @property
+    def type_name(self) -> str:
+        return self.vertex_ai_resource._resource_noun
+
+    def list(self) -> Any:
+        return self.vertex_ai_resource.list()
+
+    def resource_name(self, resource: Any) -> str:
+        return resource.display_name
+
+    def delete(self, resource):
+        resource.delete()
+
+    def get_seconds_since_modification(self, resource: Any) -> bool:
+        update_time = resource.update_time
+        current_time = DatetimeWithNanoseconds.now(tz=update_time.tzinfo)
+        return (current_time - update_time).total_seconds()
+
+
+class DatasetResourceCleanupManager(VertexAIResourceCleanupManager):
+    vertex_ai_resource = aiplatform.datasets._Dataset
+
+
+class EndpointResourceCleanupManager(VertexAIResourceCleanupManager):
+    vertex_ai_resource = aiplatform.Endpoint
+
+    def delete(self, resource):
+        resource.delete(force=True)
+
+
+class ModelResourceCleanupManager(VertexAIResourceCleanupManager):
+    vertex_ai_resource = aiplatform.Model

.cloud-build/notebook-execution-test-cloudbuild.yaml

@@ -17,12 +17,16 @@ steps:
     args:
     - -c
     - 'if [ -n "${_BASE_BRANCH}" ]; then git fetch origin "${_BASE_BRANCH}":refs/remotes/origin/"${_BASE_BRANCH}"; else echo "Skipping fetch."; fi'
+  # Install Python dependencies
+  - name: ${_PYTHON_IMAGE}
+    entrypoint: pip
+    args: ['install', '--upgrade',  '--user', '--requirement', '.cloud-build/requirements.txt']
   # Install Python dependencies and run testing script
   - name: ${_PYTHON_IMAGE}
     entrypoint: /bin/sh
     args:
     - -c
-    - 'python3 -m pip install -U -r .cloud-build/requirements.txt && python3 -m pip freeze && python3 .cloud-build/ExecuteChangedNotebooks.py --test_paths_file .cloud-build/test_folders.txt --base_branch "${_FORCED_BASE_BRANCH}" --output_folder ${BUILD_ID} --variable_project_id ${PROJECT_ID} --variable_region ${_GCP_REGION}'
+    - 'python3 -m pip freeze && python3 .cloud-build/ExecuteChangedNotebooks.py --test_paths_file "${_TEST_PATHS_FILE}" --base_branch "${_FORCED_BASE_BRANCH}" --output_folder ${BUILD_ID} --variable_project_id ${PROJECT_ID} --variable_region ${_GCP_REGION}'
     env:
     - 'IS_TESTING=1'
   # Manually copy artifacts to GCS
@@ -30,7 +34,7 @@ steps:
     entrypoint: /bin/sh
     args: 
     - -c
-    - 'if [ $(ls -pR "/workspace/${BUILD_ID}"  | grep -v / | grep -v ^$ | wc -l) -ne 0 ]; then gsutil rsync -r "/workspace/${BUILD_ID}" "gs://${_GCS_ARTIFACTS_BUCKET}/test-artifacts/PR_${_PR_NUMBER}/BUILD_${BUILD_ID}/"; else echo "No artifacts to copy."; fi'
+    - 'if [ $(ls -pR "/workspace/${BUILD_ID}"  | grep -v / | grep -v ^$ | wc -l) -ne 0 ]; then gsutil -m -q rsync -r "/workspace/${BUILD_ID}" "gs://${_GCS_ARTIFACTS_BUCKET}/test-artifacts/PR_${_PR_NUMBER}/BUILD_${BUILD_ID}/"; else echo "No artifacts to copy."; fi'
   # Fail if there is anything in the failure folder
   - name: ${_PYTHON_IMAGE}
     entrypoint: /bin/sh

.cloud-build/requirements.txt

@@ -1,5 +1,8 @@
-ipython 
-jupyter>=1.0 
+ipython>=7.0
+jupyter>=1.0
 nbconvert>=6.0
-numpy
-pandas
+papermill>=2.3
+numpy>=1.19
+pandas>=1.2
+matplotlib>=3.4
+tabulate

.cloud-build/test_folders.txt

@@ -1,3 +1,2 @@
 notebooks/official
-tutorials/official
-notebooks/notebook_template.ipynb
+notebooks/notebook_template.ipynb