debug: force use of newest cloud-build

* updating custom container with PyTorch v1.13

* moved etcd install to custom container build

.cloud-build/execute_changed_notebooks_helper.py

@@ -245,7 +245,7 @@ def process_and_execute_notebook(
         result.logs_bucket = operation_metadata.build.logs_bucket
 
         # Block and wait for the result
-        operation_result = operation.result()
+        operation_result = operation.result(timeout=86400)
 
         result.duration = datetime.datetime.now() - time_start
         result.is_pass = True

community-content/pytorch_efficient_training/README.md

@@ -15,15 +15,19 @@ pip install -r requirements.txt
 * resnet_dp.py - Train ResNet-50 on single node multiple GPUs with `DataParallel` strategy.
 * resnet_ddp.py - Train ResNet-50 on single node multiple GPUs with `DistributedDataParallel` strategy.
 * resnet_ddp_wds.py - Train ResNet-50 on single node multiple GPUs with `DistributedDataParallel` strategy and `Webdataset`.
+* resnet_fsdp.py - Train ResNet-50 on single node multiple GPUs with `FullyShardedDataParallel` strategy.
+* resnet_fsdp_wds.py - Train ResNet-50 on single node multiple GPUs with `FullyShardedDataParallel` strategy and `Webdataset`.
 * shard_imagenet.py - Shard ImagNet individual files into `tar` files.
 
 ## Benchmark
 
 When run the benchmark on Nvidia T4 GPUs using ImageNet validation dataset, you can get the result like:
-Strategy              | Seconds/Epoch - Local Data | Seconds/Epoch - Cloud Data
---------------------- | -------------------------- | --------------------------
-On 1 GPU              | 489                        | 804 (2x slower)
-On 4 GPUs (DP)        | 157                        | 738 (5x slower)
-On 4 GPUs (DDP)       | 134                        | 432 (3x slower)
-On 4 GPUs (DDP + WDS) | 131                        | 133 (same performance)
+Strategy               | Seconds/Epoch - Local Data | Seconds/Epoch - Cloud Data
+---------------------- | -------------------------- | --------------------------
+On 1 GPU               | 489                        | 804 (2x slower)
+On 4 GPUs (DP)         | 157                        | 738 (5x slower)
+On 4 GPUs (DDP)        | 134                        | 432 (3x slower)
+On 4 GPUs (DDP + WDS)  | 131                        | 133 (same performance)
+On 4 GPUs (FSDP)       | 139                        | 353 (3x slower)
+On 4 GPUs (FSDP + WDS) | 138                        | 135 (same performance)
 

community-content/pytorch_efficient_training/resnet_fsdp.py

@@ -0,0 +1,242 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the \"License\");
+# you may not use this file except in compliance with the License.\n",
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an \"AS IS\" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Train resnet on multiple GPUs with FSDP."""
+
+import argparse
+import functools
+import os
+import time
+
+from PIL import Image
+import torch
+from torch import nn
+import torch.distributed as dist
+from torch.distributed.fsdp import FullyShardedDataParallel as FSDP
+from torch.distributed.fsdp.wrap import size_based_auto_wrap_policy
+import torch.multiprocessing as mp
+import torchmetrics
+import torchvision
+from torchvision.models import resnet50
+
+
+class ImageFolder(torchvision.datasets.ImageFolder):
+  """Class for loading imagenet."""
+
+  def __init__(self, image_list_file, transform=None, target_transform=None):
+    self.samples = self._make_dataset(image_list_file)
+    self.loader = self._loader
+
+    self.imgs = self.samples
+    self.targets = [s[1] for s in self.samples]
+
+    self.transform = transform
+    self.target_transform = target_transform
+
+  def _make_dataset(self, image_list_file):
+    items = []
+    with open(image_list_file, 'r') as f:
+      for line in f:
+        item = line.strip().split(' ')
+        items.append((item[0], int(item[1])))
+    return items
+
+  def _loader(self, image_path):
+    with open(image_path, 'rb') as f:
+      img = Image.open(f)
+      img = img.convert('RGB')
+      return img
+
+
+def train(model, device, dataloader, optimizer):
+  model.train()
+  for image, target in dataloader:
+    image = image.to(device, non_blocking=True)
+    target = target.to(device, non_blocking=True)
+    pred = model(image)
+    # pred.shape (N, C), target.shape (N)
+    loss = nn.functional.cross_entropy(pred, target)
+    optimizer.zero_grad()
+    loss.backward()
+    optimizer.step()
+  return loss
+
+
+def evaluate(model, device, dataloader, metric):
+  model.eval()
+  with torch.no_grad():
+    for image, target in dataloader:
+      image = image.to(device, non_blocking=True)
+      target = target.to(device, non_blocking=True)
+      pred = model(image)
+      metric.update(pred, target)
+  accuracy = metric.compute()
+  metric.reset()
+  return accuracy
+
+
+def worker(gpu, args):
+  """Run training and evaluation."""
+  # Init process group.
+  print(f'Initiating process {gpu}')
+  dist.init_process_group(
+      backend='nccl',
+      init_method='env://',
+      world_size=args.gpus,
+      rank=gpu)
+
+  # Create train dataloader.
+  train_dataset = ImageFolder(
+      image_list_file=args.train_data_path,
+      transform=torchvision.transforms.Compose([
+          torchvision.transforms.RandomResizedCrop(224),
+          torchvision.transforms.RandomHorizontalFlip(),
+          torchvision.transforms.ToTensor(),
+          torchvision.transforms.Normalize(
+              mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
+      ]))
+  train_sampler = torch.utils.data.distributed.DistributedSampler(
+      train_dataset, num_replicas=args.gpus, rank=gpu)
+  train_dataloader = torch.utils.data.DataLoader(
+      dataset=train_dataset,
+      batch_size=args.train_batch_size,
+      shuffle=False,
+      num_workers=args.dataloader_num_workers,
+      pin_memory=True,
+      sampler=train_sampler)
+  if gpu == 0:
+    print(f'Train dataloader | samples: {len(train_dataloader.dataset)}, '
+          f'num workers: {train_dataloader.num_workers}, '
+          f'global batch size: {args.train_batch_size * args.gpus}, '
+          f'batches/epoch: {len(train_dataloader)}')
+
+  # Create eval dataloader.
+  eval_dataset = ImageFolder(
+      image_list_file=args.eval_data_path,
+      transform=torchvision.transforms.Compose([
+          torchvision.transforms.Resize(256),
+          torchvision.transforms.CenterCrop(224),
+          torchvision.transforms.ToTensor(),
+          torchvision.transforms.Normalize(
+              mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
+      ]))
+  eval_sampler = torch.utils.data.distributed.DistributedSampler(
+      eval_dataset, num_replicas=args.gpus, rank=gpu)
+  eval_dataloader = torch.utils.data.DataLoader(
+      dataset=eval_dataset,
+      batch_size=args.eval_batch_size,
+      shuffle=False,
+      num_workers=args.dataloader_num_workers,
+      pin_memory=True,
+      drop_last=True,
+      sampler=eval_sampler)
+  if gpu == 0:
+    print(f'Eval dataloader | samples: {len(eval_dataloader.dataset)}, '
+          f'num workers: {eval_dataloader.num_workers}, '
+          f'batch size: {args.eval_batch_size}, '
+          f'batches/epoch: {len(eval_dataloader)}')
+
+  # Wrap policy.
+  my_auto_wrap_policy = functools.partial(
+      size_based_auto_wrap_policy, min_num_params=100)
+  torch.cuda.set_device(gpu)
+
+  # Create model.
+  model = resnet50(weights=None)
+  model.to(args.device)
+  model = FSDP(model, auto_wrap_policy=my_auto_wrap_policy)
+
+  # Optimizer.
+  optimizer = torch.optim.SGD(model.parameters(), 0.1)
+
+  # Main loop.
+  metric = torchmetrics.classification.Accuracy(top_k=1).to(args.device)
+  for epoch in range(1, args.epochs + 1):
+    if gpu == 0:
+      print(f'Running epoch {epoch}')
+    train_sampler.set_epoch(epoch)
+
+    start = time.time()
+    train(model, args.device, train_dataloader, optimizer)
+    end = time.time()
+    if gpu == 0:
+      print(f'Training finished in {(end - start):>0.3f} seconds')
+
+    start = time.time()
+    evaluate(model, args.device, eval_dataloader, metric)
+    end = time.time()
+    if gpu == 0:
+      print(f'Evaluation finished in {(end - start):>0.3f} seconds')
+
+  if gpu == 0:
+    print('Done')
+  dist.destroy_process_group()
+
+
+def create_args():
+  """Create main args."""
+  parser = argparse.ArgumentParser(
+      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+  parser.add_argument(
+      '--gpus',
+      default=4,
+      type=int,
+      help='number of gpus to use')
+  parser.add_argument(
+      '--epochs',
+      default=2,
+      type=int,
+      help='number of total epochs to run')
+  parser.add_argument(
+      '--dataloader_num_workers',
+      default=2,
+      type=int,
+      help='number of workders for dataloader')
+  parser.add_argument(
+      '--train_data_path',
+      default='',
+      type=str,
+      help='path to training data')
+  parser.add_argument(
+      '--train_batch_size',
+      default=32,
+      type=int,
+      help='batch size for training per gpu')
+  parser.add_argument(
+      '--eval_data_path',
+      default='',
+      type=str,
+      help='path to evaluation data')
+  parser.add_argument(
+      '--eval_batch_size',
+      default=32,
+      type=int,
+      help='batch size for evaluation per gpu')
+  args = parser.parse_args()
+  return args
+
+
+def main():
+  args = create_args()
+
+  os.environ['MASTER_ADDR'] = 'localhost'
+  os.environ['MASTER_PORT'] = '8888'
+
+  args.device = 'cuda' if torch.cuda.is_available() else 'cpu'
+  print(f'Launch job on {args.gpus} GPUs with FSDP')
+  mp.spawn(worker, nprocs=args.gpus, args=(args,))
+
+
+if __name__ == '__main__':
+  main()

community-content/pytorch_efficient_training/resnet_fsdp_wds.py

@@ -0,0 +1,240 @@
+"""Train resnet on multiple GPUs with DDP."""
+
+import argparse
+import functools
+import itertools
+import math
+import os
+import time
+
+import torch
+from torch import nn
+import torch.distributed as dist
+from torch.distributed.fsdp import FullyShardedDataParallel as FSDP
+from torch.distributed.fsdp.wrap import size_based_auto_wrap_policy
+import torch.multiprocessing as mp
+import torchmetrics
+from torchvision.models import resnet50
+from torchvision.transforms import transforms
+import webdataset as wds
+
+
+def wds_split(src, rank, world_size):
+  """Shards split function for webdataset."""
+  # The context of caller of this function is within multiple processes
+  # (by DDP world_size) and multiple workers (by dataloader_num_workers).
+  # So we totally have (world_size * num_workers) workers for processing data.
+  # NOTE: Raw data should be sharded to enough shards to make sure one process
+  # can handle at least one shard, otherwise the process may hang.
+  worker_id = 0
+  num_workers = 1
+  worker_info = torch.utils.data.get_worker_info()
+  if worker_info:
+    worker_id = worker_info.id
+    num_workers = worker_info.num_workers
+  for s in itertools.islice(src, rank * num_workers + worker_id, None,
+                            world_size * num_workers):
+    yield s
+
+
+def identity(x):
+  return x
+
+
+def create_wds_dataloader(rank, args, mode):
+  """Create webdataset dataset and dataloader."""
+  if mode == 'train':
+    transform = transforms.Compose([
+        transforms.RandomResizedCrop(224),
+        transforms.RandomHorizontalFlip(),
+        transforms.ToTensor(),
+        transforms.Normalize(
+            mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
+    ])
+    data_path = args.train_data_path
+    data_size = args.train_data_size
+    batch_size_local = args.train_batch_size
+    batch_size_global = args.train_batch_size * args.gpus
+    # Since webdataset disallows partial batch, we pad the last batch for train.
+    batches = int(math.ceil(data_size / batch_size_global))
+  else:
+    transform = transforms.Compose([
+        transforms.Resize(256),
+        transforms.CenterCrop(224),
+        transforms.ToTensor(),
+        transforms.Normalize(
+            mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
+    ])
+    data_path = args.eval_data_path
+    data_size = args.eval_data_size
+    batch_size_local = args.eval_batch_size
+    batch_size_global = args.eval_batch_size * args.gpus
+    # Since webdataset disallows partial batch, we drop the last batch for eval.
+    batches = int(data_size / batch_size_global)
+
+  dataset = wds.DataPipeline(
+      wds.SimpleShardList(data_path),
+      functools.partial(wds_split, rank=rank, world_size=args.gpus),
+      wds.tarfile_to_samples(),
+      wds.decode('pil'),
+      wds.to_tuple('jpg;png;jpeg cls'),
+      wds.map_tuple(transform, identity),
+      wds.batched(batch_size_local, partial=False),
+  )
+  num_workers = args.dataloader_num_workers
+  dataloader = wds.WebLoader(
+      dataset=dataset,
+      batch_size=None,
+      shuffle=False,
+      num_workers=num_workers,
+      persistent_workers=True if num_workers > 0 else False,
+      pin_memory=True).repeat(nbatches=batches)
+  print(f'{mode} dataloader | samples: {data_size}, '
+        f'num_workers: {num_workers}, '
+        f'local batch size: {batch_size_local}, '
+        f'global batch size: {batch_size_global}, '
+        f'batches: {batches}')
+  return dataloader
+
+
+def train(model, device, dataloader, optimizer):
+  model.train()
+  for image, target in dataloader:
+    image = image.to(device, non_blocking=True)
+    target = target.to(device, non_blocking=True)
+    pred = model(image)
+    # pred.shape (N, C), target.shape (N)
+    loss = nn.functional.cross_entropy(pred, target)
+    optimizer.zero_grad()
+    loss.backward()
+    optimizer.step()
+  return loss
+
+
+def evaluate(model, device, dataloader, metric):
+  model.eval()
+  with torch.no_grad():
+    for image, target in dataloader:
+      image = image.to(device, non_blocking=True)
+      target = target.to(device, non_blocking=True)
+      pred = model(image)
+      metric.update(pred, target)
+  accuracy = metric.compute()
+  metric.reset()
+  return accuracy
+
+
+def worker(gpu, args):
+  """Run training and evaluation."""
+  # Init process group.
+  print(f'Initiating process {gpu}')
+  dist.init_process_group(
+      backend='nccl',
+      init_method='env://',
+      world_size=args.gpus,
+      rank=gpu)
+
+  # Create dataloader.
+  train_dataloader = create_wds_dataloader(gpu, args, 'train')
+  eval_dataloader = create_wds_dataloader(gpu, args, 'eval')
+
+  # Wrap policy.
+  my_auto_wrap_policy = functools.partial(
+      size_based_auto_wrap_policy, min_num_params=100)
+  torch.cuda.set_device(gpu)
+
+  # Create model.
+  model = resnet50(weights=None)
+  model.to(args.device)
+  model = FSDP(model, auto_wrap_policy=my_auto_wrap_policy)
+
+  # Optimizer.
+  optimizer = torch.optim.SGD(model.parameters(), 0.1)
+
+  # Main loop.
+  metric = torchmetrics.classification.Accuracy(top_k=1).to(args.device)
+  for epoch in range(1, args.epochs + 1):
+    if gpu == 0:
+      print(f'Running epoch {epoch}')
+
+    start = time.time()
+    train(model, args.device, train_dataloader, optimizer)
+    end = time.time()
+    if gpu == 0:
+      print(f'Training finished in {(end - start):>0.3f} seconds')
+
+    start = time.time()
+    evaluate(model, args.device, eval_dataloader, metric)
+    end = time.time()
+    if gpu == 0:
+      print(f'Evaluation finished in {(end - start):>0.3f} seconds')
+
+  if gpu == 0:
+    print('Done')
+
+
+def create_args():
+  """Create main args."""
+  parser = argparse.ArgumentParser(
+      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+  parser.add_argument(
+      '--gpus',
+      default=4,
+      type=int,
+      help='number of gpus to use')
+  parser.add_argument(
+      '--epochs',
+      default=2,
+      type=int,
+      help='number of total epochs to run')
+  parser.add_argument(
+      '--dataloader_num_workers',
+      default=2,
+      type=int,
+      help='number of workders for dataloader')
+  parser.add_argument(
+      '--train_data_path',
+      default='',
+      type=str,
+      help='path to training data')
+  parser.add_argument(
+      '--train_batch_size',
+      default=32,
+      type=int,
+      help='batch size for training per gpu')
+  parser.add_argument(
+      '--train_data_size',
+      default=50000,
+      type=int,
+      help='data size for training')
+  parser.add_argument(
+      '--eval_data_path',
+      default='',
+      type=str,
+      help='path to evaluation data')
+  parser.add_argument(
+      '--eval_batch_size',
+      default=32,
+      type=int,
+      help='batch size for evaluation per gpu')
+  parser.add_argument(
+      '--eval_data_size',
+      default=50000,
+      type=int,
+      help='data size for evaluation')
+  args = parser.parse_args()
+  return args
+
+
+def main():
+  args = create_args()
+  os.environ['MASTER_ADDR'] = 'localhost'
+  os.environ['MASTER_PORT'] = '8888'
+
+  args.device = 'cuda' if torch.cuda.is_available() else 'cpu'
+  print(f'Launch job on {args.gpus} GPUs with FSDP')
+  mp.spawn(worker, nprocs=args.gpus, args=(args,))
+
+
+if __name__ == '__main__':
+  main()

notebooks/ci_notebook_ingestion.md

@@ -1,21 +1,25 @@
 # Administrative Howto notes on CI Notebook Ingestion
 
-This readme covers administrative actions that are perform on an as-needed basis.
+
+This readme covers administrative actions that are performed on an as-needed basis.
 
 ## Team: vertex-ai-owners
 
-Members of the vertex-ai-owners (git team) have administrative priveleges. 
+Members of the vertex-ai-owners (git team) have administrative privileges. 
+
 
 ### Viewing members
 
 1. Goto the repo
-2. From top-level menu, select: (Settings -> Colaborators and Teams)[https://github.com/GoogleCloudPlatform/vertex-ai-samples/settings/access]
+2. From top-level menu, select: (Settings -> Collaborators and Teams)[https://github.com/GoogleCloudPlatform/vertex-ai-samples/settings/access]
+
 
 ### Adding a new member
 
 If another member needs to be added:
-   - Have the new member make request to join the team.
-   - vertex-ai-owners with the `Maintainer` tag may the new member.
+   - Have the new member make a request to join the team.
+   - vertex-ai-owners with the `Maintainer` tag may add the new member.
+
    
 ## Executing CI notebook ingestion checks on a PR
    
@@ -24,7 +28,7 @@ If another member needs to be added:
 If the CI notebook ingestion test is stuck (not terminating), you can kill the process by:
 
 1. Goto the PR
-2. Under checks, find the entry: vertex-ai-notebook-execution-test (python-docs-samples-tests) In progress — Summary
+2. Under checks, find the entry: vertex-ai-notebook-execution-test (python-docs-samples-tests) In progress —> Summary
 3. Select Details
 4. At bottom of details page, select: View more details on Google Cloud Build
 5. In Cloud Build history page, select Cancel on the top menu bar.

notebooks/community/matching_engine/stream_update_for_matching_engine.ipynb

@@ -33,7 +33,7 @@
         "\n",
         "  <td>\n",
         "    <a href=\"https://console.cloud.google.com/vertex-ai/notebooks/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/community/matching_engine/stream_update_matching_engine.ipynb\">\n",
-        "      Run in Google Cloud Notebooks\n",
+        "      Run in Workbench AI Notebooks\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
@@ -53,7 +53,7 @@
       "source": [
         "## Overview\n",
         "\n",
-        "This example demonstrates how to use the GCP matching engine Stream Update Service. \n",
+        "This example demonstrates how to use the Vertex AI Matching Engine Stream Update Service. \n",
         "\n",
         "### Dataset\n",
         "\n",
@@ -150,7 +150,7 @@
       "source": [
         "### Installation\n",
         "\n",
-        "Download and install the latest (preview) version of the Vertex SDK for Python."
+        "Download and install the latest (preview) version of the Vertex AI SDK for Python."
       ]
     },
     {
@@ -442,7 +442,7 @@
         "id": "8292bcedab58"
       },
       "source": [
-        "## Prepare the Data\n",
+        "## Prepare the data\n",
         "\n",
         "The GloVe dataset consists of a set of pre-trained embeddings. The embeddings are split into a \"train\" split, and a \"test\" split.\n",
         "We will create a vector search index from the \"train\" split, and use the embedding vectors in the \"test\" split as query vectors to test the vector search index.\n",
@@ -525,7 +525,7 @@
         "        f.write('{\"id\":\"' + str(i) + '\",')\n",
         "        f.write('\"embedding\":[' + \",\".join(str(x) for x in train[i]) + \"],\")\n",
         "        f.write(\n",
-        "            '\"restricts\":[{\"namespace\": \"class\", \"allow_list\": [\"' + str(i) + '\"]}],'\n",
+        "            '\"restricts\":[{\"namespace\": \"class\", \"allow\": [\"' + str(i) + '\"]}],'\n",
         "        )\n",
         "        f.write('\"crowding_tag\":' + ('\"a\"' if i % 2 == 0 else '\"b\"') + \"}\")\n",
         "        f.write(\"\\n\")\n",
@@ -854,7 +854,7 @@
         "id": "00c606bc97b5"
       },
       "source": [
-        "## Create Online Queries\n",
+        "## Create online queries\n",
         "\n",
         "After you built your indexes, you may query against the deployed index through the online querying gRPC API (Match service) within the virtual machine instances from the same region (for example 'us-central1' in this tutorial).  \n",
         "\n",

notebooks/community/ml_ops/stage1/README.md

@@ -28,9 +28,11 @@ The first stage in MLOps is the collection and preparation for the purpose of de
 
 ### Get Started
 
-[Get started with Dataflow](get_started_dataflow.ipynb)
 
-In this tutorial, you learn how to use `Dataflow` for training with `Vertex AI`.
+[Get started with Dataflow](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/get_started_dataflow.ipynb)
+
+```
+Learn how to use `Dataflow` for training with `Vertex AI`.
 
 The steps performed include:
 
@@ -40,10 +42,13 @@ The steps performed include:
 - Upstream preprocessing of data:
     - tabular data
     - image data
+```
 
-[Get started with Vertex AI datasets](get_started_vertex_datasets.ipynb)
 
-In this tutorial, you learn how to use `Vertex AI Dataset` for training with `Vertex AI`.
+[Get started with Vertex AI datasets](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/get_started_vertex_datasets.ipynb)
+
+```
+Learn how to use `Vertex AI Dataset` for training with `Vertex AI`.
 
 The steps performed include:
 
@@ -61,10 +66,13 @@ The steps performed include:
 - Detect anomalies in new data using TensorFlow Data Validation.
 - Generate a TFRecord feature specification using TensorFlow Transform from the data schema.
 - Export a dataset and convert to TFRecords.
+```
 
-[Get started with BigQuery datasets](get_started_bq_datasets.ipynb)
 
-In this tutorial, you learn how to use `BigQuery` as a dataset for training with `Vertex AI`.
+[Get started with BigQuery datasets](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/get_started_bq_datasets.ipynb)
+
+```
+Learn how to use `BigQuery` as a dataset for training with `Vertex AI`.
 
 The steps performed include:
 
@@ -75,10 +83,13 @@ The steps performed include:
 - Select rows from extracted CSV files into a `tf.data.Dataset` -- compatible for custom training `TensorFlow` models.
 - Create a `BigQuery` dataset from CSV files.
 - Extract data from `BigQuery` table into a `DMatrix` -- compatible for custom training `XGBoost` models.
+```
 
-[Get started with Vertex AI Data Labeling](get_started_with_data_labeling.ipynb)
 
-In this tutorial, you learn how to use the `Vertex AI Data Labeling` service.
+[Get started with Vertex AI Data Labeling](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/get_started_with_data_labeling.ipynb)
+
+```
+Learn how to use the `Vertex AI Data Labeling` service/
 
 The steps performed include:
 
@@ -87,28 +98,31 @@ The steps performed include:
 - Submit the data labeling job.
 - List data labeling jobs.
 - Cancel a data labeling job.
+```
 
 
+[Create an unlabelled Vertex AI AutoML text entity extraction dataset from PDFs using Vision API](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/get_started_with_visionapi_and_vertex_datasets.ipynb)
 
-[Create an unlabelled Vertex AI AutoML text entity extraction dataset from PDFs using Vision API](get_started_with_visionapi_and_vertex_datasets.ipynb)
-
-In this tutorial, you learn to use `Vision API` to extract text from PDF files stored on a Cloud Storage bucket. You then process the results and create an unlabelled `Vertex AI Dataset`, compatible with `AutoML`, for text entity extraction.
+```
+Learn to use `Vision API` to extract text from PDF files stored on a Cloud Storage bucket.
 
 The steps performed include:
 
 1. Using `Vision API` to perform Optical Character Recognition (OCR) to extract text from PDF files.
 2. Processing the results and saving them to text files.
 3. Generating a `Vertex AI Dataset` import file.
-4. Creating a new unlabelled text entity extraction `Vertex AI Dataset` resource in `Vertex AI`.
+4. Cr
 
 
 ### E2E Stage Example
 
-[Stage 1: Data Management](mlops_data_management.ipynb)
-
+[Data management](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage1/mlops_data_management.ipynb)
 
 ```
+In this tutorial, you create a MLOps stage 1: data management process.
+
 The steps performed include:
+
 - Explore and visualize the data.
 - Create a Vertex AI `Dataset` resource from `BigQuery` table -- for AutoML training.
 - Extract a copy of the dataset to a CSV file in Cloud Storage.
@@ -117,4 +131,4 @@ The steps performed include:
 - Generate statistics and data schema using TensorFlow Data Validation from the samples in the dataframe.
 - Generate a TFRecord feature specification using TensorFlow Data Validation from the data schema.
 - Preprocess a portion of the BigQuery data using `Dataflow` -- for custom training.
-```
+```

notebooks/community/ml_ops/stage2/README.md

@@ -35,9 +35,10 @@ The second stage in MLOps is experimenting in developing one or more baseline mo
 
 ### Get Started
 
-[Get started with Vertex AI Training for R](community/ml_ops/stage2/get_started_vertex_training_r.ipynb)
+[Get started with Vertex AI Training for R](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_training_r.ipynb)
 
-In this tutorial, you learn how to use `Vertex AI Training` for training a R custom model.
+```
+Learn how to use `Vertex AI Training` for training a R custom model.
 
 The steps performed include:
 
@@ -51,18 +52,26 @@ The steps performed include:
 - Create a training image for training the model.
 - Train a R model using `Vertex AI Trainingh` service with the R-to-Python training package.
 
-[Get started with Logging](community/ml_ops/stage2/get_started_with_logging.ipynb)
+```
 
-In this tutorial, you learn how to use Python and Cloud logging awhen training with `Vertex AI`.
+
+[Get started with Logging](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_with_logging.ipynb)
+
+```
+Learn how to use Python and Cloud logging when training with `Vertex AI`.
 
 The steps performed include:
 
 - Use Python logging to log training configuration/results locally.
 - Use Google Cloud Logging to log training configuration/results in cloud storage.
 
-[Get started with Vertex AI Hyperparameter Tuning for XGBoost] (community/ml_ops/stage2/get_started_vertex_hpt_xgboost.ipynb)
+```
 
-In this tutorial, you learn how to use `Vertex AI Hyperparameter Tuning` for training a XGBoost custom model.
+
+[Get started with Vertex AI Hyperparameter Tuning for XGBoost](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_hpt_xgboost.ipynb)
+
+```
+Learn how to use `Vertex AI Hyperparameter Tuning` for training a XGBoost custom model.
 
 The steps performed include:
 
@@ -71,9 +80,13 @@ The steps performed include:
 - Save the model artifacts to Cloud Storage using GCSFuse.
 - Create a `Vertex AI Model` resource.
 
-[Get started with Vertex AI Training for XGBoost](community/ml_ops/stage2/get_started_vertex_training_xgboost.ipynb)
+```
 
-In this tutorial, you learn how to use `Vertex AI Training` for training a XGBoost custom model.
+
+[Get started with Vertex AI Training for XGBoost](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_training_xgboost.ipynb)
+
+```
+Learn how to use `Vertex AI Training` for training a XGBoost custom model.
 
 The steps performed include:
 
@@ -82,9 +95,13 @@ The steps performed include:
 - Save the model artifacts to Cloud Storage using GCSFuse.
 - Create a `Vertex AI Model` resource.
 
-[Get started with TabNet builtin algorithm for training tabular models](community/ml_ops/stage2/get_started_with_tabnet.ipynb)
+```
 
-In this notebook, you learn how to run `Vertex AI TabNet` built algorithm for training custom tabular models.
+
+[Get started with TabNet builtin algorithm for training tabular models](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_with_tabnet.ipynb)
+
+```
+Learn how to run `Vertex AI TabNet` built algorithm for training custom tabular models.
 
 The steps performed include:
 
@@ -97,9 +114,13 @@ The steps performed include:
 - Hyperparameter tuning the `Vertex AI TabNet` model.
 - Train the model using `Vertex AI Training` using BigQuery table.
 
-[Get started with prebuilt TFHub models](community/ml_ops/stage2/get_started_with_tfhub_models.ipynb)
+```
 
-In this tutorial, you learn how to use `Vertex AI Training` with prebuilt models from TensorFlow Hub.
+
+[Get started with prebuilt TFHub models](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_with_tfhub_models.ipynb)
+
+```
+Learn how to use `Vertex AI Training` with prebuilt models from TensorFlow Hub.
 
 The steps performed include:
 
@@ -112,23 +133,31 @@ The steps performed include:
     - Train then model
     - Save model artifacts and upload as Vertex AI Model resource.
 
-[Get started with BigQuery ML Training](community/ml_ops/stage2/get_started_bqml_training.ipynb)
+```
 
-In this tutorial, you learn how to use `BigQueryML` (BQML) for training with `Vertex AI`.
+
+[Get started with BigQuery ML Training](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_bqml_training.ipynb)
+
+```
+Learn how to use `BigQueryML` for training with `Vertex AI`.
 
 The steps performed include:
 
 - Create a local BigQuery table in your project
-- Train a BQML model
-- Evaluate the BQML model
-- Export the BQML model as a cloud model
+- Train a BigQuery ML model
+- Evaluate the BigQuery ML model
+- Export the BigQuery ML model as a cloud model
 - Upload the exported model as a `Vertex AI Model` resource
-- Hyperparameter tune a BQML model with `Vertex AI Vizier`
-- Automatically register a BQML model to `Vertex AI Model Registry`
+- Hyperparameter tune a BigQuery ML model with `Vertex AI Vizier`
+- Automatically register a BigQuery ML model to `Vertex AI Model Registry`
 
-[Get started with Vertex AI Vizier](community/ml_ops/stage2/get_started_vertex_vizier.ipynb)
+```
 
-In this tutorial, you learn how to use `Vertex AI Vizier` for when training with `Vertex AI`.
+
+[Get started with Vertex AI Vizier](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_vizier.ipynb)
+
+```
+Learn how to use `Vertex AI Vizier` for when training with `Vertex AI`.
 
 The steps performed include:
 
@@ -136,9 +165,13 @@ The steps performed include:
 - Hyperparameter tuning with Vizier (Bayesian) algorithm.
 - Suggesting trials and updating results for Vizier study
 
-[Get started with distributed training using DASK](community/ml_ops/stage2/get_started_with_distributed_training_xgboost.ipynb)
+```
 
-In this tutorial, you learn how to use `Vertex AI Training` for distributed training of XGBoost model using the OSS package DASK. Additionally, you learn to construct and deploy a custom serving container using a Flask web server.
+
+[Get started with distributed training using DASK](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_with_distributed_training_xgboost.ipynb)
+
+```
+Learn how to use `Vertex AI Training` for distributed training of XGBoost model using the OSS package DASK.
 
 The steps performed include:
 
@@ -152,9 +185,13 @@ The steps performed include:
 - Deploy the `Vertex AI Model` resource to `Vertex AI Endpoint` resource.
 - Make a prediction.
 
-[Get started with Vertex AI TensorBoard](community/ml_ops/stage2/get_started_vertex_tensorboard.ipynb)
+```
 
-In this tutorial, you learn how to use `Vertex AI TensorBoard` when training with `Vertex AI`.
+
+[Get started with Vertex AI TensorBoard](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_tensorboard.ipynb)
+
+```
+Learn how to use `Vertex AI TensorBoard` when training with `Vertex AI`.
 
 The steps performed include:
 
@@ -162,9 +199,13 @@ The steps performed include:
 - Using TensorBoard with locally trained model.
 - Using Vertex AI TensorBoard with Vertex AI Training.
 
-[Get started with Vertex AI Training for R using R Kernel](community/ml_ops/stage2/get_started_vertex_training_r_using_r_kernel.ipynb)
+```
 
-In this tutorial, you learn how to use `Vertex AI`, using an R kernel, for training and deploying an R custom model.
+
+[Get started with Vertex AI Training for R using R Kernel](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_training_r_using_r_kernel.ipynb)
+
+```
+Learn how to use `Vertex AI`, using an R kernel, for training and deploying an R custom model.
 
 The steps performed include:
 
@@ -176,10 +217,13 @@ The steps performed include:
 - Deploy the `Model` resource (trained R model) to the `Endpoint` resource.
 - Make an online prediction.
 
+```
 
-[Get started Vision API test preprocessing and AutoML text model generation](community/ml_ops/stage2/get_started_with_visionapi_and_automl.ipynb)
 
-In this tutorial, you create an `AutoML` text entity extraction model pre-existing extracted data by generating a custom import file. You deploy this mode for online prediction from a Python script using the `BigQuery`, `Vision AI`, Cloud Storage and `Vertex AI SDK` for Python. 
+[Get started Vision API test preprocessing and AutoML text model generation](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_with_visionapi_and_automl.ipynb)
+
+```
+In this tutorial, you create an `AutoML` text entity extraction model pre-existing extracted data by generating a custom import file.
 
 The steps performed include:
 
@@ -192,9 +236,13 @@ The steps performed include:
 - Make a prediction.
 - Undeploy the `Model`.
 
-[Get started with Vertex AI Experiments](community/ml_ops/stage2/get_started_vertex_experiments.ipynb)
+```
 
-In this tutorial, you learn how to use `Vertex AI Experiments` when training with `Vertex AI`.
+
+[Get started with Vertex AI Experiments](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_experiments.ipynb)
+
+```
+Learn how to use `Vertex AI Experiments` when training with `Vertex AI`.
 
 The steps performed include:
 
@@ -215,9 +263,13 @@ The steps performed include:
     - Execute the custom job
     - Visualize the experiment results
 
-[AutoML Image Classfication Training with Customer Managed Encryption Keys (CMEK)](community/ml_ops/stage2/get_started_with_cmek_training.ipynb)
+```
 
-In this tutorial, you learn how to use a customer managed encryption key (CMEK) for `Vertex AI AutoML` training.
+
+[AutoML Image Classfication Training with Customer Managed Encryption Keys (CMEK)](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_with_cmek_training.ipynb)
+
+```
+Learn how to use a customer managed encryption key (CMEK) for `Vertex AI AutoML` training.
 
 The steps performed include:
 
@@ -225,9 +277,13 @@ The steps performed include:
 - Creating an image dataset with CMEK encryption.
 - Train an AutoML model with CMEK encryption.
 
-[Get started with Vertex AI Feature Store](community/ml_ops/stage2/get_started_vertex_feature_store.ipynb)
+```
 
-In this tutorial, you learn how to use `Vertex AI Feature Store` when training and predicting with `Vertex AI`.
+
+[Get started with Vertex AI Feature Store](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_feature_store.ipynb)
+
+```
+Learn how to use `Vertex AI Feature Store` when training and predicting with `Vertex AI`.
 
 The steps performed include:
 
@@ -240,9 +296,13 @@ The steps performed include:
 - Perform online serving from a `Featurestore` resource.
 - Perform batch serving from a `Featurestore` resource.
 
-[Get started with AutoML Training](community/ml_ops/stage2/get_started_automl_training.ipynb)
+```
 
-In this tutorial, you learn how to use `AutoML` for training with `Vertex AI`.
+
+[Get started with AutoML Training](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_automl_training.ipynb)
+
+```
+Learn how to use `AutoML` for training with `Vertex AI`.
 
 The steps performed include:
 
@@ -253,9 +313,29 @@ The steps performed include:
 - Train a text model
 - Train a video model
 
-[Get started with Vertex AI Training for LightGBM](community/ml_ops/stage2/get_started_vertex_training_lightgbm.ipynb)
+```
 
-In this tutorial, you learn how to use `Vertex AI Training` for training a LightGBM custom model.
+
+[Get started with autologging using Vertex AI Experiments for XGBoost models](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_with_vertex_experiments_autologging_xgboost.ipynb)
+
+```
+Learn how to create an experiment for training an XGBoost model, and automatically log parameters and metrics using the enclosed do-it-yourself (DIY) code.
+
+The steps performed include:
+
+- Construct the DIY autologging code.
+- Construct training package with call to autologging.
+- Train a model.
+- View the experiment
+- Delete the experiment.
+
+```
+
+
+[Get started with Vertex AI Training for LightGBM](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_training_lightgbm.ipynb)
+
+```
+Learn how to use `Vertex AI Training` for training a LightGBM custom model.
 
 The steps performed include:
 
@@ -266,9 +346,26 @@ The steps performed include:
 - Test the deployment image locally.
 - Create a `Vertex AI Model` resource.
 
-[Get started with Vertex AI Training for Scikit-Learn](community/ml_ops/stage2/get_started_vertex_training_sklearn.ipynb)
+```
 
-In this tutorial, you learn how to use `Vertex AI Training` for training a Scikit-Learn custom model.
+
+[Vertex AI Hyperparameter Tuning with R kernel](None)
+
+```
+Learn how to use `Vertex AI`, using an R kernel, for tuning hyperparameters of a R custom model.
+
+The steps performed include:
+
+- Create a custom R training script
+- Create a custom R deployment container.
+- Perform hyperparameter tuning using `Vertex AI`.
+
+```
+
+[Get started with Vertex AI Training for Scikit-Learn](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_training_sklearn.ipynb)
+
+```
+Learn how to use `Vertex AI Training` for training a Scikit-Learn custom model.
 
 The steps performed include:
 
@@ -277,9 +374,13 @@ The steps performed include:
 - Save the model artifacts to Cloud Storage using GCSFuse.
 - Create a `Vertex AI Model` resource.
 
-[Get started with Vertex AI Training](community/ml_ops/stage2/get_started_vertex_training.ipynb)
+```
 
-In this tutorial, you learn how to use `Vertex AI Training` for custom models when training with `Vertex AI`.
+
+[Get started with Vertex AI Training](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_training.ipynb)
+
+```
+Learn how to use `Vertex AI Training` for custom models when training with `Vertex AI`.
 
 The steps performed include:
 
@@ -288,10 +389,13 @@ The steps performed include:
 - Training using a custom training image.
 - Laying out a training package.
 
+```
 
-[Get started with Vertex AI Training for Pytorch](community/ml_ops/stage2/get_started_vertex_training_pytorch.ipynb)
 
-In this tutorial, you learn how to use `Vertex AI Training` for training a Pytorch custom model.
+[Get started with Vertex AI Training for PyTorch](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_training_pytorch.ipynb)
+
+```
+Learn how to use `Vertex AI Training` for training a PyTorch custom model.
 
 The steps performed include:
 
@@ -300,9 +404,31 @@ The steps performed include:
 - Save the model artifacts to Cloud Storage using GCSFuse.
 - Create a `Vertex AI Model` resource.
 
-[Get started with Vertex AI Distributed Training](community/ml_ops/stage2/get_started_vertex_distributed_training.ipynb)
+```
 
-In this tutorial, you learn how to use `Vertex AI Distributed Training` for when training with `Vertex AI`.
+
+[Get started with autologging using Vertex AI Experiments for TensorFlow models](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_with_vertex_experiments_autologging_tf.ipynb)
+
+```
+Learn how to create an experiment for training a TensorFlow model, and automatically log parameters and metrics using the enclosed do-it-yourself (DIY) code.
+
+The steps performed include:
+
+- Construct the DIY autologging code.
+- Construct training package for TensorFlow Sequential model with call to autologging.
+- Train a model.
+- View the experiment
+- Construct training package for TensorFlow Functional model with call to autologging.
+- Compare the experiment runs.
+- Delete the experiment.
+
+```
+
+
+[Get started with Vertex AI Distributed Training](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/get_started_vertex_distributed_training.ipynb)
+
+```
+Learn how to use `Vertex AI Distributed Training` for when training with `Vertex AI`.
 
 The steps performed include:
 
@@ -312,12 +438,17 @@ The steps performed include:
 - `ReductionServer`: Train on multiple VMS and sync updates across VMS with `Vertex AI Reduction Server`.
 - `TPUTraining`: Train with multiple Cloud TPUs.
 
+```
+
 ### E2E Stage Example
 
-[Stage 2: Experimentation](mlops_experimentation.ipynb)
+[Experimentation](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage2/mlops_experimentation.ipynb)
 
 ```
+In this tutorial, you create a MLOps stage 2: experimentation process.
+
 The steps performed include:
+
 - Review the `Dataset` resource created during stage 1.
 - Train an AutoML tabular binary classifier model in the background.
 - Build the experimental model architecture.
@@ -334,4 +465,5 @@ The steps performed include:
 - Set the evaluation results of the AutoML model as the baseline.
 - If the evaluation of the custom model is below baseline, continue to experiment with the custom model.
 - If the evaluation of the custom model is above baseline, save the model as the first best model.
+
 ```

notebooks/community/ml_ops/stage3/README.md

@@ -34,9 +34,10 @@ The third stage in MLOps is formalization to develop an automated pipeline proce
 ### Get Started
 
 
-[Get started with Vertex AI Model Registry](get_started_with_model_registry.ipynb)
+[Get started with Vertex AI Model Registry](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_model_registry.ipynb)
 
-In this tutorial, you learn how to use `Vertex AI Model Registry` to create and register multiple versions of a model.
+```
+Learn how to use `Vertex AI Model Registry` to create and register multiple versions of a model.
 
 The steps performed include:
 
@@ -46,9 +47,13 @@ The steps performed include:
 - Deleting a model version.
 - Retraining the next model version.
 
-[Get started with Dataflow pipeline components](get_started_with_dataflow_pipeline_components.ipynb)
+```
 
-In this tutorial, you learn how to use prebuilt `Google Cloud Pipeline Components` for `Dataflow`.
+
+[Get started with Dataflow pipeline components](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_dataflow_pipeline_components.ipynb)
+
+```
+Learn how to use prebuilt `Google Cloud Pipeline Components` for `Dataflow`.
 
 The steps performed include:
 
@@ -56,9 +61,13 @@ The steps performed include:
 - Encapsulate the Apache Beam data pipeline with a Dataflow component in a Vertex AI pipeline.
 - Execute a Vertex AI pipeline.
 
-[Get started with Apache Airflow and Vertex AI Pipelines](get_started_with_airflow_and_vertex_pipelines.ipynb)
+```
 
-In this tutorial, you learn how to use Apache Airflow with `Vertex AI Pipelines`.
+
+[Get started with Apache Airflow and Vertex AI Pipelines](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_airflow_and_vertex_pipelines.ipynb)
+
+```
+Learn how to use Apache Airflow with `Vertex AI Pipelines`.
 
 The steps performed include:
 
@@ -67,9 +76,13 @@ The steps performed include:
 - Create a `Vertex AI Pipeline` that triggers the Airflow DAG.
 - Execute the `Vertex AI Pipeline`.
 
-[Get started with Kubeflow Pipelines](get_started_with_kubeflow_pipelines.ipynb)
+```
 
-In this tutorial, you learn how to use `Kubeflow Pipelines`(KFP).
+
+[Get started with Kubeflow Pipelines](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_kubeflow_pipelines.ipynb)
+
+```
+Learn how to use `Kubeflow Pipelines`(KFP).
 
 The steps performed include:
 
@@ -80,9 +93,13 @@ The steps performed include:
 - Building sequential, parallel, multiple output components.
 - Building control flow into pipelines.
 
-[Get started with Vertex AI custom training pipeline components](get_started_with_custom_training_pipeline_components.ipynb)
+```
 
-In this tutorial, you learn how to use prebuilt `Google Cloud Pipeline Components` for `Vertex AI Training`.
+
+[Get started with Vertex AI custom training pipeline components](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_custom_training_pipeline_components.ipynb)
+
+```
+Learn how to use prebuilt `Google Cloud Pipeline Components` for `Vertex AI Training`.
 
 The steps performed include:
 
@@ -98,11 +115,13 @@ The steps performed include:
     - Deploying a Vertex AI custom trained model.
 - Execute a Vertex AI pipeline.
 
-[Get started with Dataproc Serverless pipeline components](get_started_with_dataproc_serverless_pipeline_components.ipynb)
+```
 
 
-In this tutorial, you learn how to use prebuilt `Google Cloud Pipeline Components` for `Dataproc Serverless` service. 
+[Get started with Dataproc Serverless pipeline components](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_dataproc_serverless_pipeline_components.ipynb)
 
+```
+Learn how to use prebuilt `Google Cloud Pipeline Components` for `Dataproc Serverless` service.
 
 The steps performed include:
 
@@ -111,9 +130,13 @@ The steps performed include:
 - `DataprocSparkSqlBatchOp` for running Spark SQL batch workloads.
 - `DataprocSparkRBatchOp` for running SparkR batch workloads.
 
-[Get started with Vertex AI Hyperparameter Tuning pipeline components](get_started_with_hpt_pipeline_components.ipynb)
+```
 
-In this tutorial, you learn how to use prebuilt `Google Cloud Pipeline Components` for `Vertex AI Hyperparameter Tuning`.
+
+[Get started with Vertex AI Hyperparameter Tuning pipeline components](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_hpt_pipeline_components.ipynb)
+
+```
+Learn how to use prebuilt `Google Cloud Pipeline Components` for `Vertex AI Hyperparameter Tuning`.
 
 The steps performed include:
 
@@ -125,23 +148,28 @@ The steps performed include:
       - Upload the model artifacts to a `Vertex AI Model` resource.
 - Execute a Vertex AI pipeline.
 
-[Get started with machine management for Vertex AI Pipelines](get_started_with_machine_management.ipynb)
+```
 
-In this tutorial, you convert a self-contained custom training component into a `Vertex AI CustomJob`, whereby:
 
-    - The training job and artifacts are trackable.
-    - Set machine resources, such as machine-type, cpu/gpu, memory, disk, etc.
+[Get started with machine management for Vertex AI Pipelines](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_machine_management.ipynb)
+
+```
+Learn how to convert a self-contained custom training component into a `Vertex AI CustomJob`, whereby:
 
 The steps performed in this tutorial include:
 
 - Create a custom component with a self-contained training job.
 - Execute pipeline using component-level settings for machine resources
 - Convert the self-contained training component into a `Vertex AI CustomJob`.
-- Execute pipeline using customjob-level settings for machine resources 
+- Execute pipeline using customjob-level settings for machine resources
 
-[Get started with TFX pipelines](get_started_with_tfx_pipeline.ipynb)
+```
 
-In this tutorial, you learn how to use TensorFlow Extended (TFX) with `Vertex AI Pipelines`.
+
+[Get started with TFX pipelines](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_tfx_pipeline.ipynb)
+
+```
+Learn how to use TensorFlow Extended (TFX) with `Vertex AI Pipelines`.
 
 The steps performed include:
 
@@ -150,9 +178,28 @@ The steps performed include:
 - Execute the pipeline on Google Cloud using `Vertex AI Training`
 - Execute the pipeline using `Vertex AI Pipelines`.
 
-[Get started with BigQuery ML pipeline components](get_started_with_bqml_pipeline_components.ipynb)
+```
 
-In this tutorial, you learn how to use prebuilt `Google Cloud Pipeline Components` for `BigQuery ML`.
+
+[Orchestrating a workflow to train and deploy an scikit-learn model using Vertex AI Pipelines with online prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_vertex_pipelines_sklearn_with_prediction.ipynb)
+
+```
+Learn how to use prebuilt components in `Vertex AI Pipelines` for training and deploying a scikit-Learn custom model, and then using `Vertex AI Prediction` to make an online prediction.
+
+The steps performed include:
+
+- Construct a scikit-learn training package.
+- Construct a pipeline to train and deploy a scikit-learn model.
+- Execute the pipeline.
+- Make an online prediction.
+
+```
+
+
+[Get started with BigQuery ML pipeline components](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_bqml_pipeline_components.ipynb)
+
+```
+Learn how to use prebuilt `Google Cloud Pipeline Components` for `BigQuery ML`.
 
 The steps performed include:
 
@@ -165,9 +212,28 @@ The steps performed include:
 - Execute a Vertex AI pipeline.
 - Make a prediction with the deployed Vertex AI model.
 
-[Get started with AutoML tabular pipeline workflows](get_started_with_automl_tabular_pipeline_workflow.ipynb)
+```
 
-In this tutorial, you learn how to use `AutoML Tabular Pipeline Template` for training, exporting and tuning an AutoML tabular model.
+
+[Orchestrating a workflow to train and deploy an XGBoost model using Vertex AI Pipelines with Vertex AI Experiments](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_vertex_pipelines_xgboost_with_experiments.ipynb)
+
+```
+Learn how to use prebuilt components in `Vertex AI Pipelines` for training and deploying a XGBoost custom model, and using `Vertex AI Experiments` to log the corresponding training parameters and metrics, from within the training package.
+
+The steps performed include:
+
+- Construct a XGBoost training package.
+  - Add tracking the experiment
+- Construct a pipeline to train and deploy a XGBoost model.
+- Execute the pipeline.
+
+```
+
+
+[Get started with AutoML tabular pipeline workflows](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_automl_tabular_pipeline_workflow.ipynb)
+
+```
+Learn how to use `AutoML Tabular Pipeline Template` for training, exporting and tuning an AutoML tabular model.
 
 The steps performed include:
 
@@ -183,9 +249,13 @@ The steps performed include:
 - Deploy exported OSS TF model.
 - Make a prediction.
 
-[Get started with rapid prototyping with AutoML and BigQuery ML](cget_started_with_rapid_prototyping_bqml_automl.ipynb)
+```
 
-In this tutorial, you learn how to use `Vertex AI Predictions` for rapid prototyping a model.
+
+[Get started with rapid prototyping with AutoML and BigQuery ML](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_rapid_prototyping_bqml_automl.ipynb)
+
+```
+Learn how to use `Vertex AI Predictions` for rapid prototyping a model.
 
 The steps performed include:
 
@@ -196,9 +266,13 @@ The steps performed include:
 - Deploying the best trained model.
 - Testing the deployed model infrastructure.
 
-[Get started with AutoML pipeline components](get_started_with_automl_pipeline_components.ipynb)
+```
 
-In this tutorial, you learn how to use prebuilt `Google Cloud Pipeline Components` for `Vertex AI AutoML`.
+
+[Get started with AutoML pipeline components](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_automl_pipeline_components.ipynb)
+
+```
+Learn how to use prebuilt `Google Cloud Pipeline Components` for `Vertex AI AutoML`.
 
 The steps performed include:
 
@@ -208,10 +282,28 @@ The steps performed include:
     - Deploying a Vertex AI AutoML trained model.
 - Execute a Vertex AI pipeline.
 
+```
 
-[Get started with BigQuery and TFDV pipeline components](get_started_with_bq_tfdv_pipeline_components.ipynb)
 
-In this tutorial, you learn how to use build lightweight Python components for BigQuery and TensorFlow Data Validation.
+[Orchestrating a workflow to train and deploy an XGBoost model using Vertex AI Pipelines with online prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_vertex_pipelines_xgboost_with_prediction.ipynb)
+
+```
+Learn how to use prebuilt components in `Vertex AI Pipelines` for training and deploying a XGBoost custom model, and then using `Vertex AI Prediction` to make an online prediction.
+
+The steps performed include:
+
+- Construct a XGBoost training package.
+- Construct a pipeline to train and deploy a XGBoost model.
+- Execute the pipeline.
+- Make an online prediction.
+
+```
+
+
+[Get started with BigQuery and TFDV pipeline components](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/get_started_with_bq_tfdv_pipeline_components.ipynb)
+
+```
+Learn how to use build lightweight Python components for BigQuery and TensorFlow Data Validation.
 
 The steps performed include:
 
@@ -219,22 +311,28 @@ The steps performed include:
 - Build and execute a pipeline component for generating TFDV statistics and schema from a Vertex AI Tabular Dataset.
 - Execute a Vertex AI pipeline.
 
+```
+
 ### E2E Stage Example
 
-[Stage 3: Formalization](mlops_formalization.ipynb)
+[Formalization](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage3/mlops_formalization.ipynb)
 
 ```
+In this tutorial, you create a MLOps stage 3: formalization process.
+
 The steps performed include:
+
 - Obtain resources from the experimentation stage.
     - Baseline model.
     - Dataset schema/statistics for baseline model.
 - Formalize a data preprocessing pipeline.
     - Extract columns/rows from BigQuery table to local BigQuery table.
-    - Use Tensorflow Data Validation library to determine statistics, schema, and features.
+    - Use TensorFlow Data Validation library to determine statistics, schema, and features.
     - Use Dataflow to preprocess the data.
     - Create a Vertex AI Dataset.
 - Formalize a build model architecture pipeline.
     - Create the Vertex AI Model base model.
 - Formalize a training pipeline.
+
 ```
 

notebooks/community/ml_ops/stage4/README.md

@@ -43,191 +43,104 @@ This stage may be done entirely by MLOps. We recommend:
 ### Get Started
 
 
-[Get started with Vertex AI Model Registry](community/ml_ops/stage3/get_started_with_model_registry.ipynb)
+[Get started with Vertex ML Metadata](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage4/get_started_with_vertex_ml_metadata.ipynb)
 
-In this tutorial, you learn how to use `Vertex AI Model Registry` to create and register multiple versions of a model.
+```
+Learn how to use `Vertex ML Metadata`.
 
 The steps performed include:
 
-- Create and register a first version of a model to `Vertex AI Model Registry`.
-- Create and register a second version of a model to `Vertex AI Model Registry`.
-- Updating the model version which is the default (blessed).
-- Deleting a model version.
-- Retraining the next model version.
+- Create a `Metadatastore` resource.
+- Create (record)/List an `Artifact`, with artifacts and metadata.
+- Create (record)/List an `Execution`.
+- Create (record)/List a `Context`.
+- Add `Artifact` to `Execution` as events.
+- Add `Execution` and `Artifact` into the `Context`
+- Delete `Artifact`, `Execution` and `Context`.
+- Create and run a `Vertex AI Pipeline` ML workflow to train and deploy a scikit-learn model.
+    - Create custom pipeline components that generate artifacts and metadata.
+    - Compare Vertex AI Pipelines runs.
+    - Trace the lineage for pipeline-generated artifacts.
+    - Query your pipeline run metadata.
 
-[Get started with Dataflow pipeline components](community/ml_ops/stage3/get_started_with_dataflow_pipeline_components.ipynb)
+```
 
-In this tutorial, you learn how to use prebuilt `Google Cloud Pipeline Components` for `Dataflow`.
+
+[Get started with Google Artifact Registry](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage4/get_started_with_google_artifact_registry.ipynb)
+
+```
+Learn how to use `Google Artifact Registry`.
 
 The steps performed include:
 
-- Build an Apache Beam data pipeline.
-- Encapsulate the Apache Beam data pipeline with a Dataflow component in a Vertex AI pipeline.
-- Execute a Vertex AI pipeline.
+- Creating a private Docker repository.
+- Tagging a container image, specific to the private Docker repository.
+- Pushing a container image to the private Docker repository.
+- Pulling a container image from the private Docker repository.
+- Deleting a private Docker repository.
 
-[Get started with Apache Airflow and Vertex AI Pipelines](community/ml_ops/stage3/get_started_with_airflow_and_vertex_pipelines.ipynb)
+```
 
-In this tutorial, you learn how to use Apache Airflow with `Vertex AI Pipelines`.
+
+[Get started with Vertex AI Model Evaluation](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage4/get_started_with_model_evaluation.ipynb)
+
+```
+Learn how to use `Vertex AI Model Evaluation`.
 
 The steps performed include:
 
-- Create Cloud Composer environment.
-- Upload Airflow DAG to Composer environment that performs data processing -- i.e., creates a BigQuery table from a CSV file.
-- Create a `Vertex AI Pipeline` that triggers the Airflow DAG.
-- Execute the `Vertex AI Pipeline`.
+```
 
-[Get started with Kubeflow Pipelines](community/ml_ops/stage3/get_started_with_kubeflow_pipelines.ipynb)
 
-In this tutorial, you learn how to use `Kubeflow Pipelines`(KFP).
+[Get started with Vertex Explainable AI](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage4/get_started_with_vertex_xai.ipynb)
+
+```
+Learn how to use `Vertex AI Explainable AI`.
 
 The steps performed include:
 
-- Building KFP lightweight Python function components.
-- Assembling and compiling KFP components into a pipeline.
-- Executing a KFP pipeline using Vertex AI Pipelines.
-- Loading component and pipeline definitions from a source code repository.
-- Building sequential, parallel, multiple output components.
-- Building control flow into pipelines.
+- Train an AutoML tabular model.
+    - Do a batch prediction with explanations.
+    - Do an online prediction with explanations.
+- Train an custom TensorFlow tabular model.
+    - Manually set configuration metadata.
+    - Do a batch prediction with explanations.
+    - Do an online prediction with explanations.
+    - Automatically set configuration metadata.
+- Train an custom TensorFlow image model.
+    - Manually set configuration metadata.
+    - Do a batch prediction with explanations.
+    - Do an online prediction with explanations.
+- Train an custom XGBoost tabular model.
+    - Manually set configuration metadata.
+    - Do an online prediction with explanations.
+- Train an custom scikit-learn tabular model.
+    - Manually set configuration metadata.
+    - Do an online prediction with explanations.
 
-[Get started with Vertex AI custom training pipeline components](community/ml_ops/stage3/get_started_with_custom_training_pipeline_components.ipynb)
+```
 
-In this tutorial, you learn how to use prebuilt `Google Cloud Pipeline Components` for `Vertex AI Training`.
+
+[Get started with AutoML Training and ML Metadata](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage4/get_started_with_vertex_ml_metadata_and_automl.ipynb)
+
+```
+Learn how to use `AutoML` for training and assemble the corresponding artifact linkage for `Vertex ML Metadata`.
 
 The steps performed include:
 
-- Construct a pipeline for:
-    - Training a Vertex AI custom trained model.
-    - Test the serving binary with a batch prediction job.
-    - Deploying a Vertex AI custom trained model.
-- Execute a Vertex AI pipeline.
-- Construct a pipeline for:
-     - Construct a custom training component.
-     - Convert custom training component to CustomTrainingJobOp.
-     - Training a Vertex AI custom trained model using the converted component.
-    - Deploying a Vertex AI custom trained model.
-- Execute a Vertex AI pipeline.
+- Create a `Dataset` resource.
+- Create a corresponding `google.VertexDataset` artifact.
+- Train a model using `AutoML`.
+- Create a corresponding `google.VertexModel` artifact.
+- Create an `Endpoint` resource.
+- Create a corresponding `google.Endpoint` artifact.
+- Deploy the train model to the `Endpoint`.
+- Create an execution and context for the `AutoML` training job and deployment.
+- Add the corresponding artifacts and context to the execution.
+- Add artifact links (event) to the execution.
+- Display the execution graph.
 
-[Get started with Dataproc Serverless pipeline components](community/ml_ops/stage3/get_started_with_dataproc_serverless_pipeline_components.ipynb)
-
-
-In this tutorial, you learn how to use prebuilt `Google Cloud Pipeline Components` for `Dataproc Serverless` service. 
-
-
-The steps performed include:
-
-- `DataprocPySparkBatchOp` for running PySpark batch workloads.
-- `DataprocSparkBatchOp` for running Spark batch workloads.
-- `DataprocSparkSqlBatchOp` for running Spark SQL batch workloads.
-- `DataprocSparkRBatchOp` for running SparkR batch workloads.
-
-[Get started with Vertex AI Hyperparameter Tuning pipeline components](community/ml_ops/stage3/get_started_with_hpt_pipeline_components.ipynb)
-
-In this tutorial, you learn how to use prebuilt `Google Cloud Pipeline Components` for `Vertex AI Hyperparameter Tuning`.
-
-The steps performed include:
-
-- Construct a pipeline for:
-    - Hyperparameter tune/train a custom model.
-    - Retrieve the tuned hyperparameter values and metrics to optimize.
-    - If the metrics exceed a specified threshold.
-      - Get the location of the model artifacts for the best tuned model.
-      - Upload the model artifacts to a `Vertex AI Model` resource.
-- Execute a Vertex AI pipeline.
-
-[Get started with machine management for Vertex AI Pipelines](community/ml_ops/stage3/get_started_with_machine_management.ipynb)
-
-In this tutorial, you convert a self-contained custom training component into a `Vertex AI CustomJob`, whereby:
-
-    - The training job and artifacts are trackable.
-    - Set machine resources, such as machine-type, cpu/gpu, memory, disk, etc.
-
-The steps performed in this tutorial include:
-
-- Create a custom component with a self-contained training job.
-- Execute pipeline using component-level settings for machine resources
-- Convert the self-contained training component into a `Vertex AI CustomJob`.
-- Execute pipeline using customjob-level settings for machine resources 
-
-[Get started with TFX pipelines](community/ml_ops/stage3/get_started_with_tfx_pipeline.ipynb)
-
-In this tutorial, you learn how to use TensorFlow Extended (TFX) with `Vertex AI Pipelines`.
-
-The steps performed include:
-
-- Create a TFX e2e pipeline.
-- Execute the pipeline locally.
-- Execute the pipeline on Google Cloud using `Vertex AI Training`
-- Execute the pipeline using `Vertex AI Pipelines`.
-
-[Get started with BigQuery ML pipeline components](community/ml_ops/stage3/get_started_with_bqml_pipeline_components.ipynb)
-
-In this tutorial, you learn how to use prebuilt `Google Cloud Pipeline Components` for `BigQuery ML`.
-
-The steps performed include:
-
-- Construct a pipeline for:
-    - Training BigQuery ML model.
-    - Evaluating the BigQuery ML model.
-    - Exporting the BigQuery ML model.
-    - Importing the BigQuery ML model to a Vertex AI model.
-    - Deploy the Vertex AI model.
-- Execute a Vertex AI pipeline.
-- Make a prediction with the deployed Vertex AI model.
-
-[Get started with AutoML tabular pipeline workflows](community/ml_ops/stage3/get_started_with_automl_tabular_pipeline_workflow.ipynb)
-
-In this tutorial, you learn how to use `AutoML Tabular Pipeline Template` for training, exporting and tuning an AutoML tabular model.
-
-The steps performed include:
-
-- Define training specification.
-    - Dataset specification
-    - Hyperparameter overide specification
-    - machine specifications
-- Construct tabular workflow pipeline.
-- Compile and execute pipeline.
-- View evaluation metrics artifact.
-- Export AutoML model as an OSS TF model.
-- Create `Endpoint` resource.
-- Deploy exported OSS TF model.
-- Make a prediction.
-
-[Get started with rapid prototyping with AutoML and BigQuery ML](community/ml_ops/stage3/get_started_with_rapid_prototyping_bqml_automl.ipynb)
-
-In this tutorial, you learn how to use `Vertex AI Predictions` for rapid prototyping a model.
-
-The steps performed include:
-
-- Creating a BigQuery and Vertex AI training dataset.
-- Training a BigQuery ML and AutoML model.
-- Extracting evaluation metrics from the BigQueryML and AutoML models.
-- Selecting the best trained model.
-- Deploying the best trained model.
-- Testing the deployed model infrastructure.
-
-[Get started with AutoML pipeline components](community/ml_ops/stage3/get_started_with_automl_pipeline_components.ipynb)
-
-In this tutorial, you learn how to use prebuilt `Google Cloud Pipeline Components` for `Vertex AI AutoML`.
-
-The steps performed include:
-
-- Construct a pipeline for:
-    - Training a Vertex AI AutoML trained model.
-    - Test the serving binary with a batch prediction job.
-    - Deploying a Vertex AI AutoML trained model.
-- Execute a Vertex AI pipeline.
-
-
-[Get started with BigQuery and TFDV pipeline components](community/ml_ops/stage3/get_started_with_bq_tfdv_pipeline_components.ipynb)
-
-In this tutorial, you learn how to use build lightweight Python components for BigQuery and TensorFlow Data Validation.
-
-The steps performed include:
-
-- Build and execute a pipeline component for creating a Vertex AI Tabular Dataset from a BigQuery table.
-- Build and execute a pipeline component for generating TFDV statistics and schema from a Vertex AI Tabular Dataset.
-- Execute a Vertex AI pipeline.
+```
 
 ### E2E Stage Example
 
-Stage 4: Evaluation

notebooks/community/ml_ops/stage5/README.md

@@ -25,9 +25,10 @@ The fifth stage in MLOps is deployment to production of the blessed model, which
 ### Get Started
 
 
-[Get started with Vertex AI Endpoints](community/ml_ops/stage5/get_started_with_vertex_endpoints.ipynb)
+[Get started with Vertex AI Endpoints](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage5/get_started_with_vertex_endpoints.ipynb)
 
-In this tutorial, you learn how to use `Vertex AI Endpoint` resources.
+```
+Learn how to use `Vertex AI Endpoint` resources.
 
 The steps performed include:
 
@@ -46,9 +47,13 @@ The steps performed include:
 - In pipeline: Create an `Endpoint` resource and deploy an existing `Model` resource to the `Endpoint` resource.
 - In pipeline: Deploy an existing `Model` resource to an existing `Endpoint` resource.
 
-[Get started with Vertex AI Endpoint and shared VM](community/ml_ops/stage5/get_started_with_vertex_endpoint_and_shared_vm.ipynb)
+```
 
-In this tutorial, you learn how to use deployment resource pools for deploying models. A deployment resouce pool provides one with the ability to co-host more than one model on the same (shared) VM.
+
+[Get started with Vertex AI Endpoint and shared VM](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage5/get_started_with_vertex_endpoint_and_shared_vm.ipynb)
+
+```
+Learn how to use deployment resource pools for deploying models.
 
 The steps performed include:
 
@@ -62,9 +67,13 @@ The steps performed include:
 - Make a prediction request with first deployed model (model A).
 - Make a prediction request with second deployed model (model B).
 
-[Get started with configuring autoscaling for Vertex AI Endpoint deployment](community/ml_ops/stage5/get_started_with_autoscaling.ipynb)
+```
 
-In this tutorial, you learn how to use fine-tune control auto-scaling configuration when deploying a `Model` resource to an `Endpoint` resource.
+
+[Get started with configuring autoscaling for Vertex AI Endpoint deployment](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage5/get_started_with_autoscaling.ipynb)
+
+```
+Learn how to use fine-tune control auto-scaling configuration when deploying a `Model` resource to an `Endpoint` resource.
 
 The steps performed include:
 
@@ -78,9 +87,13 @@ The steps performed include:
 - Fine-tune scaling thresholds for GPU utilization.
 - Deploy mix of CPU and GPU model instances with auto-scaling to an `Endpoint` resource.
 
-[Get started with Vertex AI Private Endpoints](community/ml_ops/stage5/get_started_with_vertex_private_endpoints.ipynb)
+```
 
-In this tutorial, you learn how to use `Vertex AI Private Endpoint` resources.
+
+[Get started with Vertex AI Private Endpoints](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage5/get_started_with_vertex_private_endpoints.ipynb)
+
+```
+Learn how to use `Vertex AI Private Endpoint` resources.
 
 The steps performed include:
 
@@ -89,8 +102,5 @@ The steps performed include:
 - Configuring the serving binary of a `Model` resource for deployment to a `Private Endpoint` resource.
 - Deploying a `Model` resource to a `Private Endpoint` resource.
 - Send a prediction request to a `Private Endpoint`
-- Enable two additional APIs: Service Networking and Cloud DNS.
-- Add Compute Admin Network role to your (default) service account.
-- Issue two gcloud commands to setup the VPC peering for your service account.
-- There is *currently* no SDK support yet, so private endpoint is created with GAPIC client and has an extra argument for the peering network.
-- To send a request, you can't use SDK/GAPIC since they do a HTTP internet request. Instead, you use curl to send a peer-to-peer request.
+
+```

notebooks/community/ml_ops/stage5/get_started_with_vertex_endpoint_and_shared_vm.ipynb

@@ -159,9 +159,9 @@
         "\n",
         "# Install the packages\n",
         "\n",
-        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG -q\n",
-        "! pip3 install --upgrade tensorflow $USER_FLAG -q\n",
-        "! pip3 install --upgrade tensorflow-hub $USER_FLAG -q"
+        "! pip3 install --upgrade google-cloud-aiplatform \\\n",
+        "                         tensorflow \\\n",
+        "                         tensorflow-hub $USER_FLAG -q"
       ]
     },
     {
@@ -307,22 +307,29 @@
         "id": "timestamp"
       },
       "source": [
-        "#### Timestamp\n",
+        "#### UUID\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "timestamp"
+        "id": "84Vdv7R-QEH6"
       },
       "outputs": [],
       "source": [
-        "from datetime import datetime\n",
+        "import random\n",
+        "import string\n",
         "\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
+        "\n",
+        "# Generate a uuid of a specifed length(default=8)\n",
+        "def generate_uuid(length: int = 8) -> str:\n",
+        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
+        "\n",
+        "\n",
+        "UUID = generate_uuid()"
       ]
     },
     {
@@ -421,7 +428,7 @@
       "outputs": [],
       "source": [
         "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
         "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
       ]
     },
@@ -523,7 +530,7 @@
         "\n",
         "Setup up the following constants for Vertex AI:\n",
         "\n",
-        "- `API_ENDPOINT`: The Vertex AI API service endpoint for `Endpoint` services."
+        "- `API_ENDPOINT`: The Vertex AI API service endpoint."
       ]
     },
     {
@@ -538,46 +545,10 @@
         "API_ENDPOINT = \"{}-aiplatform.googleapis.com\".format(REGION)\n",
         "\n",
         "# Vertex location root path for your dataset, model and endpoint resources\n",
-        "PARENT = \"projects/\" + PROJECT_ID + \"/locations/\" + REGION"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "clients:metadata"
-      },
-      "source": [
-        "## Set up clients\n",
+        "PARENT = \"projects/\" + PROJECT_ID + \"/locations/\" + REGION\n",
         "\n",
-        "The Vertex  works as a client/server model. On your side (the Python script) you will create a client that sends requests and receives responses from the Vertex AI server.\n",
-        "\n",
-        "You will use different clients in this tutorial for different steps in the workflow. So set them all up upfront.\n",
-        "\n",
-        "- Endpoint Service for creating endpoints, and deploying models to endpoints."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "clients:metadata"
-      },
-      "outputs": [],
-      "source": [
         "# client options same for all services\n",
-        "client_options = {\"api_endpoint\": API_ENDPOINT}\n",
-        "\n",
-        "\n",
-        "def create_endpoint_client():\n",
-        "    client = aip_beta.EndpointServiceClient(client_options=client_options)\n",
-        "    return client\n",
-        "\n",
-        "\n",
-        "clients = {}\n",
-        "clients[\"endpoint\"] = create_endpoint_client()\n",
-        "\n",
-        "for client in clients.items():\n",
-        "    print(client)"
+        "client_options = {\"api_endpoint\": API_ENDPOINT}"
       ]
     },
     {
@@ -592,7 +563,7 @@
         "\n",
         "Set the variables `DEPLOY_GPU/DEPLOY_NGPU` to use a container image supporting a GPU and the number of GPUs allocated to the virtual machine (VM) instance. For example, to use a GPU container image with 4 Nvidia Telsa K80 GPUs allocated to each VM, you would specify:\n",
         "\n",
-        "    (aip.AcceleratorType.NVIDIA_TESLA_K80, 4)\n",
+        "    (aip.gapic.AcceleratorType.NVIDIA_TESLA_K80, 4)\n",
         "\n",
         "\n",
         "Otherwise specify `(None, None)` to use a container image to run on a CPU.\n",
@@ -902,7 +873,7 @@
       "outputs": [],
       "source": [
         "model_icn = aiplatform.Model.upload(\n",
-        "    display_name=\"icn_\" + TIMESTAMP,\n",
+        "    display_name=\"icn_\" + UUID,\n",
         "    artifact_uri=MODEL_ICN_DIR,\n",
         "    serving_container_image_uri=DEPLOY_IMAGE,\n",
         ")\n",
@@ -1013,7 +984,7 @@
       "outputs": [],
       "source": [
         "model_use = aiplatform.Model.upload(\n",
-        "    display_name=\"icn_\" + TIMESTAMP,\n",
+        "    display_name=\"icn_\" + UUID,\n",
         "    artifact_uri=MODEL_USE_DIR,\n",
         "    serving_container_image_uri=DEPLOY_IMAGE,\n",
         ")\n",
@@ -1029,64 +1000,52 @@
       "source": [
         "## Creating a deployment resource pool\n",
         "\n",
-        "Currently, creating deploynent resource pools is only supported via the REST-based API (e.g., CURL).\n",
+        "Currently, creating deploynent resource pools is only supported via the REST-based API (e.g., CURL) and GAPIC APIs (Python).\n",
         "\n",
-        "Use `CreateDeploymentResourcePool` API to create a resource pool, with the following configuration:\n",
+        "Use `create_deployment_resource_pool` API to create a resource pool, with the following configuration:\n",
         "\n",
         "- `dedicated_resources`: Compute (HW) resources to allocate for the shared vm.\n",
         "- `min_replica_count`: Auto-scaling, the minimum number of compute nodes.\n",
         "- `max_replica_count`: Auto-scaling, the maximum number of compute nodes.\n",
         "\n",
-        "Learn more about [Deployment Resource Pools]()."
+        "Learn more about [Deployment Resource Pools](https://googleapis.dev/python/aiplatform/latest/aiplatform_v1beta1/deployment_resource_pool_service.html)."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "YiBmoiWYcMQt"
+        "id": "90c51b6cf34a"
       },
       "outputs": [],
       "source": [
-        "DEPLOYMENT_RESOURCE_POOL_ID = \"shared-vm\"  # @param {type: \"string\"}"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "0CHPJ4h-Slgs"
-      },
-      "outputs": [],
-      "source": [
-        "import json\n",
-        "import pprint\n",
-        "pp = pprint.PrettyPrinter(indent=4)\n",
-        "\n",
+        "DEPLOYMENT_RESOURCE_POOL_ID = f\"shared-vm-{UUID}\"  # @param {type: \"string\"}\n",
         "MIN_NODES = 1\n",
         "MAX_NODES = 2\n",
         "\n",
-        "CREATE_RP_PAYLOAD = {\n",
-        "  \"deployment_resource_pool\":{\n",
-        "    \"dedicated_resources\":{\n",
-        "      \"machine_spec\":{\n",
-        "        \"machine_type\": DEPLOY_COMPUTE\n",
-        "      },\n",
-        "      \"min_replica_count\": MIN_NODES, \n",
-        "      \"max_replica_count\": MAX_NODES\n",
-        "    }\n",
-        "  },\n",
-        "  \"deployment_resource_pool_id\":DEPLOYMENT_RESOURCE_POOL_ID\n",
-        "}\n",
-        "CREATE_RP_REQUEST=json.dumps(CREATE_RP_PAYLOAD)\n",
-        "pp.pprint(\"CREATE_RP_REQUEST: \" + CREATE_RP_REQUEST)\n",
+        "# Initialize request argument(s)\n",
+        "deployment_resource_pool = aip_beta.DeploymentResourcePool()\n",
+        "deployment_resource_pool.dedicated_resources.min_replica_count = MIN_NODES\n",
+        "deployment_resource_pool.dedicated_resources.max_replica_count = MAX_NODES\n",
+        "deployment_resource_pool.dedicated_resources.machine_spec.machine_type = DEPLOY_COMPUTE\n",
         "\n",
-        "! curl \\\n",
-        "-X POST \\\n",
-        "-H \"Authorization: Bearer $(gcloud auth print-access-token)\" \\\n",
-        "-H \"Content-Type: application/json\" \\\n",
-        "https://{REGION}-aiplatform.googleapis.com/v1beta1/projects/{PROJECT_ID}/locations/{REGION}/deploymentResourcePools \\\n",
-        "-d '{CREATE_RP_REQUEST}'"
+        "request = aip_beta.CreateDeploymentResourcePoolRequest(\n",
+        "    parent=f\"projects/{PROJECT_ID}/locations/{REGION}\",\n",
+        "    deployment_resource_pool=deployment_resource_pool,\n",
+        "    deployment_resource_pool_id=DEPLOYMENT_RESOURCE_POOL_ID,\n",
+        ")\n",
+        "\n",
+        "pool_client = aip_beta.services.deployment_resource_pool_service.DeploymentResourcePoolServiceClient(\n",
+        "    client_options=client_options\n",
+        ")\n",
+        "\n",
+        "op = pool_client.create_deployment_resource_pool(request=request)\n",
+        "print(op)\n",
+        "\n",
+        "result = op.result()\n",
+        "print(result)\n",
+        "\n",
+        "deployment_pool_id = result.name"
       ]
     },
     {
@@ -1099,21 +1058,19 @@
         "\n",
         "Use `GetDeploymentResourcePool` API to check out the deploynent resource pool that you created. \n",
         "\n",
-        "Learn more about [Get Deployment Resource Pool](https://source.corp.google.com/piper///depot/google3/google/cloud/aiplatform/master/deployment_resource_pool_service.proto;l=75?q=deployment_resource_pool&sq=package:piper%20file:%2F%2Fdepot%2Fgoogle3%20-file:google3%2Fexperimental)."
+        "Learn more about [Get Deployment Resource Pool](https://googleapis.dev/python/aiplatform/latest/aiplatform_v1beta1/deployment_resource_pool_service.html)."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "6wTLyhPraFah"
+        "id": "b740253903c0"
       },
       "outputs": [],
       "source": [
-        "! curl -X GET \\\n",
-        "-H \"Authorization: Bearer $(gcloud auth print-access-token)\" \\\n",
-        "-H \"Content-Type: application/json\" \\\n",
-        "https://{REGION}-aiplatform.googleapis.com/v1beta1/projects/{PROJECT_ID}/locations/{REGION}/deploymentResourcePools/{DEPLOYMENT_RESOURCE_POOL_ID}"
+        "response = pool_client.get_deployment_resource_pool(name=deployment_pool_id)\n",
+        "print(response)"
       ]
     },
     {
@@ -1126,21 +1083,22 @@
         "\n",
         "Use `ListDeploymentResourcePools` API to list all the deployment resource pools. \n",
         "\n",
-        "Learn more about [Listing Deployment Resource Pools](https://source.corp.google.com/piper///depot/google3/google/cloud/aiplatform/master/deployment_resource_pool_service.proto;l=101?q=deployment_resource_pool&sq=package:piper%20file:%2F%2Fdepot%2Fgoogle3%20-file:google3%2Fexperimental)."
+        "Learn more about [Listing Deployment Resource Pools](https://googleapis.dev/python/aiplatform/latest/aiplatform_v1beta1/deployment_resource_pool_service.html)."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "Pxls4sNnaltU"
+        "id": "3ebfd007bff2"
       },
       "outputs": [],
       "source": [
-        "! curl -X GET \\\n",
-        "-H \"Authorization: Bearer $(gcloud auth print-access-token)\" \\\n",
-        "-H \"Content-Type: application/json\" \\\n",
-        "https://{REGION}-aiplatform.googleapis.com/v1beta1/projects/{PROJECT_ID}/locations/{REGION}/deploymentResourcePools"
+        "pools = pool_client.list_deployment_resource_pools(\n",
+        "    parent=f\"projects/{PROJECT_ID}/locations/{REGION}\"\n",
+        ")\n",
+        "for pool in pools:\n",
+        "    print(pool)"
       ]
     },
     {
@@ -1170,11 +1128,11 @@
       },
       "outputs": [],
       "source": [
-        "endpoint_icn = aiplatform.Endpoint.create(display_name=\"icn_\" + TIMESTAMP)\n",
+        "endpoint_icn = aiplatform.Endpoint.create(display_name=\"icn_\" + UUID)\n",
         "\n",
         "print(endpoint_icn)\n",
         "\n",
-        "endpoint_use = aiplatform.Endpoint.create(display_name=\"use_\" + TIMESTAMP)\n",
+        "endpoint_use = aiplatform.Endpoint.create(display_name=\"use_\" + UUID)\n",
         "\n",
         "print(endpoint_use)"
       ]
@@ -1204,6 +1162,12 @@
       },
       "outputs": [],
       "source": [
+        "import json\n",
+        "import pprint\n",
+        "\n",
+        "pp = pprint.PrettyPrinter(indent=4)\n",
+        "\n",
+        "\n",
         "SHARED_RESOURCE = \"projects/{project_id}/locations/{region}/deploymentResourcePools/{deployment_resource_pool_id}\".format(\n",
         "    project_id=PROJECT_ID,\n",
         "    region=REGION,\n",
@@ -1363,18 +1327,27 @@
         "    time.sleep(30)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "52248c450776"
+      },
+      "source": [
+        "### Get deployment details for the endpoint\n",
+        "\n",
+        "List the deployed models on the endpoint."
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "86a659bf60f0"
+        "id": "3b768614e7c6"
       },
       "outputs": [],
       "source": [
-        "! curl -X GET \\\n",
-        " -H \"Authorization: Bearer $(gcloud auth print-access-token)\" \\\n",
-        " -H \"Content-Type: application/json\" \\\n",
-        "https://{REGION}-aiplatform.googleapis.com/v1/projects/759209241365/locations/us-central1/endpoints/2259566763823857664"
+        "print(endpoint_icn.list_models())\n",
+        "print(endpoint_use.list_models())"
       ]
     },
     {
@@ -1557,21 +1530,19 @@
       "source": [
         "#### Delete the `DeploymentResourcePool`\n",
         "\n",
-        "The method 'delete()' will delete your deployment resource pool."
+        "The method 'delete_deployment_resource_pool()' will delete your deployment resource pool."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "ac40cc1d594a"
+        "id": "b76a4de1e57e"
       },
       "outputs": [],
       "source": [
-        "! curl -X DELETE \\\n",
-        "-H \"Authorization: Bearer $(gcloud auth print-access-token)\" \\\n",
-        "-H \"Content-Type: application/json\" \\\n",
-        "https://{REGION}-aiplatform.googleapis.com/v1beta1/projects/{PROJECT_ID}/locations/{REGION}/deploymentResourcePools/{DEPLOYMENT_RESOURCE_POOL_ID}"
+        "response = pool_client.delete_deployment_resource_pool(name=deployment_pool_id)\n",
+        "print(response)"
       ]
     },
     {

notebooks/community/ml_ops/stage6/README.md

@@ -30,19 +30,23 @@ This stage may be done entirely by MLOps. We recommend:
 ### Get Started
 
 
-[Get started with Vertex AI Batch Prediction for AutoML image models](community/ml_ops/stage6/get_started_with_automl_image_model_batch.ipynb)
+[Get started with Vertex AI Batch Prediction for AutoML image models](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage6/get_started_with_automl_image_model_batch.ipynb)
 
-In this tutorial, you create an AutoML image classification model from a Python script, and then do a batch prediction using the Vertex AI SDK. 
+```
+Learn how to create an AutoML image classification model from a Python script, and then do a batch prediction using the Vertex AI SDK.
 
 The steps performed include:
 
 - Create a Vertex `Dataset` resource.
 - Train an `AutoML` image classification model.
 - Make a batch prediction with JSONL input.
+```
 
-[Get started with Vertex AI Matching Engine and Swivel builtin algorithm](community/ml_ops/stage6/get_started_with_matching_engine_swivel.ipynb)
 
-In this notebook, you learn how to train custom embeddings using Vertex AI Pipelines and subsequently train and deploy a matching engine index using the embeddings.
+[Get started with Vertex AI Matching Engine and Swivel builtin algorithm](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage6/get_started_with_matching_engine_swivel.ipynb)
+
+```
+Learn how to train custom embeddings using Vertex AI Pipelines and subsequently train and deploy a matching engine index using the embeddings.
 
 The steps performed include:
 
@@ -54,9 +58,13 @@ The steps performed include:
 6. Deploy the `Matching Engine Index` to a `Index Endpoint`.
 7. Make a matching engine prediction request.
 
-[Get started with Vertex AI Matching Engine](community/ml_ops/stage6/get_started_with_matching_engine.ipynb)
+```
 
-In this notebook, you learn how to create Approximate Nearest Neighbor (ANN) Index, query against indexes. 
+
+[Get started with Vertex AI Matching Engine](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage6/get_started_with_matching_engine.ipynb)
+
+```
+Learn how to create Approximate Nearest Neighbor (ANN) Index, query against indexes.
 
 The steps performed include:
 
@@ -67,10 +75,13 @@ The steps performed include:
 - Deploy brute force Index.
 - Perform calibration between ANN and brute force index.
 
-[Get started with Vertex AI Matching Engine and Two Towers builtin algorithm](community/ml_ops/stage6/get_started_with_matching_engine_twotowers.ipynb)
+```
 
 
-In this notebook, you learn how to use the `Two-Tower` builtin algorithms for generating embeddings for a dataset, for use with generating an `Matching Engine Index`, with the `Vertex AI Matching Engine` service.
+[Get started with Vertex AI Matching Engine and Two Towers builtin algorithm](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage6/get_started_with_matching_engine_twotowers.ipynb)
+
+```
+Learn how to use the `Two-Tower` builtin algorithms for generating embeddings for a dataset, for use with generating an `Matching Engine Index`, with the `Vertex AI Matching Engine` service.
 
 The steps performed include:
 
@@ -83,9 +94,30 @@ The steps performed include:
 7. Deploy the `Matching Engine Index` to a `Index Endpoint`.
 8. Make a matching engine prediction request.
 
-[Get started with Vertex AI Batch Prediction for custom tabular models](community/ml_ops/stage6/get_started_with_custom_tabular_model_batch.ipynb)
+```
 
-In this tutorial, you learn how to use `Vertex AI Batch Prediction`  with a custom tabular model.
+
+[Get started with TensorFlow Serving with Vertex AI Prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage6/get_started_with_tf_serving_tabular.ipynb)
+
+```
+Learn how to use `Vertex AI Prediction` on a `Vertex AI Endpoint` resource with `TensorFlow Serving` serving binary.
+
+The steps performed include:
+
+- Download a pretrained TensorFlow tabular model.
+- Upload the TensorFlow model as a `Vertex AI Model` resource.
+- Creating an `Endpoint` resource.
+- Deploying the `Model` resource to an `Endpoint` resource with `TensorFlow Serving` serving binary.
+- Make an online prediction to the `Model` resource instance deployed to the `Endpoint` resource.
+- Make a batch prediction to the `Model` resource instance.
+
+```
+
+
+[Get started with Vertex AI Batch Prediction for custom tabular models](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage6/get_started_with_custom_tabular_model_batch.ipynb)
+
+```
+Learn how to use `Vertex AI Batch Prediction`  with a custom tabular model.
 
 The steps performed include:
 
@@ -93,10 +125,13 @@ The steps performed include:
 - Make batch prediction to the `Model` resource, in JSONL format.
 - Make batch prediction to the `Model` resource, in CSV format.
 - Make batch prediction to the `Model` resource, in BigQuery format.
+```
 
-[Get started with Optimized TensorFlow Enterprise container with Vertex AI Prediction / text models](community/ml_ops/stage6/get_started_with_optimized_tfe_bert.ipynb)
 
-In this tutorial, you learn how to use `TensorFlow Enterprise Optimized` container for TensorFlow models deployed to a `Vertex AI Endpoint` resource.
+[Get started with Optimized TensorFlow Enterprise container with Vertex AI Prediction / text models](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage6/get_started_with_optimized_tfe_bert.ipynb)
+
+```
+Learn how to use `TensorFlow Enterprise Optimized` container for TensorFlow models deployed to a `Vertex AI Endpoint` resource.
 
 The steps performed include:
 
@@ -113,9 +148,13 @@ The steps performed include:
 - Deploy the `Model` resoure with then `TensorFlow Enterprise Optimized` to the `Private Endpoint` resource.
 - Make an online prediction request to the `Private Endpoint` resource.
 
-[Get started with Vertex AI Batch Prediction and Explainable AI for AutoML tabular models](community/ml_ops/stage6/get_started_with_automl_tabular_model_batch.ipynb)
+```
 
-In this tutorial, you create an AutoML tabular binary classification model from a Python script, and then do a batch prediction with Explainable AI using the Vertex AI SDK. You can alternatively create and deploy models using the `gcloud` command-line tool or online using the Cloud Console.
+
+[Get started with Vertex AI Batch Prediction and Explainable AI for AutoML tabular models](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage6/get_started_with_automl_tabular_model_batch.ipynb)
+
+```
+In this tutorial, you create an AutoML tabular binary classification model from a Python script, and then do a batch prediction with Explainable AI using the Vertex AI SDK.
 
 The steps performed include:
 
@@ -126,10 +165,13 @@ The steps performed include:
 - Make a batch prediction with JSONL list input.
 - Make a batch prediction with BigQuery table input.
 - Make a batch prediction with explanations.
+```
 
-[Get started with re-importing AutoML tabular models](community/ml_ops/stage6/get_started_with_automl_tabular_exported_deploy.ipynb)
 
-In this tutorial, you learn how to use `AutoML Tabular` for re-importing exported model artifacts as a `Model` resource. This is useful for example, if one wants to move the exported model across projects.
+[Get started with re-importing AutoML tabular models](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage6/get_started_with_automl_tabular_exported_deploy.ipynb)
+
+```
+Learn how to use `AutoML Tabular` for re-importing exported model artifacts as a `Model` resource.
 
 The steps performed include:
 
@@ -138,19 +180,44 @@ The steps performed include:
 - Deploy the `Model` resource to the `Endpoint` resource.
 - Make a prediction.
 
-[Get started with Vertex AI Batch Prediction for AutoML text models](community/ml_ops/stage6/get_started_with_automl_text_model_batch.ipynb)
+```
 
-In this tutorial, you learn how to use `Vertex AI Batch Prediction`  with a `AutoML` text model.
+
+[Get started with Vertex AI Online Prediction for XGBoost custom models](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage6/get_started_with_xgboost_model_online.ipynb)
+
+```
+In this tutorial, you deploy an XGBoost model, and then do an online prediction using the Vertex AI SDK.
+
+The steps performed include:
+
+- Upload an XGBoost model as a Vertex AI Model resource.
+- Deploy the model to a Vertex AI Endpoint resource.
+- Make an online prediction.
+- Construct a Vertex AI Pipeline:
+    - Upload an XGBoost model as a Vertex AI Model resource.
+    - Deploy the model to a Vertex AI Endpoint resource.
+- Make an online prediction
+
+```
+
+
+[Get started with Vertex AI Batch Prediction for AutoML text models](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage6/get_started_with_automl_text_model_batch.ipynb)
+
+```
+Learn how to use `Vertex AI Batch Prediction`  with a `AutoML` text model.
 
 The steps performed include:
 
 - Create a Vertex `Dataset` resource.
 - Train an `AutoML` model.
 - Make a batch prediction with JSONL input
+```
 
-[Get started with Vertex AI Prediction for AutoML text models](community/ml_ops/stage6/get_started_with_automl_text_model_online.ipynb)
 
-In this tutorial, you learn how to use `Vertex AI Prediction`  with a `AutoML` text model.
+[Get started with Vertex AI Prediction for AutoML text models](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage6/get_started_with_automl_text_model_online.ipynb)
+
+```
+Learn how to use `Vertex AI Prediction`  with a `AutoML` text model.
 
 The steps performed include:
 
@@ -159,9 +226,13 @@ The steps performed include:
 - Deploy the model to an `Endpoint` resource.
 - Make an online prediction.
 
-[Get started with TensorFlow serving functions with Vertex AI Raw Prediction](community/ml_ops/stage6/get_started_with_raw_predict.ipynb)
+```
 
-In this tutorial, you learn how to use `Vertex AI Raw Prediction` on a `Vertex AI Endpoint` resource.
+
+[Get started with TensorFlow serving functions with Vertex AI Raw Prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage6/get_started_with_raw_predict.ipynb)
+
+```
+Learn how to use `Vertex AI Raw Prediction` on a `Vertex AI Endpoint` resource.
 
 The steps performed include:
 
@@ -171,9 +242,13 @@ The steps performed include:
 - Deploying the `Model` resource to an `Endpoint` resource.
 - Make an online raw prediction to the `Model` resource instance deployed to the `Endpoint` resource.
 
-[Get started with TensorFlow serving functions with Vertex AI Prediction](community/ml_ops/stage6/get_started_with_tf_serving_function.ipynb)
+```
 
-In this tutorial, you learn how to use `Vertex AI Prediction` on a `Vertex AI Endpoint` resource with a serving function.
+
+[Get started with TensorFlow serving functions with Vertex AI Prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage6/get_started_with_tf_serving_function.ipynb)
+
+```
+Learn how to use `Vertex AI Prediction` on a `Vertex AI Endpoint` resource with a serving function.
 
 The steps performed include:
 
@@ -184,13 +259,17 @@ The steps performed include:
 - Deploying the `Model` resource to an `Endpoint` resource.
 - Make an online prediction to the `Model` resource instance deployed to the `Endpoint` resource.
 
-[Get started with Vertex Explainable AI using custom deployment container](community/ml_ops/stage6/get_started_with_xai_and_custom_server.ipynb)
+```
 
-In this tutorial, you learn to build a custom container to serve a PyTorch model on `Vertex AI Endpoint`. 
+
+[Get started with Vertex Explainable AI using custom deployment container](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage6/get_started_with_xai_and_custom_server.ipynb)
+
+```
+Learn to build a custom container to serve a PyTorch model on `Vertex AI Endpoint`.
 
 The steps performed include:
 
-- Locally train a Pytorch tabular classifier.
+- Locally train a PyTorch tabular classifier.
 - Locally test the trained model.
 - Build a HTTP server using FastAPI.
 - Create a custom serving container with the trained model and FastAPI server.
@@ -201,9 +280,13 @@ The steps performed include:
 - Make a prediction request to the deployed custom serving container.
 - Make an explanation request to the deployed custom serving container.
 
-[Get started with Vertex AI Online Prediction for AutoML image models](community/ml_ops/stage6/get_started_with_automl_image_model_online.ipynb)
+```
 
-In this tutorial, you create an AutoML image classification model from a Python script, and then do an online prediction using the Vertex AI SDK. 
+
+[Get started with Vertex AI Online Prediction for AutoML image models](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage6/get_started_with_automl_image_model_online.ipynb)
+
+```
+In this tutorial, you create an AutoML image classification model from a Python script, and then do an online prediction using the Vertex AI SDK.
 
 The steps performed include:
 
@@ -211,9 +294,13 @@ The steps performed include:
 - Train an `AutoML` image classification model.
 - Make an online prediction.
 
-[Get started with FastAPI with Vertex AI Prediction](community/ml_ops/stage6/get_started_with_fastapi.ipynb)
+```
 
-In this tutorial, you learn how to use `Vertex AI Prediction` on a `Vertex AI Endpoint` with a custom serving binary using `FastAPI`.
+
+[Get started with FastAPI with Vertex AI Prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage6/get_started_with_fastapi.ipynb)
+
+```
+Learn how to use `Vertex AI Prediction` on a `Vertex AI Endpoint` with a custom serving binary using `FastAPI`.
 
 The steps performed include:
 
@@ -224,9 +311,13 @@ The steps performed include:
 - Deploying the `Model` resource to an `Endpoint` resource with `FastAPI` custom serving binary.
 - Make an online prediction to the `Model` resource instance deployed to the `Endpoint` resource.
 
-[Get started with Vertex AI Online Prediction for AutoML tabular models](community/ml_ops/stage6/get_started_with_automl_tabular_model_online.ipynb)
+```
 
-In this tutorial, you create an AutoML tabular binary classification model from a Python script, and then do an online prediction using the Vertex AI SDK. 
+
+[Get started with Vertex AI Online Prediction for AutoML tabular models](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage6/get_started_with_automl_tabular_model_online.ipynb)
+
+```
+In this tutorial, you create an AutoML tabular binary classification model from a Python script, and then do an online prediction using the Vertex AI SDK.
 
 The steps performed include:
 
@@ -236,9 +327,13 @@ The steps performed include:
 - Make an online prediction.
 - Make an online prediction with explanations.
 
-[Get started with TensorFlow Serving with Vertex AI Prediction](community/ml_ops/stage6/get_started_with_tf_serving.ipynb)
+```
 
-In this tutorial, you learn how to use `Vertex AI Prediction` on a `Vertex AI Endpoint` resource with `TensorFlow Serving` serving binary.
+
+[Get started with TensorFlow Serving with Vertex AI Prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage6/get_started_with_tf_serving.ipynb)
+
+```
+Learn how to use `Vertex AI Prediction` on a `Vertex AI Endpoint` resource with `TensorFlow Serving` serving binary.
 
 The steps performed include:
 
@@ -250,9 +345,13 @@ The steps performed include:
 - Make an online prediction to the `Model` resource instance deployed to the `Endpoint` resource.
 - Make a batch prediction to the `Model` resource instance.
 
-[Get started with Custom Prediction Routine (CPR)](community/ml_ops/stage6/get_started_with_cpr.ipynb)
+```
 
-In this tutorial, you learn how to use Custom Prediction Routine (CPR) for `Vertex AI Predictions`.
+
+[Get started with Custom Prediction Routine (CPR)](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage6/get_started_with_cpr.ipynb)
+
+```
+Learn how to use Custom Prediction Routine (CPR) for `Vertex AI Predictions`.
 
 The steps performed include:
 
@@ -278,19 +377,26 @@ The steps performed include:
     - Upload and deploy the model serving container to Vertex AI Endpoint.
     - Make a prediction request.
 
-[Get started with Vertex AI Batch Prediction for custom text models](community/ml_ops/stage6/get_started_with_custom_text_model_batch.ipynb)
+```
 
-In this tutorial, you learn how to use `Vertex AI Batch Prediction`  with a custom text model.
+
+[Get started with Vertex AI Batch Prediction for custom text models](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage6/get_started_with_custom_text_model_batch.ipynb)
+
+```
+Learn how to use `Vertex AI Batch Prediction`  with a custom text model.
 
 The steps performed include:
 
 - Download a pretrained TensorFlow RNN model.
 - Upload the pretrained model as a `Vertex AI Model` resource.
 - Make batch prediction to the `Model` resource, in JSONL format.
+```
 
-[Get started with NVIDIA Triton server](community/ml_ops/stage6/get_started_with_nvidia_triton_serving.ipynb)
 
-In this tutorial, you deploy a container running Nvidia Triton Server with a `Vertex AI Model` resource to a `Vertex AI Endpoint` for making online predictions.
+[Get started with NVIDIA Triton server](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage6/get_started_with_nvidia_triton_serving.ipynb)
+
+```
+Learn how to deploy a container running Nvidia Triton Server with a `Vertex AI Model` resource to a `Vertex AI Endpoint` for making online predictions.
 
 The steps performed in this tutorial include:
 
@@ -302,9 +408,13 @@ The steps performed in this tutorial include:
 - Make a prediction request
 - Undeploy the `Model` resource and delete the `Endpoint`
 
-[Get started with Vertex AI Batch Prediction for custom image models](community/ml_ops/stage6/get_started_with_custom_image_model_batch.ipynb)
+```
 
-In this tutorial, you learn how to use `Vertex AI Batch Prediction`  with a custom image model.
+
+[Get started with Vertex AI Batch Prediction for custom image models](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage6/get_started_with_custom_image_model_batch.ipynb)
+
+```
+Learn how to use `Vertex AI Batch Prediction`  with a custom image model.
 
 The steps performed include:
 
@@ -314,13 +424,17 @@ The steps performed include:
 - Create a serving function to receive compressed image data, and output decomopressed preprocessed data for the model input.
 - Upload the TensorFlow Hub model and serving function as a `Vertex AI Model` resource.
 - Make batch prediction with compressed image data to the `Model` resource, in File-List format.
+```
 
-[Get started with Vertex AI Batch Prediction for AutoML video models](community/ml_ops/stage6/get_started_with_automl_video_model_batch.ipynb)
 
-In this tutorial, you learn how to use `Vertex AI Batch Prediction`  with a `AutoML` video model.
+[Get started with Vertex AI Batch Prediction for AutoML video models](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage6/get_started_with_automl_video_model_batch.ipynb)
+
+```
+Learn how to use `Vertex AI Batch Prediction`  with a `AutoML` video model.
 
 The steps performed include:
 
 - Create a Vertex `Dataset` resource.
 - Train an `AutoML` model.
-- Make a batch prediction with JSONL input.
+- Make a batch prediction with JSONL input
+```

notebooks/community/ml_ops/stage7/README.md

@@ -35,9 +35,28 @@ This stage may be done entirely by MLOps. We recommend:
 
 ### Get Started
 
-[Vertex AI Model Monitoring for custom tabular models with TensorFlow Serving container](community/ml_ops/stage7/get_started_with_model_monitoring_custom_tf_serving.ipynb)
+[Vertex AI Model Monitoring for XGBoost models](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage7/get_started_with_model_monitoring_xgboost.ipynb)
 
-In this notebook, you learn to use the `Vertex AI Model Monitoring` service to detect feature skew and drift in the input predict requests, for custom tabular models, using a custom deployment container. 
+```
+Learn to use the `Vertex AI Model Monitoring` service to detect feature skew and drift in the input predict requests for XGBoost models.
+
+The steps performed include:
+
+- Download a pre-trained XGBoost model.
+- Upload the pre-trained model as a `Model` resource.
+- Deploy the `Model` resource to the `Endpoint` resource.
+- Configure the `Endpoint` resource for model monitoring:
+  - drift detection only -- no access to training data.
+  - predefine the input schema to map feature alias names to the unnamed array input to the model.
+- Generate synthetic prediction requests for drift.
+
+```
+
+
+[Vertex AI Model Monitoring for custom tabular models with TensorFlow Serving container](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage7/get_started_with_model_monitoring_custom_tf_serving.ipynb)
+
+```
+Learn to use the `Vertex AI Model Monitoring` service to detect feature skew and drift in the input predict requests, for custom tabular models, using a custom deployment container.
 
 The steps performed include:
 
@@ -50,11 +69,13 @@ The steps performed include:
 - Generate synthetic prediction requests for drift.
 - Wait for email alert notification.
 
+```
 
 
-[Vertex AI Model Monitoring for AutoML tabular models](community/ml_ops/stage7/get_started_with_model_monitoring_automl.ipynb)
+[Vertex AI Model Monitoring for AutoML tabular models](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage7/get_started_with_model_monitoring_automl.ipynb)
 
-In this notebook, you learn to use the `Vertex AI Model Monitoring` service to detect feature skew and drift in the input predict requests, for AutoML tabular models.
+```
+Learn to use the `Vertex AI Model Monitoring` service to detect feature skew and drift in the input predict requests, for AutoML tabular models.
 
 The steps performed include:
 
@@ -66,10 +87,13 @@ The steps performed include:
 - Generate synthetic prediction requests for drift.
 - Wait for email alert notification.
 
+```
 
-[Vertex AI Model Monitoring for custom tabular models](community/ml_ops/stage7/get_started_with_model_monitoring_custom.ipynb)
 
-In this notebook, you learn to use the `Vertex AI Model Monitoring` service to detect feature skew and drift in the input predict requests, for custom tabular models.
+[Vertex AI Model Monitoring for custom tabular models](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage7/get_started_with_model_monitoring_custom.ipynb)
+
+```
+Learn to use the `Vertex AI Model Monitoring` service to detect feature skew and drift in the input predict requests, for custom tabular models.
 
 The steps performed include:
 
@@ -82,11 +106,13 @@ The steps performed include:
 - Generate synthetic prediction requests for drift.
 - Wait for email alert notification.
 
+```
 
 
-[Vertex AI Model Monitoring for setup for tabular models](community/ml_ops/stage7/get_started_with_model_monitoring_setup.ipynb)
+[Vertex AI Model Monitoring for setup for tabular models](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/ml_ops/stage7/get_started_with_model_monitoring_setup.ipynb)
 
-In this notebook, you learn to setup the `Vertex AI Model Monitoring` service to detect feature skew and drift in the input predict requests.
+```
+Learn to setup the `Vertex AI Model Monitoring` service to detect feature skew and drift in the input predict requests.
 
 The steps performed include:
 
@@ -100,3 +126,5 @@ The steps performed include:
 - List, pause, resume and delete monitoring jobs.
 - Restart monitoring job with predefined `input schema`.
 - View logged monitored data.
+
+```

notebooks/community/model_registry/vertex_ai_model_registry_bqml_custom_model_versioning.ipynb

@@ -29,7 +29,7 @@
         "id": "JAPoU8Sm5E6e"
       },
       "source": [
-        "# Model Versioning with Vertex AI Model Registry\n",
+        "# Model Management with Vertex AI Model Registry\n",
         "\n",
         "\n",
         "<table align=\"left\">\n",
@@ -1292,10 +1292,10 @@
         "      pandas \\\n",
         "      python \\\n",
         "      pyspark \\\n",
-        "      findspark\n",
+        "      findspark \n",
         "\n",
         "# Use conda to install spark-nlp\n",
-        "RUN ${CONDA_HOME}/bin/conda install -n base -c johnsnowlabs spark-nlp\n",
+        "RUN ${CONDA_HOME}/bin/conda install -n base -c johnsnowlabs 'spark-nlp=4.0.2'\n",
         "\n",
         "# Add lemma dictionary\n",
         "# ENV CONFIG_DIR='/home/app/build'\n",
@@ -1425,7 +1425,7 @@
         "import sparknlp\n",
         "from sparknlp.base import *\n",
         "from sparknlp.annotator import *\n",
-        "from pyspark.ml.feature import CountVectorizer\n",
+        "from pyspark.ml.feature import CountVectorizer, SQLTransformer\n",
         "from pyspark.ml import Pipeline\n",
         "\n",
         "# Variables ------------------------------------------------------------------------------------------------------------\n",
@@ -1473,7 +1473,7 @@
         "    Returns:\n",
         "        preliminary_steps: The preliminary steps for the preprocessing.\n",
         "    '''\n",
-        "\n",
+        "    \n",
         "    document_assembler = DocumentAssembler().setInputCol(\"text\").setOutputCol(\"document\").setCleanupMode('shrink_full')\n",
         "    sentence_detector = SentenceDetector().setInputCols(\"document\").setOutputCol(\"sentence\")\n",
         "    tokenizer = Tokenizer().setInputCols(\"sentence\").setOutputCol(\"token\")\n",
@@ -1512,6 +1512,16 @@
         "    feature_extraction_steps = [count_vectorizer]\n",
         "    return feature_extraction_steps\n",
         "\n",
+        "def build_postprocessing_steps():\n",
+        "    '''\n",
+        "    This function builds the postprocessing steps.\n",
+        "    Returns:\n",
+        "        target_conversion_step: The target conversion step.\n",
+        "    '''\n",
+        "\n",
+        "    sql_transformer = SQLTransformer(statement=\"SELECT CASE WHEN (category != 'business') THEN 'other' ELSE category END AS category, text, lemma_features, features  FROM __THIS__\")\n",
+        "    build_postprocessing_steps = [sql_transformer]\n",
+        "    return build_postprocessing_steps\n",
         "\n",
         "def read_data(spark_session, data_schema, input_dir):\n",
         "    '''\n",
@@ -1599,7 +1609,8 @@
         "    preliminary_steps = build_preliminary_steps()\n",
         "    common_preprocess_steps = build_common_preprocess_steps(lemma_uri)\n",
         "    feature_extraction_steps = build_feature_extraction_steps()\n",
-        "    pipeline = Pipeline(stages=preliminary_steps + common_preprocess_steps + feature_extraction_steps)\n",
+        "    postprocessing_steps = build_postprocessing_steps()\n",
+        "    pipeline = Pipeline(stages=preliminary_steps + common_preprocess_steps + feature_extraction_steps + postprocessing_steps)\n",
         "\n",
         "    # Read data\n",
         "    logger.info('Reading data')\n",
@@ -1697,6 +1708,7 @@
         "  --batch=$PREPROCESS_BATCH_ID \\\n",
         "  --container-image=$DATAPROC_RUNTIME_CONTAINER_IMAGE \\\n",
         "  --region=$REGION \\\n",
+        "  --version='1.0.21' \\\n",
         "  --subnet='default' \\\n",
         "  --properties spark.executor.instances=2,spark.driver.cores=4,spark.executor.cores=4,spark.app.name=spark_preprocessing_job \\\n",
         "  -- --input_path=$PREPARED_FILE_PATH --lemmas_path=$LEMMA_DICTIONARY_PATH --gcs_output_path=$PROCESS_DATA_PATH --bq_output_table_uri=$BQ_OUTPUT_TABLE_URI --bucket=$BUCKET_NAME --project=$PROJECT_ID"
@@ -1954,7 +1966,7 @@
         "        \"accuracy\": round(accuracy_score(y_test, y_pred, sample_weight=get_weights(y_test)), 5),\n",
         "        \"f1_score\": round(f1_score(y_test, y_pred, sample_weight=get_weights(y_test), average=\"weighted\"), 5),\n",
         "        \"log_loss\": round(log_loss(y_test, y_pred_proba, sample_weight=get_weights(y_test)), 5),\n",
-        "        \"roc_auc\": round(roc_auc_score(y_test, y_pred_proba, multi_class='ovr'), 5)\n",
+        "        \"roc_auc\": round(roc_auc_score(y_test, y_pred_proba[:,1], sample_weight=get_weights(y_test), average=\"weighted\"), 5)\n",
         "    }\n",
         "    return metrics\n",
         "\n",
@@ -2709,7 +2721,11 @@
         "\n",
         "versions = registry.list_versions()\n",
         "for version in versions:\n",
-        "    registry.delete_version(version=version.version_id)\n",
+        "    if \"default\" not in version.version_aliases:\n",
+        "        registry.delete_version(version=version.version_id)\n",
+        "    else:\n",
+        "        model = registry.get_model(version=\"default\")\n",
+        "        model.delete()\n",
         "\n",
         "naive_bayes_train_job.delete()\n",
         "\n",

notebooks/official.csv

@@ -0,0 +1,70 @@
+tag,notebook,doc
+"AutoML, Text data",official/automl/automl-text-classification.ipynb,vertex-ai/docs/text-data/classification/train-model
+"AutoML, Text data",official/automl/sdk_automl_text_entity_extraction_online.ipynb,
+"AutoML, Text data",official/automl/sdk_automl_text_sentiment_analysis_online.ipynb,
+"AutoML, Tabular data",official/automl/sdk_automl_tabular_forecasting_batch.ipynb,vertex-ai/docs/tabular-data/forecasting/tutorials-samples
+"AutoML, Tabular Data",official/automl/automl_tabular_on_vertex_pipelines.ipynb,vertex-ai/docs/tabular-data/tabular-workflows/e2e-automl
+"AutoML, Tabular Data",official/automl/sdk_automl_tabular_regression_batch_bq.ipynb,
+"AutoML, Tabular Data",official/automl/sdk_automl_tabular_regression_batch_bq.ipynb,
+"AutoML, Forecasting",official/automl/automl_forecasting_bqml_arima_plus_comparison.ipynb,vertex-ai/docs/tabular-data/forecasting-arima/overview
+"AutoML, Forecasting",official/automl/sdk_automl_tabular_forecasting_batch.ipynb,
+"AutoML, Image data",official/automl/sdk_automl_text_sentiment_analysis_online.ipynb,
+"AutoML, Video data",official/automl/sdk_automl_text_sentiment_analysis_online.ipynb,
+"AutoML, Video data",official/automl/sdk_automl_video_classification_batch.ipynb,
+"AutoML, Video data",official/automl/sdk_automl_video_object_tracking_batch.ipynb,
+"AutoML, Video data",official/sdk/SDK_AutoML_Video_Classification.ipynb,
+"BigQuery, Vertex AI Workbench",official/workbench/exploratory_data_analysis/explore_data_in_bigquery_with_workbench.ipynb,
+"BigQuery ML, Vertex AI Model Registry, Batch prediction",official/model_registry/bqml_vertexai_model_registry.ipynb,
+"BigQuery ML, Vertex AI Model Registry, Online prediction",official/bigquery_ml/bqml-online-prediction.ipynb,
+"BigQuery ML",official/structured_data/rapid_prototyping_bqml_automl.ipynb,
+Custom Training,official/custom/sdk-custom-image-classification-batch.ipynb,
+Custom Training,official/custom/sdk-custom-image-classification-online.ipynb,
+Custom Training,official/custom/SDK_Custom_Container_Prediction.ipynb,
+"Custom Training, BiqQuery dataset",official/custom/custom-tabular-bq-managed-dataset.ipynb,
+"Custom Training, TensorBoard",official/custom/custom-tabular-bq-managed-dataset.ipynb,
+"Custom Training, TensorBoard",official/tensorboard/tensorboard_custom_training_with_custom_container.ipynb,
+"Custom Training, TensorBoard",official/tensorboard/tensorboard_custom_training_with_prebuilt_container.ipynb
+"Custom Training, Managed dataset",official/sdk/SDK_Custom_Training_Python_Package_Managed_Text_Dataset_Tensorflow_Serving_Container.ipynb,
+"Custom Training, Distributed",official/training/multi_node_ddp_gloo_vertex_training_with_custom_container.ipynb,
+"Custom Training, Distributed",official/training/multi_node_ddp_nccl_vertex_training_with_custom_container.ipynb
+Vertex AI Experiments,official/experiments/comparing_pipeline_runs.ipynb,
+Vertex AI Experiments,official/experiments/build_model_experimentation_lineage_with_prebuild_code.ipynb,
+Vertex AI Experiments,official/experiments/comparing_local_trained_models.ipynb,
+"Vertex Explainable AI, Tabular data",official/explainable_ai/sdk_automl_tabular_binary_classification_batch_explain.ipynb,vertex-ai/docs/explainable-ai/overview
+"Vertex Explainable AI, Tabular data",official/explainable_ai/sdk_automl_tabular_classification_online_explain.ipynb,vertex-ai/docs/explainable-ai/overview
+"Vertex Explainable AI, Image data",official/explainable_ai/sdk_custom_image_classification_batch_explain.ipynb,vertex-ai/docs/explainable-ai/overview
+"Vertex Explainable AI, Tabular data",official/explainable_ai/sdk_custom_tabular_regression_batch_explain.ipynb,vertex-ai/docs/explainable-ai/overview
+"Vertex Explainable AI, Tabular data",official/explainable_ai/sdk_custom_tabular_regression_online_explain.ipynb,vertex-ai/docs/explainable-ai/overview
+Vertex ML Metadata,official/ml_metadata/sdk-metric-parameter-tracking-for-custom-jobs.ipynb,
+"Vertex Explainable AI, Image data",official/explainable_ai/sdk_custom_image_classification_online_explain.ipynb,vertex-ai/docs/explainable-ai/overview
+Vertex AI Feature Store,official/feature_store/sdk-feature-store.ipynb,
+Vertex AI Feature Store,official/feature_store/sdk-feature-store-pandas.ipynb,
+Vertex AI Matching Engine,official/matching_engine/sdk_matching_engine_for_indexing.ipynb,
+Vertex ML Metadata,official/ml_metadata/sdk-metric-parameter-tracking-for-custom-jobs.ipynb,
+Vertex ML Metadata,official/ml_metadata/sdk-metric-parameter-tracking-for-locally-trained-models.ipynb,
+"Vertex ML Metadata, Vertex AI Pipelines",official/ml_metadata/sdk-metric-parameter-tracking-for-locally-trained-models.ipynb,
+"Vertex AI Model Evaluation, AutoML",official/model_evaluation/automl_tabular_classification_model_evaluation.ipynb,
+"Vertex AI Model Evaluation, AutoML",official/model_evaluation/automl_tabular_regression_model_evaluation.ipynb,
+"Vertex AI Model Evaluation, AutoML",official/model_evaluation/automl_text_classification_model_evaluation.ipynb,
+"Vertex AI Model Evaluation, AutoML",official/model_evaluation/automl_video_classification_model_evaluation.ipynb,
+"Vertex AI Model Evaluation, Custom Training",official/model_evaluation/custom_tabular_regression_model_evaluation.ipynb,
+Model Monitoring,official/model_monitoring/model_monitoring.ipynb,
+Vertex AI Pipelines,official/pipelines/pipelines_intro_kfp.ipynb,
+Vertex AI Pipelines,official/pipelines/control_flow_kfp.ipynb,
+Vertex AI Pipelines,official/pipelines/metrics_viz_run_compare_kfp.ipynb,
+Vertex AI Pipelines,official/pipelines/lightweight_functions_component_io_kfp.ipynb,
+"Vertex AI Pipelines Image data",official/pipelines/google_cloud_pipeline_components_automl_images.ipynb,
+"Vertex AI Pipelines, Tabular data",official/pipelines/automl_tabular_classification_beans.ipynb,
+"Vertex AI Pipelines, Tabular data",official/pipelines/google_cloud_pipeline_components_automl_tabular.ipynb,
+"Vertex AI Pipelines, Tabular data",official/pipelines/google_cloud_pipeline_components_dataproc_tabular.ipynb,
+"Vertex AI Pipelines, Text data",official/pipelines/google_cloud_pipeline_components_automl_text.ipynb,
+"Vertex AI Pipelines, Text data",official/pipelines/google_cloud_pipeline_components_bqml_text.ipynb,
+Vertex AI Pipelines,official/pipelines/custom_model_training_and_batch_prediction.ipynb,
+Vertex AI Pipelines,official/pipelines/google_cloud_pipeline_components_model_train_upload_deploy.ipynb,
+Vertex AI Pipelines,official/pipelines/google_cloud_pipeline_components_model_upload_predict_evaluate.ipynb,
+"Vertex AI Training, Reduction Server, PyTorch",official/reduction_server/pytorch_distributed_training_reduction_server.ipynb,
+"Tabular Workflows, Vertex AI TabNet",official/tabnet/tabnet_vertex_tutorial.ipynb,
+"Tabular Workflows, Vertex AI TabNet, Vertex Explainablee AI",official/tabnet/ai-explanations-tabnet-algorithm.ipynb,
+"Tabular Workflows, Vertex AI TabNet, Vertex AI Pipelines",official/tabular_workflows/tabnet_on_vertex_pipelines.ipynb,
+"Tabular Workflows, Vertex AI Wide and Deep",official/tabular_workflows/wide_and_deep_on_vertex_pipelines.ipynb,
+Vertex AI Vizier,official/vizier/gapic-vizier-multi-objective-optimization.ipynb,vertex-ai/docs/vizier/using-vizier

notebooks/official/CODEOWNERS

@@ -42,3 +42,4 @@
 /sdk/SDK_FBProphet_Forecasting_Online.ipynb @brianchunkang
 /automl/sdk_automl_forecasting_hierarchical_batch.ipynb @ivanmkc
 /prediction/custom_batch_prediction_feature_filter.ipynb @soheilazangeneh
+/feature_store/feature_store_streaming_ingestion_sdk.ipynb @soheilazangeneh

notebooks/official/automl/README.md

@@ -1,6 +1,6 @@
-
 [AutoML Tabular Training and Prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/automl-tabular-classification.ipynb)
 
+```
 Learn how to train and make predictions on an AutoML model based on a tabular dataset.
 
 The steps performed include the following:
@@ -11,8 +11,12 @@ The steps performed include the following:
 - Make a prediction by sending data.
 - Undeploy the `Model` resource.
 
+```
+
+
 [Create, train, and deploy an AutoML text classification model](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/automl-text-classification.ipynb)
 
+```
 Learn how to use `AutoML` to train a text classification model.
 
 The steps performed include:
@@ -25,8 +29,12 @@ The steps performed include:
 * Make an online prediction
 * Make a batch prediction
 
+```
+
+
 [AutoML training video classification model for batch prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_video_classification_batch.ipynb)
 
+```
 Learn how to create an AutoML video classification model from a Python script, and then do a batch prediction using the Vertex AI SDK.
 
 The steps performed include:
@@ -36,13 +44,12 @@ The steps performed include:
 - View the model evaluation.
 - Make a batch prediction.
 
+```
 
-* Prediction Service: Does an on-demand prediction for the entire set of instances (i.e., one or more data items) and returns the results in real-time.
-
-* Batch Prediction Service: Does a queued (batch) prediction for the entire set of instances in the background and stores the results in a Cloud Storage bucket when ready.
 
 [AutoML training text entity extraction model for online prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_text_entity_extraction_online.ipynb)
 
+```
 Learn how to create an AutoML text entity extraction model and deploy for online prediction from a Python script using the Vertex SDK.
 
 The steps performed include:
@@ -54,8 +61,12 @@ The steps performed include:
 - Make a prediction.
 - Undeploy the `Model`.
 
+```
+
+
 [AutoML tabular forecasting model for batch prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_tabular_forecasting_batch.ipynb)
 
+```
 Learn how to create an `AutoML` tabular forecasting model from a Python script, and then do a batch prediction using the Vertex AI SDK.
 
 The steps performed include:
@@ -65,40 +76,26 @@ The steps performed include:
 - Obtain the evaluation metrics for the `Model` resource.
 - Make a batch prediction.
 
-[AutoML training image object detection model for batch prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_image_object_detection_batch.ipynb)
+```
 
-Learn how to create an AutoML image object detection model from a Python script, and then do a batch prediction using the Vertex AI SDK.
-
-The steps performed include:
-
-- Create a Vertex `Dataset` resource.
-- Train the model.
-- View the model evaluation.
-- Make a batch prediction.
-
-
-* Prediction Service: Does an on-demand prediction for the entire set of instances (i.e., one or more data items) and returns the results in real-time.
-
-* Batch Prediction Service: Does a queued (batch) prediction for the entire set of instances in the background and stores the results in a Cloud Storage bucket when ready.
 
 [AutoML training video action recognition model for batch prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_video_action_recognition_batch.ipynb)
 
+```
 Learn how to create an AutoML video action recognition model from a Python script, and then do a batch prediction using the Vertex AI SDK.
 
 The steps performed include:
 
-- Create a Vertex `Dataset` resource.
+- Create a `Vertex AI Dataset` resource.
 - Train the model.
 - View the model evaluation.
 - Make a batch prediction.
+```
 
 
-* Prediction Service: Does an on-demand prediction for the entire set of instances (i.e., one or more data items) and returns the results in real-time.
-
-* Batch Prediction Service: Does a queued (batch) prediction for the entire set of instances in the background and stores the results in a Cloud Storage bucket when ready.
-
 [AutoML Tabular Pipeline](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/automl_tabular_on_vertex_pipelines.ipynb)
 
+```
 Learn how to create two regression models using [Vertex Pipelines](https://cloud.
 
 The steps performed are:
@@ -106,36 +103,48 @@ The steps performed are:
 - Create a training pipeline that reduces the search space from the default to save time.
 - Create a training pipeline that reuses the architecture search results from the previous pipeline to save time.
 
-[AutoML training text sentiment analysis model for online prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_text_sentiment_analysis_online.ipynb)
+```
 
-Learn how to create an AutoML text sentiment analysis model and deploy for online prediction from a Python script using the Vertex SDK.
+
+[Training an AutoML text sentiment analysis model for online predictions](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_text_sentiment_analysis_online.ipynb)
+
+```
+Learn how to create an AutoML text sentiment analysis model and deploy it for online predictions from a Python script using the Vertex AI SDK.
 
 The steps performed include:
 
-- Create a Vertex `Dataset` resource.
-- Create a training job for the model.
-- View the model evaluation.
-- Deploy the `Model` resource to a serving `Endpoint` resource.
-- Make a prediction.
-- Undeploy the `Model`.
+- Create a `Vertex AI Dataset` resource.
+- Create a training job for the AutoML model on the dataset.
+- View the model evaluation metrics.
+- Deploy the `Vertex AI Model` resource to a serving `Vertex AI Endpoint`.
+- Make a prediction request to the deployed model.
+- Undeploy the model from endpoint.
+- Perform clean up process.
+
+```
+
 
 [Compare Vertex AI Forecasting and BigQuery ML ARIMA_PLUS](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/automl_forecasting_bqml_arima_plus_comparison.ipynb)
 
-Learn how to create an BQML ARIMA_PLUS model using a training [Vertex AI Pipeline](https://cloud.
+```
+Learn how to create an BigQuery ML ARIMA_PLUS model using a training [Vertex AI Pipeline](https://cloud.
 
 The steps performed are:
 
-- Train the BQML ARIMA_PLUS model.
-- View BQML model evaluation.
-- Make a batch prediction with the BQML model.
+- Train the BigQuery ML ARIMA_PLUS model.
+- View BigQuery ML model evaluation.
+- Make a batch prediction with the BigQuery ML model.
 - Create a Vertex AI `Dataset` resource.
 - Train the Vertex AI Forecasting model.
 - View the Model evaluation.
 - Make a batch prediction with the Model.
 
+```
 
-[AutoML training tabular regression model for online prediction using BigQuery](https://github.com/GoogleCloudPlatform/vertex-ai-samples/tree/master/notebooks/official/automl/sdk_automl_tabular_regression_online_bq.ipynb)
 
+[AutoML training tabular regression model for online prediction using BigQuery](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_tabular_regression_online_bq.ipynb)
+
+```
 Learn how to create an AutoML tabular regression model and deploy for online prediction from a Python script using the Vertex AI SDK.
 
 The steps performed include:
@@ -147,9 +156,13 @@ The steps performed include:
 - Make a prediction.
 - Undeploy the `Model`.
 
+```
+
+
 [AutoML training video object tracking model for batch prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_video_object_tracking_batch.ipynb)
 
-Learn how to create an AutoML video object tracking model from a Python script, and then do a batch prediction using the Vertex SDK.
+```
+Learn how to create an AutoML video object tracking model from a Python script, and then do a batch prediction using the Vertex AI SDK.
 
 The steps performed include:
 
@@ -157,14 +170,12 @@ The steps performed include:
 - Train the model.
 - View the model evaluation.
 - Make a batch prediction.
+```
 
 
-* Prediction Service: Does an on-demand prediction for the entire set of instances (i.e., one or more data items) and returns the results in real-time.
-
-* Batch Prediction Service: Does a queued (batch) prediction for the entire set of instances in the background and stores the results in a Cloud Storage bucket when ready.
-
 [AutoML training tabular regression model for batch prediction using BigQuery](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_tabular_regression_batch_bq.ipynb)
 
+```
 Learn how to create an AutoML tabular regression model and deploy it for batch prediction using the Vertex AI SDK for Python.
 
 The steps performed include:
@@ -175,3 +186,5 @@ The steps performed include:
 - Deploy the `Model` resource to a serving `Endpoint` resource.
 - Make a prediction.
 - Undeploy the `Model`.
+
+```

notebooks/official/automl/automl-tabular-classification.ipynb

@@ -29,7 +29,7 @@
         "id": "JAPoU8Sm5E6e"
       },
       "source": [
-        "# Vertex AI SDK for Python: AutoML Tabular Training and Prediction\n",
+        "# Vertex AI SDK for Python: AutoML Tabular training and prediction\n",
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
@@ -63,7 +63,7 @@
         "\n",
         "This tutorial demonstrates how to use the Vertex AI Python client library to train and deploy a tabular classification model for online prediction.\n",
         "\n",
-        "**Note**: you may incur charges for training, prediction, storage, or usage of other GCP products in connection with testing this SDK."
+        "**Note**: you may incur charges for training, prediction, storage, or usage of other Google Cloud products in connection with testing this SDK."
       ]
     },
     {
@@ -76,6 +76,11 @@
         "\n",
         "In this tutorial, you learn how to train and make predictions on an AutoML model based on a tabular dataset. Alternatively, you can train and make predictions on models by using the `gcloud` command-line tool or by using the online Cloud Console.\n",
         "\n",
+        "This tutorial uses the following Google Cloud ML services and resources:\n",
+        "\n",
+        "- Vertex AI\n",
+        "- AutoML Tabular\n",
+        "\n",
         "The steps performed include the following:\n",
         "\n",
         "- Create a Vertex AI model training job.\n",
@@ -122,7 +127,9 @@
         "id": "install_aip"
       },
       "source": [
-        "## Installation"
+        "## Installation\n",
+        "\n",
+        "Install the packages required for executing this notebook."
       ]
     },
     {
@@ -135,55 +142,20 @@
       "source": [
         "import os\n",
         "\n",
-        "# The Google Cloud Notebook product has specific requirements\n",
-        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
+        "# The Vertex AI Workbench Notebook product has specific requirements\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
+        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
+        "    \"/opt/deeplearning/metadata/env_version\"\n",
+        ")\n",
         "\n",
-        "# Google Cloud Notebook requires dependencies to be installed with '--user'\n",
+        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
         "USER_FLAG = \"\"\n",
-        "if IS_GOOGLE_CLOUD_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\""
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "b03b7f4487ff"
-      },
-      "source": [
-        "Install the latest version of the Vertex AI client library.\n",
+        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\"\n",
         "\n",
-        "Run the following command in your virtual environment to install the Vertex SDK for Python:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "d489d38261dd"
-      },
-      "outputs": [],
-      "source": [
-        "! pip install {USER_FLAG} --upgrade google-cloud-aiplatform"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "install_storage"
-      },
-      "source": [
-        "Install the Cloud Storage library:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "qssss-KSlugo"
-      },
-      "outputs": [],
-      "source": [
-        "! pip install {USER_FLAG} --upgrade google-cloud-storage"
+        "# Install the packagesimport os\n",
+        "! pip3 install {USER_FLAG} -q --upgrade google-cloud-aiplatform \\\n",
+        "                                        google-cloud-storage"
       ]
     },
     {

notebooks/official/automl/automl_tabular_on_vertex_pipelines.ipynb

@@ -29,7 +29,7 @@
         "id": "mThXALJl9Yue"
       },
       "source": [
-        "# Tabular Workflow: AutoML Tabular Pipeline\n",
+        "# AutoML Tabular Workflow pipelines\n",
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
@@ -72,7 +72,12 @@
       "source": [
         "### Objective\n",
         "\n",
-        "In this tutorial, you learn how to create two regression models using [Vertex Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction) downloaded from [Google Cloud Pipeline Components](https://cloud.google.com/vertex-ai/docs/pipelines/components-introduction) (GCPC). These pipelines will be Vertex AI Tabular Workflow pipelines which are maintained by Google. These pipelines will showcase different ways to customize the Vertex Tabular training process.\n",
+        "In this tutorial, you learn how to create two regression models using [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction) downloaded from [Google Cloud Pipeline Components](https://cloud.google.com/vertex-ai/docs/pipelines/components-introduction) (GCPC). These pipelines will be Vertex AI Tabular Workflow pipelines which are maintained by Google. These pipelines will showcase different ways to customize the Vertex Tabular training process.\n",
+        "\n",
+        "This tutorial uses the following Google Cloud ML services:\n",
+        "\n",
+        "- `AutoML Training`\n",
+        "- `Vertex AI Datasets`\n",
         "\n",
         "The steps performed are:\n",
         "\n",

notebooks/official/automl/sdk_automl_forecasting_hierarchical_batch.ipynb

@@ -64,6 +64,32 @@
         "This tutorial demonstrates how to use the Vertex AI SDK for Python to create hierarchical forecasting models using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users)and do batch prediction. Specifically, you predict a fictional store's sales based on historical sales data."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "objective:automl,training,online_prediction"
+      },
+      "source": [
+        "### Objective\n",
+        "\n",
+        "In this tutorial, you create an AutoML hierarchical forecasting model and deploy it for batch prediction using the Vertex AI SDK for Python. You can alternatively create and deploy models using the `gcloud` command-line tool or batch using the Cloud Console.\n",
+        "The rationale for a hierarchical forecasting model is to minimize the error for a given group of sales data. In this tutorial, you will be minimizing the error for sale predictions at the \"product\" level.\n",
+        "\n",
+        "This tutorial uses the following Google Cloud ML services:\n",
+        "\n",
+        "- `AutoML Training`\n",
+        "- `Vertex AI Datasets`\n",
+        "\n",
+        "The steps performed include:\n",
+        "\n",
+        "- Create a Vertex AI `TimeSeriesDataset` resource.\n",
+        "- Train the model.\n",
+        "- View the model evaluation.\n",
+        "- Deploy the `Model` resource to a serving `Endpoint` resource.\n",
+        "- Make a prediction.\n",
+        "- Undeploy the `Model`."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -90,27 +116,6 @@
         "Link to data: gs://cloud-samples-data/vertex-ai/structured_data/forecasting/synthetic_sales_data.csv"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "objective:automl,training,online_prediction"
-      },
-      "source": [
-        "### Objective\n",
-        "\n",
-        "In this tutorial, you create an AutoML hierarchical forecasting model and deploy it for batch prediction using the Vertex AI SDK for Python. You can alternatively create and deploy models using the `gcloud` command-line tool or batch using the Cloud Console.\n",
-        "The rationale for a hierarchical forecasting model is to minimize the error for a given group of sales data. In this tutorial, you will be minimizing the error for sale predictions at the \"product\" level.\n",
-        "\n",
-        "The steps performed include:\n",
-        "\n",
-        "- Create a Vertex AI `TimeSeriesDataset` resource.\n",
-        "- Train the model.\n",
-        "- View the model evaluation.\n",
-        "- Deploy the `Model` resource to a serving `Endpoint` resource.\n",
-        "- Make a prediction.\n",
-        "- Undeploy the `Model`."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {

notebooks/official/automl/sdk_automl_image_object_detection_batch.ipynb

@@ -66,17 +66,6 @@
         "This tutorial demonstrates how to use the Vertex AI SDK to create image object detection models and do batch prediction using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) model."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "dataset:salads,iod"
-      },
-      "source": [
-        "### Dataset\n",
-        "\n",
-        "The dataset used for this tutorial is the Salads category of the [OpenImages dataset](https://www.tensorflow.org/datasets/catalog/open_images_v4) from [TensorFlow Datasets](https://www.tensorflow.org/datasets/catalog/overview). This dataset does not require any feature engineering. The version of the dataset you will use in this tutorial is stored in a public Cloud Storage bucket. The trained model predicts the bounding box locations and the corresponding type of salad items in an image from a class of five items: salad, seafood, tomato, baked goods, or cheese."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -87,6 +76,11 @@
         "\n",
         "In this tutorial, you create an AutoML image object detection model from a Python script, and then do a batch prediction using the Vertex AI SDK. You can alternatively create and deploy models using the `gcloud` command-line tool or online using the Cloud Console.\n",
         "\n",
+        "This tutorial uses the following Google Cloud ML services:\n",
+        "\n",
+        "- `AutoML Training`\n",
+        "- `Vertex AI Datasets`\n",
+        "\n",
         "The steps performed include:\n",
         "\n",
         "- Create a Vertex `Dataset` resource.\n",
@@ -101,6 +95,17 @@
         "* Batch Prediction Service: Does a queued (batch) prediction for the entire set of instances in the background and stores the results in a Cloud Storage bucket when ready."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dataset:salads,iod"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The dataset used for this tutorial is the Salads category of the [OpenImages dataset](https://www.tensorflow.org/datasets/catalog/open_images_v4) from [TensorFlow Datasets](https://www.tensorflow.org/datasets/catalog/overview). This dataset does not require any feature engineering. The version of the dataset you will use in this tutorial is stored in a public Cloud Storage bucket. The trained model predicts the bounding box locations and the corresponding type of salad items in an image from a class of five items: salad, seafood, tomato, baked goods, or cheese."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {

notebooks/official/automl/sdk_automl_tabular_regression_batch_bq.ipynb

@@ -74,7 +74,7 @@
         "\n",
         "In this tutorial, you learn how to create an AutoML tabular regression model and deploy it for batch prediction using the Vertex AI SDK for Python. You can alternatively create and deploy models using the `gcloud` command-line tool or batch using the Cloud Console.\n",
         "\n",
-        "This tutorial uses the following Google Cloud ML services and resources:\n",
+        "This tutorial uses the following Google Cloud ML services:\n",
         "\n",
         "- Vertex AI Datasets (Tabular)\n",
         "- Vertex AI Training (AutoML Tabular Training)\n",

notebooks/official/automl/sdk_automl_text_entity_extraction_online.ipynb

@@ -73,7 +73,12 @@
       "source": [
         "### Objective\n",
         "\n",
-        "In this tutorial, you learn how to create an AutoML text entity extraction model and deploy for online prediction from a Python script using the Vertex SDK. You can alternatively create and deploy models using the `gcloud` command-line tool or online using the Cloud Console.\n",
+        "In this tutorial, you learn how to create an AutoML text entity extraction model and deploy for online prediction from a Python script using the Vertex AI SDK. You can alternatively create and deploy models using the `gcloud` command-line tool or online using the Cloud Console.\n",
+        "\n",
+        "This tutorial uses the following Google Cloud ML services:\n",
+        "\n",
+        "- `AutoML Training`\n",
+        "- `Vertex AI Datasets`\n",
         "\n",
         "The steps performed include:\n",
         "\n",

notebooks/official/bigquery_ml/README.md

@@ -1,6 +1,6 @@
-
 [Online prediction with BigQuery ML](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/bigquery_ml/bqml-online-prediction.ipynb)
 
+```
 Learn how to train and deploy a churn prediction model for real-time inference, with the data in BigQuery and model trained using BigQuery ML, registered to Vertex AI Model Registry, and deployed to an endpoint on Vertex AI for online predictions.
 
 The steps performed include:
@@ -12,3 +12,5 @@ The steps performed include:
 - Deploying the model to an endpoint on Vertex AI
 - Making sample online predictions to the model endpoint
 
+```
+

notebooks/official/custom/README.md

@@ -1,6 +1,25 @@
+[Training and deploying a sales forecasting model using FBProphet and Vertex AI](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/custom/SDK_FBProphet_Forecasting_Online.ipynb)
+
+```
+The objective of this notebook is to create, deploy and serve a custom forecasting model on Vertex AI.
+
+The steps performed include:
+- Train a model locally that forecasts sales for the given number of days.
+- Train another model that uses both sales and weather data for sales prediction.
+- Save both the models.
+- Build a FastAPI server to handle the predictions for the chosen model.
+- Build a custom container image of the serving application with the model artifacts.
+- Upload the model to Vertex AI Model Registry.
+- Deploy the model to a Vertex AI Endpoint.
+- Send online prediction requests to the deployed model.
+- Clean up the resources created in this session.
+
+```
+
 
 [Custom training and batch prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/custom/sdk-custom-image-classification-batch.ipynb)
 
+```
 Learn to use `Vertex AI Training` to create a custom trained model and use `Vertex AI Batch Prediction` to do a batch prediction on the trained model.
 
 The steps performed include:
@@ -9,8 +28,12 @@ The steps performed include:
 - Upload the trained model artifacts as a `Model` resource.
 - Make a batch prediction.
 
+```
+
+
 [Profile model training performance using Profiler](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/custom/custom_training_tensorboard_profiler.ipynb)
 
+```
 Learn how to enable Vertex AI TensorBoard Profiler for custom training jobs.
 
 The steps performed include:
@@ -20,9 +43,12 @@ The steps performed include:
 - Create and run a custom training job
 - View the TensorBoard Profiler dashboard
 
+```
+
 
 [Training a TensorFlow model on BigQuery data](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/custom/custom-tabular-bq-managed-dataset.ipynb)
 
+```
 Learn how to create a custom-trained model from a Python script in a Docker container using the Vertex AI SDK for Python, and then get a prediction from the deployed model by sending data.
 
 The steps performed include:
@@ -33,8 +59,12 @@ The steps performed include:
 - Make a prediction.
 - Undeploy the `Model` resource.
 
+```
+
+
 [Custom training and online prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/custom/sdk-custom-image-classification-online.ipynb)
 
+```
 Learn to use `Vertex AI Training` to create a custom-trained model from a Python script in a Docker container, and learn to use `Vertex AI Prediction` to do a prediction on the deployed model by sending data.
 
 The steps performed include:
@@ -46,8 +76,12 @@ The steps performed include:
 - Make a prediction.
 - Undeploy the `Model` resource.
 
+```
+
+
 [Deploying Iris-detection model using FastAPI and Vertex AI custom container serving](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/custom/SDK_Custom_Container_Prediction.ipynb)
 
+```
 Learn how to create, deploy and serve a custom classification model on Vertex AI.
 
 The steps performed include:
@@ -57,3 +91,6 @@ The steps performed include:
 - Build a FastAPI server to handle predictions and health checks.
 - Build a custom container with model artifacts.
 - Upload and deploy custom container to Vertex AI Endpoints.
+
+```
+

notebooks/official/experiments/README.md

@@ -1,12 +1,21 @@
-
 [Compare pipeline runs with Vertex AI Experiments](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/experiments/comparing_pipeline_runs.ipynb)
 
+```
 Learn how to use `Vertex AI Experiments` to log a pipeline job and compare different pipeline jobs.
 
+The steps performed include:
+
+* Formalize a training component
+* Build a training pipeline
+* Run several Pipeline jobs and log their results
+* Compare different Pipeline jobs
+
+```
 
 
 [Track parameters and metrics for locally trained models](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/experiments/comparing_local_trained_models.ipynb)
 
+```
 Learn how to use Vertex AI Experiments to compare and evaluate model experiments.
 
 The steps performed include:
@@ -15,9 +24,15 @@ The steps performed include:
 - log the loss and metrics on every epoch to TensorBoard
 - log the evaluation metrics
 
+```
+
 
 [Build Vertex AI Experiment lineage for custom training](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/experiments/build_model_experimentation_lineage_with_prebuild_code.ipynb)
 
+```
 Learn how to integrate preprocessing code in a Vertex AI experiments.
 
 
+
+```
+

notebooks/official/experiments/build_model_experimentation_lineage_with_prebuild_code.ipynb

@@ -72,7 +72,22 @@
       "source": [
         "### Objective\n",
         "\n",
-        "In this notebook, you learn how to integrate preprocessing code in a Vertex AI experiments. Also you build the experiment lineage lets you record, analyze, debug, and audit metadata and artifacts produced along your ML journey."
+        "In this notebook, you learn how to integrate preprocessing code in a Vertex AI experiments. Also you build the experiment lineage lets you record, analyze, debug, and audit metadata and artifacts produced along your ML journey.\n",
+        "\n",
+        "This tutorial uses the following Google Cloud ML services and resources:\n",
+        "\n",
+        "- Vertex ML Metadata\n",
+        "- Vertex AI Experiments\n",
+        "\n",
+        "The steps performed include:\n",
+        "\n",
+        "- Execute module for preprocessing data\n",
+        "  - Create a dataset artifact\n",
+        "  - Log parameters\n",
+        "-  Execute module for training the model\n",
+        "  - Log parameters\n",
+        "  - Create model artifact\n",
+        "  - Assign tracking lineage to dataset, model and parameters"
       ]
     },
     {

notebooks/official/experiments/comparing_pipeline_runs.ipynb

@@ -61,7 +61,7 @@
       "source": [
         "## Overview\n",
         "\n",
-        "Depending on the model life cycle of your data science team, you would like to experiment and track training Pipeline runs and its associated parameters. Then, you would to compare runs of these Pipelines to each others in order to figure out which is the best configuration generates the model you will register in the Vertex AI Model Registry."
+        "Depending on the model life cycle of your data science team, you would like to experiment and track training pipeline runs and its associated parameters. Then, you would to compare runs of these Pipelines to each others in order to figure out which is the best configuration generates the model you will register in the Vertex AI Model Registry."
       ]
     },
     {
@@ -74,7 +74,12 @@
         "\n",
         "In this notebook, you learn how to use `Vertex AI Experiments` to log a pipeline job and compare different pipeline jobs.\n",
         "\n",
-        "The steps covered include:\n",
+        "This tutorial uses the following Google Cloud ML services and resources:\n",
+        "\n",
+        "- Vertex AI Pipelines\n",
+        "- Vertex AI Experiments\n",
+        "\n",
+        "The steps performed include:\n",
         "\n",
         "* Formalize a training component\n",
         "* Build a training pipeline\n",

notebooks/official/explainable_ai/README.md

@@ -1,6 +1,6 @@
-
 [Custom training tabular regression model for batch prediction with explainabilty](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/explainable_ai/sdk_custom_tabular_regression_batch_explain.ipynb)
 
+```
 Learn how to use `Vertex AI Training and Explainable AI` to create a custom image classification model with explanations, and then you learn to use `Vertex AI Batch Prediction` to make a batch prediction request with explanations.
 
 The steps performed include:
@@ -11,8 +11,12 @@ The steps performed include:
 - Upload the trained model artifacts and explanations as a `Model` resource.
 - Make a batch prediction with explanations.
 
+```
+
+
 [AutoML training tabular binary classification model for batch explanation](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/explainable_ai/sdk_automl_tabular_binary_classification_batch_explain.ipynb)
 
+```
 Learn to use `AutoML` to create a tabular binary classification model from a Python script, and then learn to use `Vertex AI Batch Prediction` to make predictions with explanations.
 
 The steps performed include:
@@ -22,13 +26,12 @@ The steps performed include:
 - View the model evaluation metrics for the trained model.
 - Make a batch prediction request with explainability.
 
+```
 
-* Prediction Service: Does an on-demand prediction for the entire set of instances (i.e., one or more data items) and returns the results in real-time.
-
-* Batch Prediction Service: Does a queued (batch) prediction for the entire set of instances in the background and stores the results in a Cloud Storage bucket when ready.
 
 [AutoML training tabular classification model for online explanation](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/explainable_ai/sdk_automl_tabular_classification_online_explain.ipynb)
 
+```
 Learn how to use `AutoML` to create a tabular binary classification model from a Python script, and then learn to use `Vertex AI Online Prediction` to make online predictions with explanations.
 
 The steps performed include:
@@ -41,8 +44,12 @@ The steps performed include:
 - Make an online prediction request with explainability.
 - Undeploy the `Model` resource.
 
+```
+
+
 [Custom training image classification model for online prediction with explainabilty](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/explainable_ai/sdk_custom_image_classification_online_explain.ipynb)
 
+```
 Learn how to use `Vertex AI Training and Explainable AI` to create a custom image classification model with explanations, and then you learn to use `Vertex AI Prediction` to make an online prediction request with explanations.
 
 The steps performed include:
@@ -56,8 +63,31 @@ The steps performed include:
 - Make a prediction with explanation.
 - Undeploy the `Model` resource.
 
+```
+
+
+[Custom training tabular regression model for online prediction with explainabilty](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/explainable_ai/sdk_custom_tabular_regression_online_explain.ipynb)
+
+```
+Learn how to use `Vertex AI Training and Explainable AI` to create a custom image classification model with explanations, and then you learn to use `Vertex AI Prediction` to make an online prediction request with explanations.
+
+The steps performed include:
+
+- Create a `Vertex AI` custom job for training a TensorFlow model.
+- View the model evaluation for the trained model.
+- Set explanation parameters for when the model is deployed.
+- Upload the trained model artifacts and explanations as a `Model` resource.
+- Create a serving `Endpoint` resource.
+- Deploy the `Model` resource to a serving `Endpoint` resource.
+- Make a prediction with explanation.
+- Undeploy the `Model` resource.
+
+```
+
+
 [Custom training tabular regression model for online prediction with explainabilty using get_metadata](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/explainable_ai/sdk_custom_tabular_regression_online_explain_get_metadata.ipynb)
 
+```
 Learn how to create a custom model from a Python script in a Google prebuilt Docker container using the Vertex SDK, and then do a prediction with explanations on the deployed model by sending data.
 
 The steps performed include:
@@ -72,8 +102,12 @@ The steps performed include:
 - Make a prediction with explanation.
 - Undeploy the `Model` resource.
 
+```
+
+
 [Custom training image classification model for batch prediction with explainabilty](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/explainable_ai/sdk_custom_image_classification_batch_explain.ipynb)
 
+```
 Learn to use `Vertex AI Training and Explainable AI` to create a custom image classification model with explanations, and then you learn to use `Vertex AI Batch Prediction` to make a batch prediction request with explanations.
 
 The steps performed include:
@@ -83,3 +117,6 @@ The steps performed include:
 - Set explanation parameters for when the model is deployed.
 - Upload the trained model artifacts and explanation parameters as a `Model` resource.
 - Make a batch prediction with explanations.
+
+```
+

notebooks/official/explainable_ai/sdk_custom_tabular_regression_online_explain_get_metadata.ipynb

@@ -62,7 +62,7 @@
         "## Overview\n",
         "\n",
         "\n",
-        "This tutorial demonstrates how to use the Vertex SDK to train and deploy a custom tabular regression model for online prediction with explanation."
+        "This tutorial demonstrates how to use the Vertex AI SDK to train and deploy a custom tabular regression model for online prediction with explanation."
       ]
     },
     {
@@ -73,7 +73,16 @@
       "source": [
         "### Objective\n",
         "\n",
-        "In this tutorial, you learn how to create a custom model from a Python script in a Google prebuilt Docker container using the Vertex SDK, and then do a prediction with explanations on the deployed model by sending data. You can alternatively create custom models using `gcloud` command-line tool or online using Cloud Console.\n",
+        "In this tutorial, you learn how to create a custom model from a Python script in a Google prebuilt Docker container using the Vertex AI SDK, and then do a prediction with explanations on the deployed model by sending data. You can alternatively create custom models using `gcloud` command-line tool or online using Cloud Console.\n",
+        "\n",
+        "This tutorial uses the following Google Cloud ML services:\n",
+        "\n",
+        "- `Vertex AI Training`\n",
+        "- `Vertex AI Online Prediction`\n",
+        "- `Vertex Explainable AI`\n",
+        "- `Vertex AI Model` resource\n",
+        "- `Vertex AI Endpoint` resource\n",
+        "\n",
         "\n",
         "The steps performed include:\n",
         "\n",

notebooks/official/feature_store/README.md

@@ -1,20 +1,24 @@
+[Using Vertex AI Feature Store with Pandas Dataframe](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/feature_store/sdk-feature-store-pandas.ipynb)
 
-[Using Vertex AI Feature Store with pandas DataFrame](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/feature_store/sdk-feature-store-pandas.ipynb)
-
-Learn how to use `Vertex AI Feature Store` with pandas DataFrame.
+```
+Learn how to use `Vertex AI Feature Store` with pandas Dataframe.
 
 The steps performed include:
 
 - Ingest Feature values from Pandas DataFrame into Feature Store's Entity types.
-- Read Entity Feature values from Online Feature Store into Pandas DataFrame.
-- Batch serve Feature values from your Feature Store into Pandas DataFrame.
+- Read Entity feature values from Online Feature Store into Pandas DataFrame.
+- Batch serve feature values from your Feature Store into Pandas DataFrame.
 
 
 - Online serving with updated feature values.
 - Point-in-time correctness to fetch feature values for training.
 
+```
+
+
 [Online and Batch predictions using Vertex AI Feature Store](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/feature_store/sdk-feature-store.ipynb)
 
+```
 Learn how to use `Vertex AI Feature Store` to import feature data, and to access the feature data for both online serving and offline tasks, such as training.
 
 The steps performed include:
@@ -23,3 +27,6 @@ The steps performed include:
 - Import feature data into `Vertex AI Feature Store` resource.
 - Serve online prediction requests using the imported features.
 - Access imported features in offline jobs, such as training jobs.
+
+```
+

notebooks/official/feature_store/feature_store_streaming_ingestion_sdk.ipynb

@@ -0,0 +1,774 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ur8xi4C7S06n"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2022 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "JAPoU8Sm5E6e"
+      },
+      "source": [
+        "# Feature Store: Streaming ingestion SDK\n",
+        "\n",
+        "<table align=\"left\">\n",
+        "\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/feature_store/feature_store_streaming_ingestion_sdk.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/feature_store/feature_store_streaming_ingestion_sdk.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/feature_store/feature_store_streaming_ingestion_sdk.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "      Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "  </td>\n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "24743cf4a1e1"
+      },
+      "source": [
+        "**_NOTE_**: This notebook has been tested in the following environment:\n",
+        "\n",
+        "* Python version = 3.9"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tvgnzT1CKxrO"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "This notebook demonstrates how to use Vertex AI Feature Store's streaming ingestion at the SDK layer."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d975e698c9a4"
+      },
+      "source": [
+        "### Objective\n",
+        "\n",
+        "In this tutorial, you learn how to ingest features from a `Pandas DataFrame` into your Vertex AI Feature Store using `write_feature_values` method from the Vertex AI SDK.\n",
+        "\n",
+        "This tutorial uses the following Google Cloud ML services and resources:\n",
+        "\n",
+        "- Vertex AI Feature Store\n",
+        "\n",
+        "\n",
+        "The steps performed include:\n",
+        "\n",
+        "- Create `Feature Store`\n",
+        "- Create new `Entity Type` for your `Feature Store`\n",
+        "- Ingest feature values from `Pandas DataFrame` into `Feature Store`'s `Entity Types`."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "08d289fa873f"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The dataset used for this notebook is the penguins dataset from [BigQuery public datasets](https://cloud.google.com/bigquery/public-data). This dataset has the following features: `culmen_length_mm`, `culmen_depth_mm`, `flipper_length_mm`, `body_mass_g`, `species`, and `sex`."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "aed92deeb4a0"
+      },
+      "source": [
+        "### Costs\n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "\n",
+        "Learn about [Vertex AI\n",
+        "pricing](https://cloud.google.com/vertex-ai/pricing) and use the [Pricing\n",
+        "Calculator](https://cloud.google.com/products/calculator/)\n",
+        "to generate a cost estimate based on your projected usage.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "i7EUnXsZhAGF"
+      },
+      "source": [
+        "## Installation\n",
+        "\n",
+        "Install the following packages required to execute this notebook."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2b4ef9b72d43"
+      },
+      "outputs": [],
+      "source": [
+        "# Install the packages\n",
+        "! pip3 install --upgrade google-cloud-aiplatform\\\n",
+        "                         google-cloud-bigquery\\\n",
+        "                         numpy\\\n",
+        "                         pandas\\\n",
+        "                         pyarrow -q"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "58707a750154"
+      },
+      "source": [
+        "### Colab only: Uncomment the following cell to restart the kernel."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "f200f10a1da3"
+      },
+      "outputs": [],
+      "source": [
+        "# Automatically restart kernel after installs so that your environment can access the new packages\n",
+        "# import IPython\n",
+        "\n",
+        "# app = IPython.Application.instance()\n",
+        "# app.kernel.do_shutdown(True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "BF1j6f9HApxa"
+      },
+      "source": [
+        "## Before you begin\n",
+        "\n",
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "2. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "3. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com).\n",
+        "\n",
+        "4. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "WReHDGG5g0XY"
+      },
+      "source": [
+        "#### Set your project ID\n",
+        "\n",
+        "**If you don't know your project ID**, try the following:\n",
+        "* Run `gcloud config list`.\n",
+        "* Run `gcloud projects list`.\n",
+        "* See the support page: [Locate the project ID](https://support.google.com/googleapi/answer/7014113)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "oM1iC_MfAts1"
+      },
+      "outputs": [],
+      "source": [
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# Set the project id\n",
+        "! gcloud config set project {PROJECT_ID}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "region"
+      },
+      "source": [
+        "#### Region\n",
+        "\n",
+        "You can also change the `REGION` variable used by Vertex AI. Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "kljmKgilI_de"
+      },
+      "outputs": [],
+      "source": [
+        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "sBCra4QMA2wR"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "Depending on your Jupyter environment, you may have to manually authenticate. Follow the relevant instructions below."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "74ccc9e52986"
+      },
+      "source": [
+        "**1. Vertex AI Workbench**\n",
+        "* Do nothing as you are already authenticated."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "de775a3773ba"
+      },
+      "source": [
+        "**2. Local JupyterLab instance, uncomment and run:**"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "254614fa0c46"
+      },
+      "outputs": [],
+      "source": [
+        "# ! gcloud auth login"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ef21552ccea8"
+      },
+      "source": [
+        "**3. Colab, uncomment and run:**"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "603adbbf0532"
+      },
+      "outputs": [],
+      "source": [
+        "# from google.colab import auth\n",
+        "# auth.authenticate_user()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "f6b2ccc891ed"
+      },
+      "source": [
+        "**4. Service account or other**\n",
+        "* See how to grant Cloud Storage permissions to your service account at https://cloud.google.com/storage/docs/gsutil/commands/iam#ch-examples."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "EsCYkJ4IU-z4"
+      },
+      "source": [
+        "### UUID\n",
+        "\n",
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "4jWj2DSTU9my"
+      },
+      "outputs": [],
+      "source": [
+        "import random\n",
+        "import string\n",
+        "\n",
+        "\n",
+        "# Generate a uuid of a specifed length(default=8)\n",
+        "def generate_uuid(length: int = 8) -> str:\n",
+        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
+        "\n",
+        "\n",
+        "UUID = generate_uuid()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "960505627ddf"
+      },
+      "source": [
+        "### Import libraries"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "PyQmSRbKA8r-"
+      },
+      "outputs": [],
+      "source": [
+        "import numpy as np\n",
+        "import pandas as pd\n",
+        "from google.cloud import aiplatform, bigquery"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "init_aip:mbsdk,all"
+      },
+      "source": [
+        "### Initialize Vertex AI SDK for Python\n",
+        "\n",
+        "Initialize the Vertex AI SDK for Python for your project."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "0ep8KuQhI_df"
+      },
+      "outputs": [],
+      "source": [
+        "aiplatform.init(project=PROJECT_ID, location=REGION)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "k5XsEiAuEWUJ"
+      },
+      "source": [
+        "## Download and prepare the data"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "rOd7Ixa1pqBY"
+      },
+      "outputs": [],
+      "source": [
+        "def download_bq_table(bq_table_uri: str) -> pd.DataFrame:\n",
+        "    # Remove bq:// prefix if present\n",
+        "    prefix = \"bq://\"\n",
+        "    if bq_table_uri.startswith(prefix):\n",
+        "        bq_table_uri = bq_table_uri[len(prefix) :]\n",
+        "\n",
+        "    table = bigquery.TableReference.from_string(bq_table_uri)\n",
+        "\n",
+        "    # Create a BigQuery client\n",
+        "    bqclient = bigquery.Client(project=PROJECT_ID)\n",
+        "\n",
+        "    # Download the table rows\n",
+        "    rows = bqclient.list_rows(\n",
+        "        table,\n",
+        "    )\n",
+        "    return rows.to_dataframe()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "SdX_m1Uppkfu"
+      },
+      "outputs": [],
+      "source": [
+        "BQ_SOURCE = \"bq://bigquery-public-data.ml_datasets.penguins\"\n",
+        "\n",
+        "# Download penguins BigQuery table\n",
+        "penguins_df = download_bq_table(BQ_SOURCE)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "QuQe6mSbFbhm"
+      },
+      "source": [
+        "### Prepare the data\n",
+        "\n",
+        "Feature values to be written to the Feature Store can take the form of a list of `WriteFeatureValuesPayload` objects, a Python `dict` of the form\n",
+        "\n",
+        "`{entity_id : {feature_id : feature_value}, ...},`\n",
+        "\n",
+        "or a pandas `Dataframe`, where the `index` column holds the unique entity ID strings and each remaining column represents a feature.  In this notebook, since you use a pandas `DataFrame` for ingesting features we convert the index column data type to `string` to be used as `Entity ID`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cljxzJ3bqDer"
+      },
+      "outputs": [],
+      "source": [
+        "# Prepare the data\n",
+        "penguins_df.index = penguins_df.index.map(str)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "GSxrSdSY2ovn"
+      },
+      "outputs": [],
+      "source": [
+        "# Remove null values\n",
+        "NA_VALUES = [\"NA\", \".\"]\n",
+        "penguins_df = penguins_df.replace(to_replace=NA_VALUES, value=np.NaN).dropna()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "vgn4oQmSqdKI"
+      },
+      "source": [
+        "## Create Feature Store and define schemas\n",
+        "\n",
+        "Vertex AI Feature Store organizes resources hierarchically in the following order:\n",
+        "\n",
+        "`Featurestore -> EntityType -> Feature`\n",
+        "\n",
+        "You must create these resources before you can ingest data into Vertex AI Feature Store.\n",
+        "\n",
+        "Learn more about [Vertex AI Feature Store](https://cloud.google.com/vertex-ai/docs/featurestore)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "yaHwdbGjZWTq"
+      },
+      "source": [
+        "### Create a Feature Store\n",
+        "\n",
+        "You create a Feature Store using `aiplatform.Featurestore.create` with the following parameters:\n",
+        "\n",
+        "* `featurestore_id (str)`: The ID to use for this Featurestore, which will become the final component of the Featurestore's resource name. The value must be unique within the project and location.\n",
+        "* `online_store_fixed_node_count`: Configuration for online serving resources.\n",
+        "* `project`: Project to create EntityType in. If not set, project set in `aiplatform.init` is used.\n",
+        "* `location`: Location to create EntityType in. If not set, location set in `aiplatform.init` is used.\n",
+        "* `sync`:  Whether to execute this creation synchronously."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cImsONglqfxO"
+      },
+      "outputs": [],
+      "source": [
+        "FEATURESTORE_ID = f\"penguins_{UUID}\"\n",
+        "\n",
+        "penguins_feature_store = aiplatform.Featurestore.create(\n",
+        "    featurestore_id=FEATURESTORE_ID,\n",
+        "    online_store_fixed_node_count=1,\n",
+        "    project=PROJECT_ID,\n",
+        "    location=REGION,\n",
+        "    sync=True,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "UfXgSD1VdzKb"
+      },
+      "source": [
+        "##### Verify that the Feature Store is created\n",
+        "Check if the Feature Store was successfully created by running the following code block."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "oud1OdfQd52r"
+      },
+      "outputs": [],
+      "source": [
+        "fs = aiplatform.Featurestore(\n",
+        "    featurestore_name=FEATURESTORE_ID,\n",
+        "    project=PROJECT_ID,\n",
+        "    location=REGION,\n",
+        ")\n",
+        "print(fs.gca_resource)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ep74rSlJWF3c"
+      },
+      "source": [
+        "### Create an EntityType\n",
+        "\n",
+        "An entity type is a collection of semantically related features. You define your own entity types, based on the concepts that are relevant to your use case. For example, a movie service might have the entity types `movie` and `user`, which group related features that correspond to movies or users.\n",
+        "\n",
+        "Here, you create an entity type entity type named `penguin_entity_type` using `create_entity_type` with the following parameters:\n",
+        "* `entity_type_id (str)`: The ID to use for the EntityType, which will become the final component of the EntityType's resource name. The value must be unique within a Feature Store.\n",
+        "* `description`: Description of the EntityType."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "zNzr-FlEr3tI"
+      },
+      "outputs": [],
+      "source": [
+        "ENTITY_TYPE_ID = f\"penguin_entity_type_{UUID}\"\n",
+        "\n",
+        "# Create penguin entity type\n",
+        "penguins_entity_type = penguins_feature_store.create_entity_type(\n",
+        "    entity_type_id=ENTITY_TYPE_ID,\n",
+        "    description=\"Penguins entity type\",\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "CquSdTp7duVw"
+      },
+      "source": [
+        "##### Verify that the EntityType is created\n",
+        "Check if the Entity Type was successfully created by running the following code block."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "76ocr_hJsG-t"
+      },
+      "outputs": [],
+      "source": [
+        "entity_type = penguins_feature_store.get_entity_type(entity_type_id=ENTITY_TYPE_ID)\n",
+        "\n",
+        "print(entity_type.gca_resource)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2vYV2UUFehwZ"
+      },
+      "source": [
+        "### Create Features\n",
+        "A feature is a measurable property or attribute of an entity type. For example, `penguin` entity type has features such as `flipper_length_mm`, and `body_mass_g`. Features can be created within each entity type.\n",
+        "\n",
+        "When you create a feature, you specify its value type such as `DOUBLE`, and `STRING`. This value determines what value types you can ingest for a particular feature.\n",
+        "\n",
+        "Learn more about [Feature Value Types](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.featurestores.entityTypes.features)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "WQ5EsPPbsSuE"
+      },
+      "outputs": [],
+      "source": [
+        "penguins_feature_configs = {\n",
+        "    \"species\": {\n",
+        "        \"value_type\": \"STRING\",\n",
+        "    },\n",
+        "    \"island\": {\n",
+        "        \"value_type\": \"STRING\",\n",
+        "    },\n",
+        "    \"culmen_length_mm\": {\n",
+        "        \"value_type\": \"DOUBLE\",\n",
+        "    },\n",
+        "    \"culmen_depth_mm\": {\n",
+        "        \"value_type\": \"DOUBLE\",\n",
+        "    },\n",
+        "    \"flipper_length_mm\": {\n",
+        "        \"value_type\": \"DOUBLE\",\n",
+        "    },\n",
+        "    \"body_mass_g\": {\"value_type\": \"DOUBLE\"},\n",
+        "    \"sex\": {\"value_type\": \"STRING\"},\n",
+        "}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "AKRXJCPijM8w"
+      },
+      "source": [
+        "You can create features either using `create_feature` or `batch_create_features`. Here, for convinience, you have added all feature configs in one variabel, so we use `batch_create_features`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "tXOI1Onhs46x"
+      },
+      "outputs": [],
+      "source": [
+        "penguin_features = penguins_entity_type.batch_create_features(\n",
+        "    feature_configs=penguins_feature_configs,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "WBx26pZItUN4"
+      },
+      "source": [
+        "### Write features to the Feature Store\n",
+        "Use the `write_feature_values` API to write a feature to the Feature Store with the following parameter:\n",
+        "\n",
+        "* `instances`: Feature values to be written to the Feature Store that can take the form of a list of WriteFeatureValuesPayload objects, a Python dict, or a pandas Dataframe.\n",
+        "\n",
+        "This streaming ingestion feature has been introduced to the Vertex AI SDK under the **preview** namespace. Here, you pass the pandas `Dataframe` you created from penguins dataset as `instances` parameter.\n",
+        "\n",
+        "Learn more about [Streaming ingestion API](https://github.com/googleapis/python-aiplatform/blob/e6933503d2d3a0f8a8f7ef8c178ed50a69ac2268/google/cloud/aiplatform/preview/featurestore/entity_type.py#L36)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "iUGI-ftltXqE"
+      },
+      "outputs": [],
+      "source": [
+        "penguins_entity_type.preview.write_feature_values(instances=penguins_df)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "STq67KHO3q_e"
+      },
+      "source": [
+        "## Read back written features\n",
+        "\n",
+        "Wait a few seconds for the write to propagate, then do an online read to confirm the write was successful."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "lwoMnze43r9G"
+      },
+      "outputs": [],
+      "source": [
+        "ENTITY_IDS = [str(x) for x in range(100)]\n",
+        "penguins_entity_type.read(entity_ids=ENTITY_IDS)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "TpV-iwP9qw9c"
+      },
+      "source": [
+        "## Cleaning up\n",
+        "\n",
+        "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
+        "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
+        "\n",
+        "Otherwise, you can delete the individual resources you created in this tutorial:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "sx_vKniMq9ZX"
+      },
+      "outputs": [],
+      "source": [
+        "penguins_feature_store.delete(force=True)"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "feature_store_streaming_ingestion_sdk.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/official/matching_engine/README.md

@@ -1,7 +1,6 @@
-## Vertex-AI: Matching Engine Notebook
-
-<a id="sdk_matching_engine_for_indexing"></a>[Create Vertex AI Matching Engine index](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/matching_engine/sdk_matching_engine_for_indexing.ipynb)
+[Create Vertex AI Matching Engine index](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/matching_engine/sdk_matching_engine_for_indexing.ipynb)
 
+```
 Learn how to create Approximate Nearest Neighbor (ANN) Index, query against indexes, and validate the performance of the index.
 
 The steps performed include:
@@ -12,20 +11,39 @@ The steps performed include:
 * Perform online query
 * Compute recall
 
-    <details>
-    <summary>Example code snippet from the Notebook:</summary>
-    
-    * Create an IndexEndpoint with VPC Network
-        ```python
-        # [START aiplatform_sdk_matching_engine_for_indexing]
-        VPC_NETWORK = "[your-network-name]"
-        VPC_NETWORK_FULL = "projects/{}/global/networks/{}".format(PROJECT_NUMBER, VPC_NETWORK)
-        my_index_endpoint = aiplatform.MatchingEngineIndexEndpoint.create(
-            display_name="index_endpoint_for_demo",
-            description="index endpoint description",
-            network=VPC_NETWORK_FULL,
-        )
-        # [END aiplatform_sdk_matching_engine_for_indexing]
-        ```
-        [:notebook: sdk_matching_engine_for_indexing.ipynb](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/matching_engine/sdk_matching_engine_for_indexing.ipynb)
-    </details>
+```
+
+
+[Introduction to builtin Swivel embedding algorithm](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/matching_engine/intro-swivel.ipynb)
+
+```
+Learn how to train custom embeddings using Vertex AI Pipelines and deploy the model for serving.
+
+The steps performed include:
+
+1. **Setup**: Importing the required libraries and setting your global variables.
+2. **Configure parameters**: Setting the appropriate parameter values for the pipeline job.
+3. **Train on Vertex AI Pipelines**: Create a Swivel job to Vertex Pipelines using pipeline template.
+4. **Deploy on Vertex AI Prediction**: Importing and deploying the trained model to a callable endpoint.
+5. **Predict**: Calling the deployed endpoint using online prediction.
+6. **Cleaning up**: Deleting resources created by this tutorial.
+
+```
+
+
+[Introduction to builtin Two-towers embedding algorithm](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/matching_engine/two-tower-model-introduction.ipynb)
+
+```
+Learn how to run the two-tower model.
+
+The steps performed include:
+1. **Setup**: Importing the required libraries and setting your global variables.
+2. **Configure parameters**: Setting the appropriate parameter values for the training job.
+3. **Train on Vertex AI Training**: Submitting a training job.
+4. **Deploy on Vertex AI Prediction**: Importing and deploying the trained model to a callable endpoint.
+5. **Predict**: Calling the deployed endpoint using online or batch prediction.
+6. **Hyperparameter tuning**: Running a hyperparameter tuning job.
+7. **Cleaning up**: Deleting resources created by this tutorial.
+
+```
+

notebooks/official/matching_engine/intro-swivel.ipynb

@@ -81,6 +81,12 @@
         "\n",
         "In this notebook, you learn how to train custom embeddings using Vertex AI Pipelines and deploy the model for serving. \n",
         "\n",
+        "This tutorial uses the following Google Cloud ML services:\n",
+        "\n",
+        "- `Vertex AI Pipelines`\n",
+        "- `Vertex AI Training`\n",
+        "- `Swivel builtin algorithm`\n",
+        "\n",
         "The steps performed include:\n",
         "\n",
         "1. **Setup**: Importing the required libraries and setting your global variables.\n",

notebooks/official/matching_engine/sdk_matching_engine_for_indexing.ipynb

@@ -60,7 +60,7 @@
       "source": [
         "## Overview\n",
         "\n",
-        "This example demonstrates how to use the GCP ANN Service. It is a high scale, low latency solution, to find similar vectors (or more specifically \"embeddings\") for a large corpus. Moreover, it is a fully managed offering, further reducing operational overhead. It is built upon [Approximate Nearest Neighbor (ANN) technology](https://ai.googleblog.com/2020/07/announcing-scann-efficient-vector.html) developed by Google Research."
+        "This example demonstrates how to use the Vertex AI ANN Service. It is a high scale, low latency solution, to find similar vectors (or more specifically \"embeddings\") for a large corpus. Moreover, it is a fully managed offering, further reducing operational overhead. It is built upon [Approximate Nearest Neighbor (ANN) technology](https://ai.googleblog.com/2020/07/announcing-scann-efficient-vector.html) developed by Google Research."
       ]
     },
     {
@@ -73,6 +73,10 @@
         "\n",
         "In this notebook, you learn how to create Approximate Nearest Neighbor (ANN) Index, query against indexes, and validate the performance of the index. \n",
         "\n",
+        "This tutorial uses the following Google Cloud ML services:\n",
+        "\n",
+        "- `Vertex AI Matching Engine`\n",
+        "\n",
         "The steps performed include:\n",
         "\n",
         "* Create ANN Index and Brute Force Index\n",

notebooks/official/matching_engine/two-tower-model-introduction.ipynb

@@ -29,7 +29,7 @@
         "id": "JAPoU8Sm5E6e"
       },
       "source": [
-        "# Introduction to builtin Two-towers embedding algorithm\n",
+        "# Introduction to builtin Two-Towers embedding algorithm\n",
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
@@ -73,7 +73,13 @@
       "source": [
         "### Objective\n",
         "\n",
-        "In this notebook, you learn how to run the two-tower model.\n",
+        "In this notebook, you learn how to run the Two-Tower model.\n",
+        "\n",
+        "This tutorial uses the following Google Cloud ML services:\n",
+        "\n",
+        "- `Vertex AI Training`\n",
+        "- `Vertex AI Prediction`\n",
+        "- `Two-Tower builtin algorithm`\n",
         "\n",
         "The steps performed include:\n",
         "1. **Setup**: Importing the required libraries and setting your global variables.\n",

notebooks/official/migration/README.md

@@ -0,0 +1,88 @@
+[Custom Image Classification w/custom training container](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ3 Vertex SDK Custom Image Classification with custom training container.ipynb)
+
+```
+Learn how to train a tensorflow image classification model using a custom container and Vertex AI training.
+
+The steps performed include:
+
+- *Package the training code into a python application.*
+- *Containerize the training application using Cloud Build and Artifact Registry.*
+- *Create a custom container training job in Vertex AI and run it.*
+- *Evaluate the model generated from the training job.*
+- *Create a model resource for the trained model in Vertex AI Model Registry.*
+- *Run a Vertex AI batch prediction job.*
+- *Deploy the model resource to a Vertex AI Endpoint.*
+- *Run a online prediction job on the model resource.*
+- *Clean up the resources created.*
+
+```
+
+
+[AutoML Tabular Binary Classification](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ4 Vertex SDK AutoML Tabular Binary Classification.ipynb)
+
+```
+In this tutorial, you create an AutoML tabular binary classification model and deploy for online prediction from a Python script using the Vertex AI SDK.
+
+The steps performed include:
+
+- Create a Vertex `Dataset` resource.
+- Train the model.
+- View the model evaluation.
+- Deploy the `Model` resource to a serving `Endpoint` resource.
+- Make a prediction.
+- Undeploy the `Model`
+
+```
+
+
+[Custom Scikit-Learn model with pre-built training container](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ10 Vertex SDK Custom Scikit-Learn with pre-built training container.ipynb)
+
+```
+Learn to use `Vertex AI Training` to create a custom trained model and use `Vertex AI Batch Prediction` to do a batch prediction on the trained model.
+
+The steps performed include:
+
+- Create a `Vertex AI` custom job for training a scikit-learn model.
+- Upload the trained model artifacts as a `Model` resource.
+- Make a batch prediction.
+- Deploy model to a endpoint
+- Make a online prediction
+
+```
+
+
+[AutoML Text Entity Extraction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ7 Vertex SDK AutoML Text Entity Extraction.ipynb)
+
+```
+The objective of this notebook is to build a AutoML Text Entity Extrasction Model.
+
+The steps performed include the following:
+
+* Set your task name, and GCS prefix
+* Copy AutoML video demo train data for creating managed dataset
+* Create a dataset on Vertex AI.
+* Configure a training job
+* Launch a training job and create a model on Vertex AI
+* Copy AutoML Video Demo Prediction Data for creating batch prediction job
+* Perform batch prediction job on the model
+
+```
+
+
+[AutoML Text Classification](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ6 Vertex SDK AutoML Text Classification.ipynb)
+
+```
+The objective of this notebook is to build a AutoML Video Classification Model.
+
+The steps performed include the following:
+
+* Set your task name, and GCS prefix
+* Copy AutoML video demo train data for creating managed dataset
+* Create a dataset on Vertex AI.
+* Configure a training job
+* Launch a training job and create a model on Vertex AI
+* Copy AutoML Video Demo Prediction Data for creating batch prediction job
+* Perform batch prediction job on the model
+
+```
+

notebooks/official/ml_metadata/README.md

@@ -1,6 +1,6 @@
-
 [Track parameters and metrics for locally trained models](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-locally-trained-models.ipynb)
 
+```
 Learn how to use `Vertex ML Metadata` to track training parameters and evaluation metrics.
 
 The steps performed include:
@@ -8,16 +8,24 @@ The steps performed include:
 - Track parameters and metrics for a locally trained model.
 - Extract and perform analysis for all parameters and metrics within an Experiment.
 
+```
+
+
 [Track parameters and metrics for custom training jobs](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-custom-jobs.ipynb)
 
+```
 Learn how to use Vertex AI SDK for Python to:
 
 The steps performed include:
 - Track training parameters and prediction metrics for a custom training job.
 - Extract and perform analysis for all parameters and metrics within an Experiment.
 
+```
+
+
 [Track artifacts and metrics across Vertex AI Pipelines runs using Vertex ML Metadata](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/ml_metadata/vertex-pipelines-ml-metadata.ipynb)
 
+```
 Learn how to track artifacts and metrics with `Vertex ML Metadata` in `Vertex AI Pipeline` runs.
 
 The steps performed include:
@@ -28,3 +36,6 @@ The steps performed include:
 * Compare Vertex Pipelines runs, both in the Cloud console and programmatically
 * Trace the lineage for pipeline-generated artifacts
 * Query your pipeline run metadata
+
+```
+

notebooks/official/ml_metadata/vertex-pipelines-ml-metadata.ipynb

@@ -74,6 +74,10 @@
         "\n",
         "In this notebook, you learn how to track artifacts and metrics with `Vertex ML Metadata` in `Vertex AI Pipeline` runs.\n",
         "\n",
+        "This tutorial uses the following Google Cloud ML services and resources:\n",
+        "\n",
+        "- Vertex AI Pipelines\n",
+        "- Vertex ML Metadata\n",
         "\n",
         "The steps performed include:\n",
         "\n",

notebooks/official/model_evaluation/README.md

@@ -1,32 +1,40 @@
+[Evaluating batch prediction results from an AutoML Tabular classification model](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model_evaluation/automl_tabular_classification_model_evaluation.ipynb)
 
-[Evaluating BatchPrediction results from AutoML Tabular Classification model](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model_evaluation/automl_tabular_classification_model_evaluation.ipynb)
-
-Learn how to train a Vertex AI AutoML Tabular Classification model and learn how to evaluate it through a Vertex AI pipeline job using `google_cloud_pipeline_components`:
+```
+Learn how to train a Vertex AI AutoML Tabular classification model and learn how to evaluate it through a Vertex AI pipeline job using `google_cloud_pipeline_components`:
 
 The steps performed include:
 
 - Create a Vertex AI `Dataset`.
-- Train a Automl Tabular Classification model on the `Dataset` resource.
+- Train an Automl Tabular classification model on the `Dataset` resource.
 - Import the trained `AutoML model resource` into the pipeline.
 - Run a `Batch Prediction` job.
-- Evaulate the AutoML model using the `Classification Evaluation Component`.
+- Evaluate the AutoML model using the `Classification Evaluation component`.
 - Import the classification metrics to the AutoML model resource.
 
-[Evaluating BatchPrediction results from AutoML Tabular Classification model](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model_evaluation/automl_video_classification_model_evaluation.ipynb)
+```
 
-Learn how to train a Vertex AI AutoML Tabular Classification model and learn how to evaluate it through a Vertex AI pipeline job using `google_cloud_pipeline_components`:
+
+[Evaluating batch prediction results from AutoML Video classification model](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model_evaluation/automl_video_classification_model_evaluation.ipynb)
+
+```
+Learn how to train a Vertex AI AutoML Video classification model and learn how to evaluate it through a Vertex AI pipeline job using `google_cloud_pipeline_components`:
 
 The steps performed include:
 
-- Create a Vertex AI `Dataset`.
-- Train a Automl Tabular Classification model on the `Dataset` resource.
-- Import the trained `AutoML model resource` into the pipeline.
-- Run a `Batch Prediction` job.
-- Evaulate the AutoML model using the `Classification Evaluation Component`.
-- Import the classification metrics to the AutoML model resource.
+- Create a `Vertex AI Dataset`.
+- Train a Automl Video Classification model on the `Vertex AI Dataset` resource.
+- Import the trained `AutoML Vertex AI Model resource` into the pipeline.
+- Run a batch prediction job inside the pipeline.
+- Evaulate the AutoML model using the classification evaluation component.
+- Import the classification metrics to the AutoML Vertex AI Model resource.
 
-[Evaluating BatchPrediction results from AutoML Tabular regression model](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model_evaluation/automl_tabular_regression_model_evaluation.ipynb)
+```
 
+
+[Evaluating batch prediction results from AutoML Tabular regression model](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model_evaluation/automl_tabular_regression_model_evaluation.ipynb)
+
+```
 Learn how to evaluate a Vertex AI model resource through a Vertex AI pipeline job using `google_cloud_pipeline_components`:
 
 The steps performed include:
@@ -35,12 +43,16 @@ The steps performed include:
 - Configure a `AutoMLTabularTrainingJob`
 - Run the `AutoMLTabularTrainingJob` which returns a model
 - Import a pre-trained `AutoML model resource` into the pipeline
-- Run a `batch prediction` job
+- Run a `batch prediction` job in the pipeline
 - Evaulate the AutoML model using the `regression evaluation component`
-- Import the Classification Metrics to the AutoML model resource
+- Import the Regression Metrics to the AutoML model resource
 
-[Evaluating Batch Prediction results from Custom Tabular regression model](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model_evaluation/custom_tabular_regression_model_evaluation.ipynb)
+```
 
+
+[Evaluating batch prediction results from custom tabular regression model](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model_evaluation/custom_tabular_regression_model_evaluation.ipynb)
+
+```
 Learn how to evaluate a Vertex AI model resource through a Vertex AI pipeline job using `google_cloud_pipeline_components`:
 
 The steps performed include:
@@ -50,13 +62,17 @@ The steps performed include:
 - Retrieve and load the model artifacts.
 - View the model evaluation.
 - Upload the model as a Vertex AI Model resource.
-- Import a pre-trained `Vertex AI model resource` into the pipeline
-- Run a `batch prediction` job
-- Evaulate the model using the `regression evaluation component`
-- Import the Classification Metrics to the Vertex AI model resource
+- Import a pre-trained `Vertex AI model resource` into the pipeline.
+- Run a `batch prediction` job in the pipeline.
+- Evaulate the model using the `regression evaluation component`.
+- Import the Regression Metrics to the Vertex AI model resource.
+
+```
+
 
 [AutoML text classification pipelines using google-cloud-pipeline-components](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model_evaluation/automl_text_classification_model_evaluation.ipynb)
 
+```
 Learn how to use `Vertex AI Pipelines` and `Google Cloud Pipeline Components` to build an `AutoML` text classification model.
 
 The steps performed include:
@@ -68,4 +84,28 @@ The steps performed include:
 - Evaulate the AutoML model using the `Classification Evaluation Component`.
 - Import the classification metrics to the AutoML model resource.
 
+```
+
+
+[Evaluating BatchPrediction results from a Custom Tabular classification model](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model_evaluation/custom_tabular_classification_model_evaluation.ipynb)
+
+```
+In this tutorial, you train a scikit-learn RandomForest model, save it in Vertex AI Model Registry and learn how to evaluate it through a Vertex AI pipeline job using `google_cloud_pipeline_components`.
+
+The steps performed include:
+
+- Fetch the dataset from the public source.
+- Preprocess the data locally and save test data in BigQuery.
+- Train a RandomForest classification model locally using scikit-learn Python package.
+- Create a custom container in Artifact Registry for predictions.
+- Upload the model in Vertex AI Model Registry.
+- Create and run a Vertex AI Pipeline that:
+    - Imports the trained model into the pipeline.
+    - Runs a `Batch Prediction` job on the test data in BigQuery.
+    - Evaulates the model using the evaluation component from google-cloud-pipeline-components Python SDK.
+    - Imports the classification metrics in to the model resource in Vertex AI Model Registry.
+- Print and visualize the classification evaluation metrics.
+- Clean up the resources created in this notebook.
+
+```
 

notebooks/official/model_monitoring/README.md

@@ -1,6 +1,24 @@
 
+### model_monitoring 
+
+
+[Vertex AI Batch Prediction with Model Monitoring](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model_monitoring/batch_prediction_model_monitoring.ipynb)
+
+```
+Learn to use the `Vertex AI Model Monitoring` service to detect drift and anomalies in batch prediction.
+
+The steps performed include:
+
+- Upload a pre-trained model as a Vertex AI Model resource.
+- Generate batch prediction requests.
+- Interpret the statistics, visualizations, other data reported by the model monitoring feature.
+
+```
+
+
 [Vertex AI Model Monitoring with Explainable AI Feature Attributions](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model_monitoring/model_monitoring.ipynb)
 
+```
 Learn to use the `Vertex AI Model Monitoring` service to detect drift and anomalies in prediction requests from a deployed `Vertex AI Model` resource.
 
 The steps performed include:
@@ -12,3 +30,6 @@ The steps performed include:
 - Initialize the baseline distribution for model monitoring.
 - Generate synthetic prediction requests.
 - Understand how to interpret the statistics, visualizations, other data reported by the model monitoring feature.
+
+```
+

notebooks/official/model_monitoring/batch_prediction_model_monitoring.ipynb

@@ -36,7 +36,7 @@
         "<table align=\"left\">\n",
         "  <td>\n",
         "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model_monitoring/batch_prediction_model_monitoring.ipynb\">\n",
-        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Open in Colab\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
@@ -44,9 +44,10 @@
         "        <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
-        "  </td><td>\n",
-        "    <a href=\"https://console.cloud.google.com/ai-platform/notebooks/deploy-notebook?name=Model%20Monitoring&download_url=https%3A%2F%2Fraw.githubusercontent.com%2FGoogleCloudPlatform%2Fvertex-ai-samples%2Fmain%2Fnotebooks%2Fofficial%2Fmodel_monitoring%2Fbatch_prediction_model_monitoring.ipynb\">\n",
-        "       <img src=\"https://www.gstatic.com/cloud/images/navigation/vertex-ai.svg\" alt=\"Google Cloud Notebooks\">Open in Workbench AI Notebook\n",
+        "  </td>\n",
+        "    <td>\n",
+        "<a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/model_monitoring/batch_prediction_model_monitoring.ipynb\" target='_blank'>\n",
+        "       <img src=\"https://www.gstatic.com/cloud/images/navigation/vertex-ai.svg\" alt=\"Vertex AI logo\">Open in Vertex AI Workbench\n",
         "    </a>\n",
         "  </td> \n",
         "</table>"
@@ -70,6 +71,8 @@
       },
       "source": [
         "### Objective\n",
+        "In this notebook, you learn to use the `Vertex AI Model Monitoring` service to detect drift and anomalies in batch prediction.\n",
+        "\n",
         "This tutorial uses the following Google Cloud ML services:\n",
         "\n",
         "- Vertex AI Model Monitoring\n",
@@ -80,8 +83,26 @@
         "\n",
         "- Upload a pre-trained model as a Vertex AI Model resource.\n",
         "- Generate batch prediction requests.\n",
-        "- Interpret the statistics, visualizations, other data reported by the model monitoring feature.\n",
+        "- Interpret the statistics, visualizations, other data reported by the model monitoring feature."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "35b52a5fba4a"
+      },
+      "source": [
+        "### Model\n",
         "\n",
+        "This tutorial uses a pre-trained model, where the model artifacts are stored in a public Cloud Storage bucket. The model predicts for an online gaming site, the probability that a player may churn, i.e. stop being an active player."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "5508f7979954"
+      },
+      "source": [
         "### Costs \n",
         "\n",
         "This tutorial uses billable components of Google Cloud:\n",

notebooks/official/model_registry/README.md

@@ -1,14 +1,20 @@
 
+### model_registry 
+
+
 [Deploy BiqQuery ML Model on Vertex AI Model Registry and make predictions](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model_registry/bqml_vertexai_model_registry.ipynb)
 
+```
 Learn how to use `Vertex AI Model Registry` with `BigQuery ML` and make batch predictions:
 
 The steps performed include:
 
 - Train a model with `BigQuery ML`
-- Upload the model to `Vertex AI Model Registry`
+- Upload the model to `Vertex AI Model Registry` 
 - Create a `Vertex AI Endpoint` resource
 - Deploy the `Model` resource to the `Endpoint` resource
 - Make `prediction` requests to the model endpoint
 - Run `batch prediction` job on the `Model` resource
 
+```
+

notebooks/official/pipelines/README.md

@@ -1,6 +1,10 @@
 
+### pipelines 
+
+
 [AutoML image classification pipelines using google-cloud-pipeline-components](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/google_cloud_pipeline_components_automl_images.ipynb)
 
+```
 Learn how to use `Vertex AI Pipelines` and `Google Cloud Pipeline Components` to build an `AutoML` image classification model.
 
 The steps performed include:
@@ -13,10 +17,12 @@ The steps performed include:
 - Compile the KFP pipeline.
 - Execute the KFP pipeline using `Vertex AI Pipelines`
 
+```
 
 
 [Metrics visualization and run comparison using the KFP SDK](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/metrics_viz_run_compare_kfp.ipynb)
 
+```
 Learn how to use the KFP SDK to build pipelines that generate evaluation metrics.
 
 The steps performed include:
@@ -28,8 +34,12 @@ The steps performed include:
 - Execute KFP pipelines
 - Compare metrics across pipeline runs
 
+```
+
+
 [Lightweight Python function-based components, and component I/O](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/lightweight_functions_component_io_kfp.ipynb)
 
+```
 Learn to use the KFP SDK to build lightweight Python function-based components, and then you learn to use `Vertex AI Pipelines` to execute the pipeline.
 
 The steps performed include:
@@ -41,8 +51,12 @@ The steps performed include:
 - Compile the KFP pipeline.
 - Execute the KFP pipeline using `Vertex AI Pipelines`
 
+```
+
+
 [Custom training with pre-built Google Cloud Pipeline Components](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/custom_model_training_and_batch_prediction.ipynb)
 
+```
 Learn to use `Vertex AI Pipelines` and `Google Cloud Pipeline Components` to build a custom model.
 
 The steps performed include:
@@ -54,10 +68,12 @@ The steps performed include:
     - Deploy the `Model` resource to the `Endpoint` resource.
     - Make a batch prediction request.
 
+```
 
 
 [AutoML Tabular pipelines using google-cloud-pipeline-components](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/automl_tabular_classification_beans.ipynb)
 
+```
 Learn to use `Vertex AI Pipelines` and `Google Cloud Pipeline Components` to build an `AutoML` tabular classification model.
 
 The steps performed include:
@@ -70,10 +86,12 @@ The steps performed include:
 - Compile the KFP pipeline.
 - Execute the KFP pipeline using `Vertex AI Pipelines`
 
+```
 
 
 [Training an acquisition-prediction model using Swivel, BigQuery ML and Vertex AI Pipelines](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/google_cloud_pipeline_components_bqml_text.ipynb)
 
+```
 Learn how to build a simple BigQuery ML pipeline using Vertex AI pipelines in order to calculate text embeddings of content from articles and classify them
 into the *corporate acquisitions* category.
 
@@ -85,8 +103,12 @@ The steps performed include:
 - Building and configuring a Kubeflow DSL pipeline with all the created components.
 - Compiling and running the pipeline in Vertex AI Pipelines.
 
+```
+
+
 [Pipelines introduction for KFP](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/pipelines_intro_kfp.ipynb)
 
+```
 Learn how to use the KFP SDK to build pipelines that generate evaluation metrics.
 
 The steps performed include:
@@ -94,8 +116,12 @@ The steps performed include:
 - Define and compile a `Vertex AI` pipeline.
 - Specify which service account to use for a pipeline run.
 
+```
+
+
 [AutoML tabular regression pipelines using google-cloud-pipeline-components](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/google_cloud_pipeline_components_automl_tabular.ipynb)
 
+```
 Learn to use `Vertex AI Pipelines` and `Google Cloud Pipeline Components` to build an `AutoML` tabular regression model.
 
 The steps performed include:
@@ -108,10 +134,12 @@ The steps performed include:
 - Compile the KFP pipeline.
 - Execute the KFP pipeline using `Vertex AI Pipelines`
 
+```
 
 
 [Model upload, predict, and evaluate using google-cloud-pipeline-components](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/google_cloud_pipeline_components_model_upload_predict_evaluate.ipynb)
 
+```
 Learn how to evaluate a custom model using a pipeline with components from `google_cloud_pipeline_components` and a custom pipeline component you build.
 
 The steps performed include:
@@ -121,9 +149,12 @@ The steps performed include:
 - Generate evaluation `Metrics` artifact about the `Model` resource.
 - Compare the evaluation metrics to a threshold.
 
+```
+
 
 [Pipeline control structures using the KFP SDK](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/control_flow_kfp.ipynb)
 
+```
 Learn how to use the KFP SDK to build pipelines that use loops and conditionals, including nested examples.
 
 The steps performed include:
@@ -133,8 +164,12 @@ The steps performed include:
 - Compile the KFP pipeline.
 - Execute the KFP pipeline using `Vertex AI Pipelines`
 
+```
+
+
 [Loan eligibility prediction using `google-cloud-pipeline-components` and Spark ML](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/google_cloud_pipeline_components_dataproc_tabular.ipynb)
 
+```
 Learn how to build a Vertex AI pipeline and train a Random-forest model using Spark ML for loan-eligibility classification problem.
 
 The steps performed include:
@@ -145,8 +180,30 @@ The steps performed include:
 *   Build a Vertex AI pipeline and run the training job.
 *   Use the Spark serving image in order to deploy a Spark model on Vertex AI Endpoint.
 
+```
+
+
+[Model train, upload, and deploy using Google Cloud Pipeline Components](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/google_cloud_pipeline_components_model_train_upload_deploy.ipynb)
+
+```
+Learn how to use `Vertex AI Pipelines` and `Google Cloud Pipeline Components` to build and deploy a custom model.
+
+The steps performed include:
+
+- Create a KFP pipeline:
+    - Train a custom model.
+    - Uploads the trained model as a `Model` resource.
+    - Creates an `Endpoint` resource.
+    - Deploys the `Model` resource to the `Endpoint` resource.
+- Compile the KFP pipeline.
+- Execute the KFP pipeline using `Vertex AI Pipelines`
+
+```
+
+
 [AutoML text classification pipelines using google-cloud-pipeline-components](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/google_cloud_pipeline_components_automl_text.ipynb)
 
+```
 Learn to use `Vertex AI Pipelines` and `Google Cloud Pipeline Components` to build an `AutoML` text classification model.
 
 The steps performed include:
@@ -159,4 +216,27 @@ The steps performed include:
 - Compile the KFP pipeline.
 - Execute the KFP pipeline using `Vertex AI Pipelines`
 
+```
+
+
+[Training and batch prediction with BigQuery source and destinantion for a custom tabular classification model](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/custom_tabular_train_batch_pred_bq_pipeline.ipynb)
+
+```
+In this tutorial, you train a scikit-learn tabular classification model and create batch prediction job for it through a Vertex AI pipeline using `google_cloud_pipeline_components`.
+
+The steps performed include:
+
+- Create a dataset in BigQuery.
+- Set some data aside from the source dataset for batch prediction.
+- Create a custom python package for training application.
+- Upload the python package to Cloud Storage.
+- Create a Vertex AI Pipeline that:
+    - creates a Vertex AI Dataset from the source dataset.
+    - trains a scikit-learn RandomForest classification model on the dataset.
+    - uploads the trained model to Vertex AI Model Registry.
+    - runs a batch prediction job with the model on the test data.
+- Check the prediction results from the destination table in BigQuery.
+- Clean up the resources created in this notebook.
+
+```
 

notebooks/official/prediction/README.md

@@ -0,0 +1,17 @@
+
+### prediction 
+
+
+[Custom model batch prediction with feature filtering](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/prediction/custom_batch_prediction_feature_filter.ipynb)
+
+```
+Learn how to create a custom-trained model from a Python script in a Docker container using the Vertex AI SDK for Python, and then run a batch prediction job by including or excluding a list of features.
+
+The steps performed include:
+
+- Create a Vertex AI custom `TrainingPipeline` for training a model.
+- Train a TensorFlow model.
+- Send batch prediction job.
+
+```
+

notebooks/official/pytorch/README.md

@@ -0,0 +1,21 @@
+
+### pytorch 
+
+
+[Training, tuning and deploying a PyTorch text sentiment classification model on Vertex AI](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pytorch/pytorch-text-sentiment-classification-custom-train-deploy.ipynb)
+
+```
+Learn to build, train, tune and deploy a PyTorch model on [Vertex AI](https://cloud.
+
+The steps performed include:
+
+- Create training package for the text classification model.
+- Train the model with custom training on Vertex AI.
+- Check the created model artifacts.
+- Create a custom container for predictions.
+- Deploy the trained model to a Vertex AI Endpoint using the custom container for predictions.
+- Send online prediction requests to the deployed model and validate.
+- Clean up the resources created in this notebook.
+
+```
+

notebooks/official/reduction_server/README.md

@@ -1,6 +1,10 @@
 
+### reduction_server 
+
+
 [PyTorch distributed training with Vertex AI Reduction Server](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/reduction_server/pytorch_distributed_training_reduction_server.ipynb)
 
+```
 Learn how to create a PyTorch distributed training job that uses PyTorch distributed training framework and tools, and run the training job on the Vertex AI Training service with Reduction Server.
 
 The steps performed include:
@@ -8,4 +12,7 @@ The steps performed include:
 * Create a PyTorch distributed training application
 * Package the training application with pre-built containers
 * Create a custom job on Vertex AI with Reduction Server
-* Submit and monitor the job 
+* Submit and monitor the job
+
+```
+

notebooks/official/sdk/README.md

@@ -1,6 +1,10 @@
 
+### sdk 
+
+
 [AutoML Video Classification Example](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/sdk/SDK_AutoML_Video_Classification.ipynb)
 
+```
 The objective of this notebook is to build a AutoML Video Classification Model.
 
 The steps performed include the following:
@@ -13,9 +17,12 @@ The steps performed include the following:
 - Copy AutoML Video Demo Prediction Data for creating batch prediction job
 - Perform batch prediction job on the model
 
+```
+
 
 [Custom training using Python package, managed text dataset, and TF Serving container](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/sdk/SDK_Custom_Training_Python_Package_Managed_Text_Dataset_Tensorflow_Serving_Container.ipynb)
 
+```
 Learn how to create a Custom Model using Custom Python Package Training and you learn how to serve the model using TensorFlow-Serving Container for online prediction.
 
 The steps performed include:
@@ -29,3 +36,6 @@ The steps performed include:
 - Deploy a Model and Create an Endpoint on Vertex AI
 - Predict on the Endpoint
 - Create a Batch Prediction Job on the Model
+
+```
+

notebooks/official/structured_data/README.md

@@ -1,6 +1,10 @@
 
-[BQML and AutoML - Experimenting with Vertex AI](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/master/notebooks/official/structured_data/rapid_prototyping_bqml_automl.ipynb)
+### structured_data 
 
+
+[BQML and AutoML - Experimenting with Vertex AI](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/structured_data/rapid_prototyping_bqml_automl.ipynb)
+
+```
 Learn how to use `Vertex AI Predictions` for rapid prototyping a model.
 
 The steps performed include:
@@ -12,17 +16,5 @@ The steps performed include:
 - Deploying the best trained model.
 - Testing the deployed model infrastructure.
 
-
-
-
-
-
-
-
-
-
-
-
-
-
+```
 

notebooks/official/tabnet/README.md

@@ -1,6 +1,10 @@
 
+### tabnet 
+
+
 [Vertex AI Explainations with TabNet models](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/tabnet/ai-explanations-tabnet-algorithm.ipynb)
 
+```
 Learn how to provide a sample plotting tool to visualize the output of TabNet, which is helpful in explaining the algorithm.
 
 The steps performed are:
@@ -9,8 +13,12 @@ The steps performed are:
 * Visualize and understand the feature importance based on the masks output.
 * Clean up the resource created by this tutorial.
 
+```
+
+
 [Vertex AI TabNet](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/tabnet/tabnet_vertex_tutorial.ipynb)
 
+```
 Learn how to run TabNet model on Vertex AI.
 
 The steps performed are:
@@ -20,3 +28,6 @@ The steps performed are:
 4. **Hyperparameter tuning**: Running a hyperparameter tuning job.
 5. **Hyperparameter on Vertex AI Training with BigQuery input**: Submitting a training job using BigQuery input.
 6. **Cleaning up**: Deleting resources created by this tutorial.
+
+```
+

notebooks/official/tabnet/ai-explanations-tabnet-algorithm.ipynb

@@ -76,6 +76,13 @@
         "\n",
         "In this tutorial, you learn how to provide a sample plotting tool to visualize the output of TabNet, which is helpful in explaining the algorithm. \n",
         "\n",
+        "\n",
+        "\n",
+        "This tutorial uses the following Google Cloud ML services and resources:\n",
+        "\n",
+        "- Vertex Explainable AI\n",
+        "- TabNet builtin algorithm\n",
+        "\n",
         "The steps performed are:\n",
         "* Setup the the project.\n",
         "* Download the prediction data of pretrain model onf Syn2 data.\n",

notebooks/official/tabnet/tabnet_vertex_tutorial.ipynb

@@ -78,6 +78,13 @@
         "\n",
         "In this notebook, you learn how to run TabNet model on Vertex AI.\n",
         "\n",
+        "This tutorial uses the following Google Cloud ML services and resources:\n",
+        "\n",
+        "- Vertex AI Training\n",
+        "- Vertex AI Hyperparameter Tuning\n",
+        "- TabNet builtin algorithm\n",
+        "- BigQuery\n",
+        "\n",
         "The steps performed are:\n",
         "1. **Setup**: Importing the required libraries and setting your global variables.\n",
         "2. **Configure parameters**: Setting the appropriate parameter values for the training job.\n",

notebooks/official/tabular_workflows/README.md

@@ -1,6 +1,10 @@
 
+### tabular_workflows 
+
+
 [TabNet Pipeline](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/tabular_workflows/tabnet_on_vertex_pipelines.ipynb)
 
+```
 Learn how to create two classification models using Vertex AI TabNet Tabular Workflows.
 
 The steps performed include:
@@ -8,10 +12,12 @@ The steps performed include:
 - Create a TabNet CustomJob. This is the best option if you know which hyperparameters to use for training.
 - Create a TabNet HyperparameterTuningJob. This allows you to get the best set of hyperparameters for your dataset.
 
+```
 
 
 [Wide & Deep Pipeline](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/tabular_workflows/wide_and_deep_on_vertex_pipelines.ipynb)
 
+```
 Learn how to create two classification models using Vertex AI Wide & Deep Tabular Workflows.
 
 The steps performed include:
@@ -19,4 +25,5 @@ The steps performed include:
 - Create a Wide & Deep CustomJob. This is the best option if you know which hyperparameters to use for training.
 - Create a Wide & Deep HyperparameterTuningJob. This allows you to get the best set of hyperparameters for your dataset.
 
+```
 

notebooks/official/tabular_workflows/tabnet_on_vertex_pipelines.ipynb

@@ -97,7 +97,7 @@
         "### Dataset\n",
         "\n",
         "The dataset you will be using is [Bank Marketing](https://archive.ics.uci.edu/ml/datasets/bank+marketing).\n",
-        "The data is for direct marketing campaigns (phone calls) of a Portuguese banking institution. The binary classification goal is to predict if a client subscribe a term deposit. For this notebook, you randomly selected 90% of the rows in the original dataset and saved them in a train.csv file hosted on Cloud Storage. To download the file, click [here](https://storage.googleapis.com/cloud-samples-data/vertex-ai/tabular-workflows/datasets/bank-marketing/train.csv)."
+        "The data is for direct marketing campaigns (phone calls) of a Portuguese banking institution. The binary classification goal is to predict if a client subscribe a term deposit. For this notebook, you randomly selected 90% of the rows in the original dataset and saved them in a train.csv file hosted on Cloud Storage. To download the file, click [here](https://storage.googleapis.com/cloud-samples-data-us-central1/vertex-ai/tabular-workflows/datasets/bank-marketing/train.csv)."
       ]
     },
     {
@@ -746,7 +746,7 @@
       },
       "outputs": [],
       "source": [
-        "data_source_csv_filenames = \"gs://cloud-samples-data/vertex-ai/tabular-workflows/datasets/bank-marketing/train.csv\"\n",
+        "data_source_csv_filenames = \"gs://cloud-samples-data-us-central1/vertex-ai/tabular-workflows/datasets/bank-marketing/train.csv\"\n",
         "data_source_bigquery_table_path = (\n",
         "    None  # @param {type:\"string\"}, format: bq://bq_project.bq_dataset.bq_table\n",
         ")"

notebooks/official/tabular_workflows/wide_and_deep_on_vertex_pipelines.ipynb

@@ -97,7 +97,7 @@
         "### Dataset\n",
         "\n",
         "The dataset you will be using is [Bank Marketing](https://archive.ics.uci.edu/ml/datasets/bank+marketing).\n",
-        "The data is for direct marketing campaigns (phone calls) of a Portuguese banking institution. The binary classification goal is to predict if a client will subscribe a term deposit. For this notebook, we randomly selected 90% of the rows in the original dataset and saved them in a train.csv file hosted on Cloud Storage. To download the file, click [here](https://storage.googleapis.com/cloud-samples-data/vertex-ai/tabular-workflows/datasets/bank-marketing/train.csv)."
+        "The data is for direct marketing campaigns (phone calls) of a Portuguese banking institution. The binary classification goal is to predict if a client will subscribe a term deposit. For this notebook, we randomly selected 90% of the rows in the original dataset and saved them in a train.csv file hosted on Cloud Storage. To download the file, click [here](https://storage.googleapis.com/cloud-samples-data-us-central1/vertex-ai/tabular-workflows/datasets/bank-marketing/train.csv)."
       ]
     },
     {
@@ -667,7 +667,7 @@
       },
       "outputs": [],
       "source": [
-        "data_source_csv_filenames = \"gs://cloud-samples-data/vertex-ai/tabular-workflows/datasets/bank-marketing/train.csv\"\n",
+        "data_source_csv_filenames = \"gs://cloud-samples-data-us-central1/vertex-ai/tabular-workflows/datasets/bank-marketing/train.csv\"\n",
         "data_source_bigquery_table_path = (\n",
         "    None  # @param {type:\"string\"}, format: bq://bq_project.bq_dataset.bq_table\n",
         ")"

notebooks/official/tensorboard/README.md

@@ -1,6 +1,10 @@
 
+### tensorboard 
+
+
 [Vertex AI TensorBoard custom training with prebuilt container](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/tensorboard/tensorboard_custom_training_with_prebuilt_container.ipynb)
 
+```
 Learn how to create a custom training job using prebuilt containers, and monitor your training process on Vertex AI TensorBoard in near real time.
 
 The steps performed include:
@@ -10,8 +14,26 @@ The steps performed include:
 * Package and upload your training code to Google Cloud Storage.
 * Create & launch your custom training job with Tensorboard enabled for near real time monitorning.
 
+```
+
+
+[Vertex AI TensorBoard integration with Vertex AI Pipelines](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/tensorboard/tensorboard_vertex_ai_pipelines_integration.ipynb)
+
+```
+Learn how to create a training pipeline using the KFP SDK, execute the pipeline in Vertex AI Pipelines, and monitor your training process on Vertex AI TensorBoard in near real time.
+
+The steps performed include:
+
+* Setup a service account and Google Cloud Storage buckets.
+* Construct a KFP pipeline with your custom training code.
+* Compile and execute the KFP pipeline in Vertex AI Pipelines with Tensorboard enabled for near real time monitorning.
+
+```
+
+
 [Vertex AI TensorBoard Custom Training with Custom Container](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/tensorboard/tensorboard_custom_training_with_custom_container.ipynb)
 
+```
 Learn how to create a custom training job using custom containers, and monitor your training process on Vertex AI TensorBoard in near real time.
 
 The steps performed include:
@@ -21,13 +43,20 @@ The steps performed include:
 * Setup service account and Google Cloud Storage buckets.
 * Create & launch your custom training job with your custom container.
 
+```
+
+
 [Profile model training performance using Profiler](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/tensorboard/tensorboard_profiler_custom_training.ipynb)
 
+```
 Learn how to enable Vertex AI TensorBoard Profiler for custom training jobs.
 
 The steps performed include:
 
-* Setup a service account and a Cloud Storage bucket
-* Create a TensorBoard instance
-* Create and run a custom training job that enables TensorBoard Profiler
-* View the TensorBoard Profiler dashboard to debug your model training performance
+- Setup a service account and a Cloud Storage bucket
+- Create a TensorBoard instance
+- Create and run a custom training job that enables TensorBoard Profiler
+- View the TensorBoard Profiler dashboard to debug your model training performance
+
+```
+

notebooks/official/tensorboard/tensorboard_custom_training_with_custom_container.ipynb

@@ -29,7 +29,7 @@
         "id": "ed2pOXQMb8fY"
       },
       "source": [
-        "# Vertex AI TensorBoard Custom Training with Custom Container\n",
+        "# Vertex AI TensorBoard Custom Training with custom container\n",
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
@@ -102,6 +102,11 @@
         "\n",
         "In this tutorial, you learn how to create a custom training job using custom containers, and monitor your training process on Vertex AI TensorBoard in near real time.\n",
         "\n",
+        "This tutorial uses the following Google Cloud ML services and resources:\n",
+        "\n",
+        "- Vertex AI Training\n",
+        "- Vertex AI TensorBoard\n",
+        "\n",
         "The steps performed include:\n",
         "\n",
         "* Create docker repository & config.\n",

notebooks/official/tensorboard/tensorboard_custom_training_with_prebuilt_container.ipynb

@@ -102,6 +102,11 @@
         "\n",
         "In this tutorial, you learn how to create a custom training job using prebuilt containers, and monitor your training process on Vertex AI TensorBoard in near real time.\n",
         "\n",
+        "This tutorial uses the following Google Cloud ML services and resources:\n",
+        "\n",
+        "- Vertex AI Training\n",
+        "- Vertex AI TensorBoard\n",
+        "\n",
         "The steps performed include:\n",
         "\n",
         "* Setup service account and Google Cloud Storage buckets.\n",

notebooks/official/tensorboard/tensorboard_profiler_custom_training.ipynb

@@ -29,7 +29,7 @@
         "id": "l2mMvIUG9meX"
       },
       "source": [
-        "# Profile model training performance using Profiler\n",
+        "# Profile model training performance using Vertex AI TensorBoard Profiler\n",
         "\n",
         "<table align=\"left\">\n",
         "\n",
@@ -49,7 +49,7 @@
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
-        "  </td>                                                                                               \n",
+        "  </td>\n",
         "</table>"
       ]
     },
@@ -331,22 +331,16 @@
         "! gcloud projects add-iam-policy-binding $PROJECT_ID \\\n",
         "        --member=\"serviceAccount:$SERVICE_ACCOUNT\" \\\n",
         "        --role=\"roles/storage.admin\" \\\n",
-        "        --quiet"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "S_8_5jm-Gk6w"
-      },
-      "outputs": [],
-      "source": [
+        "        --quiet\n",
+        "\n",
         "# Grant AI Platform permission.\n",
         "! gcloud projects add-iam-policy-binding $PROJECT_ID \\\n",
         "        --member=\"serviceAccount:$SERVICE_ACCOUNT\" \\\n",
         "        --role=\"roles/aiplatform.user\" \\\n",
-        "        --quiet"
+        "        --quiet\n",
+        "\n",
+        "! gcloud projects get-iam-policy $PROJECT_ID \\\n",
+        "        --filter=bindings.members:serviceAccount:$SERVICE_ACCOUNT"
       ]
     },
     {
@@ -768,7 +762,7 @@
         "WORKDIR /root\n",
         "\n",
         "# Installs additional packages as you need.\n",
-        "RUN pip3 install google-cloud-aiplatform[cloud_profiler]\n",
+        "RUN pip3 install google-cloud-aiplatform[cloud_profiler]>=1.19.1\n",
         "\n",
         "# Copies the trainer code to the docker image.\n",
         "RUN mkdir /root/trainer\n",
@@ -798,7 +792,7 @@
         "IMAGE_NAME = \"tensorboard-custom-container\"\n",
         "IMAGE_URI = f\"{REGION}-docker.pkg.dev/{PROJECT_ID}/{DOCKER_REPOSITORY}/{IMAGE_NAME}\"\n",
         "\n",
-        "! gcloud builds submit --project {PROJECT_ID} --region={REGION} --tag {IMAGE_URI} --timeout=60m --quiet"
+        "! gcloud builds submit --project {PROJECT_ID} --region={REGION} --tag {IMAGE_URI} --timeout=3600s --quiet"
       ]
     },
     {

notebooks/official/tensorboard/tensorboard_vertex_ai_pipelines_integration.ipynb

@@ -101,6 +101,12 @@
         "\n",
         "In this tutorial, you learn how to create a training pipeline using the KFP SDK, execute the pipeline in Vertex AI Pipelines, and monitor your training process on Vertex AI TensorBoard in near real time.\n",
         "\n",
+        "This tutorial uses the following Google Cloud ML services and resources:\n",
+        "\n",
+        "- Vertex AI Training\n",
+        "- Vertex AI TensorBoard\n",
+        "- Vertex AI Pipelines\n",
+        "\n",
         "The steps performed include:\n",
         "\n",
         "* Setup a service account and Google Cloud Storage buckets.\n",

notebooks/official/training/README.md

@@ -1,6 +1,24 @@
 
+### training 
+
+
+[Run hyperparameter tuning for a TensorFlow model](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/training/hyperparameter_tuning_tensorflow.ipynb)
+
+```
+Learn how to run a Vertex AI Hyperparameter Tuning job for a TensorFlow model.
+
+The steps performed include:
+
+* Modify training application code for automated hyperparameter tuning.
+* Containerize training application code.
+* Configure and launch a hyperparameter tuning job with the Vertex AI Python SDK.
+
+```
+
+
 [PyTorch image classification multi-node distributed data parallel training on cpu using Vertex training with custom container](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/training/multi_node_ddp_gloo_vertex_training_with_custom_container.ipynb)
 
+```
 Learn how to create a distributed PyTorch training job using Vertex AI SDK for Python and custom containers.
 
 The steps performed include:
@@ -11,8 +29,12 @@ The steps performed include:
 - Create a Vertex AI tensorboard instance to store your Vertex AI experiment
 - Run a Vertex AI SDK CustomContainerTrainingJob
 
+```
+
+
 [PyTorch Image Classification Multi-Node Distributed Data Parallel Training on GPU using Vertex AI Training with Custom Container](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/training/multi_node_ddp_nccl_vertex_training_with_custom_container.ipynb)
 
+```
 Learn how to create a distributed PyTorch training job using Vertex AI SDK for Python and custom containers.
 
 The steps performed include:
@@ -22,3 +44,21 @@ The steps performed include:
 - Building Custom Container using Artifact Registry and Docker
 - Create a Vertex AI Tensorboard Instance to store your Vertex AI experiment
 - Run a Vertex AI SDK CustomContainerTrainingJob
+
+```
+
+
+[Create a distributed custom training job](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/training/xgboost_data_parallel_training_on_cpu_using_dask.ipynb)
+
+```
+Learn how to create a distributed training job using Vertex AI SDK for Python.
+
+The steps performed include:
+
+- Configure the `PROJECT_ID` and `REGION` variables for your Google Cloud project.
+- Create a Cloud Storage bucket to store your model artifacts.
+- Build a custom Docker container that hosts your training code and push the container image to Artifact Registry.
+- Run a Vertex AI SDK CustomContainerTrainingJob
+
+```
+

notebooks/official/vizier/README.md

@@ -1,6 +1,13 @@
 
+### vizier 
+
+
 [Optimizing multiple objectives with Vertex AI Vizier](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/vizier/gapic-vizier-multi-objective-optimization.ipynb)
 
+```
 Learn how to use `Vertex AI Vizier` to optimize a multi-objective study.
 
 
+
+```
+

notebooks/official/workbench/README.md

@@ -0,0 +1,200 @@
+
+### sentiment_analysis 
+
+
+[Sentiment Analysis using AutoML Natural Language and Vertex AI](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/sentiment_analysis/Sentiment_Analysis.ipynb)
+
+```
+<a name="section-2"></a>
+
+
+
+```
+
+
+### exploratory_data_analysis 
+
+
+[Interactive exploratory analysis of BigQuery data in a notebook](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/exploratory_data_analysis/explore_data_in_bigquery_with_workbench.ipynb)
+
+```
+Learn about various ways to explore and gain insights from BigQuery data in a Jupyter notebook environment.
+
+The steps performed include:
+
+- Using Python & SQL to query public data in BigQuery
+- Exploring the dataset using BigQuery INFORMATION_SCHEMA
+- Creating interactive elements to help explore interesting parts of the data
+- Doing some exploratory correlation and time series analysis
+- Creating static and interactive outputs (data tables and plots) in the notebook
+- Saving some outputs to Cloud Storage
+
+```
+
+
+### demand_forecasting 
+
+
+[Forecasting retail demand with Vertex AI and BigQuery ML](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/demand_forecasting/forecasting-retail-demand.ipynb)
+
+```
+Learn how to build ARIMA (Autoregressive integrated moving average) model from BigQuery ML on retail data
+
+The steps performed include:
+
+* Explore data
+* Model with BigQuery and the ARIMA model
+* Evaluate the model
+* Evaluate the model results using BigQuery ML (on training data)
+* Evalute the model results - MAE, MAPE, MSE, RMSE (on test data)
+* Use the executor feature
+
+```
+
+
+### predictive_maintainance 
+
+
+[Predictive Maintenance using Vertex AI](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/predictive_maintainance/predictive_maintenance_usecase.ipynb)
+
+```
+<a name="section-2"></a>
+
+
+
+```
+
+
+### subscriber_churn_prediction 
+
+
+[Telecom subscriber churn prediction on Vertex AI](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/subscriber_churn_prediction/telecom-subscriber-churn-prediction.ipynb)
+
+```
+This tutorial shows you how to do exploratory data analysis, preprocess data, train, deploy and get predictions from a churn prediction model on a tabular churn dataset.
+
+The steps performed include:
+
+- Load data from a Cloud Storage path
+- Perform exploratory data analysis (EDA)
+- Preprocess the data
+- Train a scikit-learn model
+- Evaluate the scikit-learn model
+- Save the model to a Cloud Storage path
+- Create a model and an endpoint in Vertex AI
+- Deploy the trained model to an endpoint
+- Generate predictions and explanations on test data from the hosted model
+- Undeploy the model resource
+
+```
+
+
+### spark 
+
+
+[SparkML with Dataproc and BigQuery](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/spark/spark_ml.ipynb)
+
+```
+This tutorial runs an Apache SparkML job that fetches data from the BigQuery dataset, performs exploratory data analysis, cleans the data, executes feature engineering, trains the model, evaluates the model, outputs results, and saves the model to a Cloud Storage bucket.
+
+
+
+```
+
+
+[Digest and analyze data from BigQuery with Dataproc](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/spark/spark_bigquery.ipynb)
+
+```
+This notebook tutorial runs an Apache Spark job that fetches data from the BigQuery "GitHub Activity Data" dataset, queries the data, and then writes the results back to BigQuery.
+
+
+
+```
+
+
+### ads_targetting 
+
+
+[Train a multi-class classification model for ads-targeting](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/ads_targetting/training-multi-class-classification-model-for-ads-targeting-usecase.ipynb)
+
+```
+Learn how to collect data from BigQuery, preprocess it, and train a multi-class classification model on an e-commerce dataset.
+
+The steps performed include:
+
+- Fetch the required data from BigQuery
+- Preprocess the data
+- Train a TensorFlow (>=2.4) classification model
+- Evaluate the loss for the trained model
+- Automate the notebook execution using the executor feature
+- Save the model to a Cloud Storage path
+- Clean up the created resources
+
+```
+
+
+### inventory-prediction 
+
+
+[Inventory prediction on ecommerce data using Vertex AI](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/inventory-prediction/inventory_prediction.ipynb)
+
+```
+This tutorial shows you how to do exploratory data analysis, preprocess data, train model, evaluate model, deploy model, configure What-If Tool.
+
+The steps performed include:
+
+* Load the dataset from BigQuery using the "BigQuery in Notebooks" integration.
+* Analyze the dataset.
+* Preprocess the features in the dataset.
+* Build a random forest classifier model that predicts whether a product will get sold in the next 60 days.
+* Evaluate the model.
+* Deploy the model using Vertex AI.
+* Configure and test with the What-If Tool.
+
+```
+
+
+### fraud_detection 
+
+
+[Build a fraud detection model on Vertex AI](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/fraud_detection/fraud-detection-model.ipynb)
+
+```
+This tutorial demonstrates data analysis and model-building using a synthetic financial dataset.
+
+The steps performed include:
+
+- Installation of required libraries
+- Reading the dataset from a Cloud Storage bucket
+- Performing exploratory analysis on the dataset
+- Preprocessing the dataset
+- Training a random forest model using scikit-learn
+- Saving the model to a Cloud Storage bucket
+- Creating a Vertex AI model resource and deploying to an endpoint
+- Running the What-If Tool on test data
+- Un-deploying the model and cleaning up the model resources
+
+```
+
+
+### chicago_taxi_fare_prediction 
+
+
+[Taxi fare prediction using the Chicago Taxi Trips dataset](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/chicago_taxi_fare_prediction/chicago_taxi_fare_prediction.ipynb)
+
+```
+The goal of this notebook is to provide an overview on the latest Vertex AI features like **Explainable AI** and **BigQuery in Notebooks** by trying to solve a taxi fare prediction problem.
+
+The steps performed include:
+
+- Loading the dataset using "BigQuery in Notebooks".
+- Performing exploratory data analysis on the dataset.
+- Feature selection and preprocessing.
+- Building a linear regression model using scikit-learn.
+- Configuring the model for Vertex Explainable AI.
+- Deploying the model to Vertex AI.
+- Testing the deployed model.
+- Clean up.
+
+```
+

notebooks/official/workbench/ads_targetting/training-multi-class-classification-model-for-ads-targeting-usecase.ipynb

@@ -44,7 +44,7 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/ads_targetting/training-multi-class-classification-model-for-ads-targeting-usecase.ipynb\">\n",
+        "<a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/workbench/ads_targetting/training-multi-class-classification-model-for-ads-targeting-usecase.ipynb\" target='_blank'>\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",

notebooks/official/workbench/chicago_taxi_fare_prediction/chicago_taxi_fare_prediction.ipynb

@@ -45,7 +45,7 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/chicago_taxi_fare_prediction/chicago_taxi_fare_prediction.ipynb\">\n",
+        "<a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/workbench/chicago_taxi_fare_prediction/chicago_taxi_fare_prediction.ipynb\" target='_blank'>\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",

notebooks/official/workbench/demand_forecasting/forecasting-retail-demand.ipynb

@@ -44,7 +44,7 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/demand_forecasting/forecasting-retail-demand.ipynb\">\n",
+        "<a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/workbench/demand_forecasting/forecasting-retail-demand.ipynb\" target='_blank'>\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",

notebooks/official/workbench/exploratory_data_analysis/explore_data_in_bigquery_with_workbench.ipynb

@@ -56,13 +56,20 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "tvgnzT1CKxrO"
+        "id": "780762457db0"
       },
       "source": [
         "## Overview\n",
         "\n",
-        "This notebook is written for data analysts and data scientists who have data in BigQuery and want to perform exploratory data analysis to gather insights from that data in an interactive environment.\n",
-        "\n",
+        "This notebook is written for data analysts and data scientists who have data in BigQuery and want to perform exploratory data analysis to gather insights from that data in an interactive environment."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tvgnzT1CKxrO"
+      },
+      "source": [
         "### Objective\n",
         "\n",
         "In this tutorial, you learn about various ways to explore and gain insights from BigQuery data in a Jupyter notebook environment.\n",
@@ -80,12 +87,26 @@
         "- Creating interactive elements to help explore interesting parts of the data\n",
         "- Doing some exploratory correlation and time series analysis\n",
         "- Creating static and interactive outputs (data tables and plots) in the notebook\n",
-        "- Saving some outputs to Cloud Storage\n",
-        "\n",
+        "- Saving some outputs to Cloud Storage"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "65aa4ba05101"
+      },
+      "source": [
         "### Dataset\n",
         "\n",
-        "The dataset, [available publicly on BigQuery](https://console.cloud.google.com/bigquery?project=bigquery-public-data&d=ga4_obfuscated_sample_ecommerce&p=bigquery-public-data&page=dataset), comes from obfuscated [Google Analytics 4 (GA4) data](https://developers.google.com/analytics/bigquery/web-ecommerce-demo-dataset) from the Google Merchandise Store.\n",
-        "\n",
+        "The dataset, [available publicly on BigQuery](https://console.cloud.google.com/bigquery?project=bigquery-public-data&d=ga4_obfuscated_sample_ecommerce&p=bigquery-public-data&page=dataset), comes from obfuscated [Google Analytics 4 (GA4) data](https://developers.google.com/analytics/bigquery/web-ecommerce-demo-dataset) from the Google Merchandise Store."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "fd44a67414d5"
+      },
+      "source": [
         "### Costs \n",
         "\n",
         "This tutorial uses billable components of Google Cloud:\n",

notebooks/official/workbench/fraud_detection/fraud-detection-model.ipynb

@@ -45,7 +45,7 @@
         "    </a>\n",
         "  </td>\n",
         "   <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/fraud_detection/fraud-detection-model.ipynb\">\n",
+        "<a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/workbench/fraud_detection/fraud-detection-model.ipynb\" target='_blank'>\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",

notebooks/official/workbench/gaming_churn_prediction/churn_prediction_for_game_developers.ipynb

@@ -29,23 +29,23 @@
         "id": "py8EYwG_91Pn"
       },
       "source": [
-        "# Churn prediction for game developers using Google Analytics 4 (GA4) and BigQuery ML\n",
+        "# Churn prediction for game developers using Google Analytics 4 and BigQuery ML\n",
         "\n",
         "<table align=\"left\">\n",
         "\n",
         "  <td>\n",
-        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/gaming_churn_prediction/churn-prediction-for-game-developers.ipynb\">\n",
+        "<a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/gaming_churn_prediction/churn_prediction_for_game_developers.ipynb\" target='_blank'>\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/gaming_churn_prediction/churn-prediction-for-game-developers.ipynb\">\n",
+        "<a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/gaming_churn_prediction/churn_prediction_for_game_developers.ipynb\" target='_blank'>\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/gaming_churn_prediction/churn-prediction-for-game-developers.ipynb\">\n",
+        "<a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/workbench/gaming_churn_prediction/churn_prediction_for_game_developers.ipynb\" target='_blank'>\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
@@ -99,7 +99,7 @@
         "id": "sH0CZGku0BPp"
       },
       "source": [
-        "## Objective\n",
+        "### Objective\n",
         "<a name=\"section-2\"></a>\n",
         "\n",
         "In this tutorial, you learn how to train, evaluate a propensity model in BigQuery ML.\n",
@@ -122,7 +122,7 @@
         "id": "b07fc9940120"
       },
       "source": [
-        "## Dataset\n",
+        "### Dataset\n",
         "<a name=\"section-3\"></a>\n",
         "\n",
         "This notebook uses [this public BigQuery dataset](https://console.cloud.google.com/bigquery?p=firebase-public-project&d=analytics_153293282&t=events_20181003&page=table), which contains raw event data from a real mobile gaming app called Flood It! ([Android app](https://play.google.com/store/apps/details?id=com.labpixies.flood), [iOS app](https://itunes.apple.com/us/app/flood-it!/id476943146?mt=8)). The [data schema](https://support.google.com/analytics/answer/7029846) originates from Google Analytics for Firebase, but is the same schema as [Google Analytics 4](https://support.google.com/analytics/answer/9358801); the techniques in this notebook can be applied to either Google Analytics for Firebase or Google Analytics 4 data.\n",
@@ -138,7 +138,7 @@
         "id": "589ffe790261"
       },
       "source": [
-        "## Costs\n",
+        "### Costs\n",
         "<a name=\"section-4\"></a>\n",
         "\n",
         "This tutorial uses the following billable components of Google Cloud:\n",

notebooks/official/workbench/inventory-prediction/inventory_prediction.ipynb

@@ -46,7 +46,7 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/inventory-prediction/inventory_prediction.ipynb\">\n",
+        "<a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/workbench/inventory-prediction/inventory_prediction.ipynb\" target='_blank'>\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",

notebooks/official/workbench/predictive_maintainance/predictive_maintenance_usecase.ipynb

@@ -44,7 +44,7 @@
         "        </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/predictive_maintainance/predictive_maintenance_usecase.ipynb\">\n",
+        "<a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/workbench/predictive_maintainance/predictive_maintenance_usecase.ipynb\" target='_blank'>\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
@@ -93,7 +93,12 @@
         "### Objective\n",
         "<a name=\"section-2\"></a>\n",
         "\n",
-        "The objectives of this notebook include:\n",
+        "This tutorial uses the following Google Cloud ML services:\n",
+        "\n",
+        "- `Vertex AI Training`\n",
+        "- `Vertex AI Model Evaluation`\n",
+        "\n",
+        "The steps performed are:\n",
         "\n",
         "- Loading the required dataset from a Cloud Storage bucket.\n",
         "- Analyzing the fields present in the dataset.\n",

notebooks/official/workbench/pricing_optimization/pricing-optimization.ipynb

@@ -94,7 +94,7 @@
         "id": "71f69cfdff2b"
       },
       "source": [
-        "## Objective\n",
+        "### Objective\n",
         "<a name=\"section-2\"></a>\n",
         "\n",
         "The objective of this notebook is to build a pricing optimization model using BigQuery ML. The following steps have been followed:  \n",
@@ -122,7 +122,7 @@
         "id": "d20422a5c34d"
       },
       "source": [
-        "## Dataset\n",
+        "### Dataset\n",
         "<a name=\"section-3\"></a>\n",
         "\n",
         "The dataset used in this notebook is a part of the [CDM Pricing dataset](https://github.com/trifacta/trifacta-google-cloud/blob/main/design-pattern-pricing-optimization/CDM_Pricing_large_table.csv), which consists of product sales information on specified dates."
@@ -134,7 +134,7 @@
         "id": "c05bcd30859d"
       },
       "source": [
-        "## Costs\n",
+        "### Costs\n",
         "<a name=\"section-4\"></a>\n",
         "\n",
         "This tutorial uses the following billable components of Google Cloud:\n",

notebooks/official/workbench/sentiment_analysis/Sentiment_Analysis.ipynb

@@ -45,7 +45,7 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/sentiment_analysis/Sentiment_Analysis.ipynb\">\n",
+        "<a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/workbench/sentiment_analysis/Sentiment_Analysis.ipynb\" target='_blank'>\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
@@ -97,13 +97,20 @@
         "### Objective\n",
         "<a name=\"section-2\"></a>\n",
         "\n",
-        "The objectives of this notebook include:\n",
+        "This tutorial uses the following Google Cloud ML services:\n",
+        "\n",
+        "- `Vertex AI Dataset`\n",
+        "- `Vertex AI Model`\n",
+        "- `AutoML training`\n",
+        "- `Vertex AI Prediction`\n",
+        "\n",
+        "The steps performed are:\n",
         "\n",
         "- Loading the required data. \n",
         "- Preprocessing the data.\n",
         "- Selecting the required data for the model.\n",
         "- Loading the dataset into Vertex AI managed datasets.\n",
-        "- Training a sentiment model using AutoML Natural Language.\n",
+        "- Training a sentiment model using AutoML Text training.\n",
         "- Evaluating the model.\n",
         "- Deploying the model on Vertex AI.\n",
         "- Getting predictions.\n",

notebooks/official/workbench/spark/spark_bigquery.ipynb

@@ -29,16 +29,23 @@
         "id": "JAPoU8Sm5E6e"
       },
       "source": [
+        "# Digest and analyze data from BigQuery with Dataproc\n",
+        "\n",
         "<table align=\"left\">\n",
         "\n",
         "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/spark/spark_sample_notebook.ipynb\">\n",
+        "<a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/spark/spark_bigquery.ipynb\" target='_blank'>\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-managed-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/workbench/spark/spark_bigquery.ipynb\">\n",
+        "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/spark/spark_bigquery.ipynb\">\n",
+        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "        </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "<a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/workbench/spark/spark_bigquery.ipynb\" target='_blank'>\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
@@ -59,17 +66,6 @@
         "To run this notebook, click the link `Open in Vertex AI Workbench` above."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "tvgnzT1CKxrO"
-      },
-      "source": [
-        "### Dataset\n",
-        "\n",
-        "The [GitHub Activity Data](https://console.cloud.google.com/marketplace/product/github/github-repos) dataset is available in [BigQuery Public Datasets](https://cloud.google.com/bigquery/public-data), and provides free querying of up to 1TB of data each month. It contains data on two different types of repositories: \"polyglot\" repos, which support multiple programming language files, and \"monoglot\" repos, which support one programming language."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -80,7 +76,12 @@
         "\n",
         "This notebook tutorial runs an Apache Spark job that fetches data from the BigQuery \"GitHub Activity Data\" dataset, queries the data, and then writes the results back to BigQuery. This job sequence represents a common data engineering use case: ingesting, transforming, and querying data, and then writing the output to a database. It also demonstrates how to submit an Apache Spark job to Dataproc.\n",
         "\n",
-        "This notebook tutorial performs the following steps:\n",
+        "This tutorial uses the following Google Cloud ML services:\n",
+        "\n",
+        "- `Dataproc`\n",
+        "- `BigQuery`\n",
+        "\n",
+        "The steps performed are:\n",
         "\n",
         "- Setting up a Google Cloud project and Dataproc cluster.\n",
         "- Configuring the spark-bigquery-connector.\n",
@@ -93,6 +94,17 @@
         "- Deleting the resources created for this notebook tutorial."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tvgnzT1CKxrO"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The [GitHub Activity Data](https://console.cloud.google.com/marketplace/product/github/github-repos) dataset is available in [BigQuery Public Datasets](https://cloud.google.com/bigquery/public-data), and provides free querying of up to 1TB of data each month. It contains data on two different types of repositories: \"polyglot\" repos, which support multiple programming language files, and \"monoglot\" repos, which support one programming language."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {

notebooks/official/workbench/spark/spark_ml.ipynb

@@ -29,16 +29,23 @@
         "id": "XoEqT2Y4DJmf"
       },
       "source": [
+        "# SparkML with Dataproc and BigQuery\n",
+        "\n",
         "<table align=\"left\">\n",
         "\n",
         "  <td>\n",
-        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/notebook_template.ipynb\">\n",
+        "<a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/spark/spark_ml.ipynb\" target='_blank'>\n",
         "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
         "      View on GitHub\n",
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-managed-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/workbench/spark/spark_ml.ipynb\">\n",
+        "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/spark/spark_ml.ipynb\">\n",
+        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "        </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "<a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/workbench/spark/spark_ml.ipynb\" target='_blank'>\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
@@ -57,17 +64,6 @@
         "This notebook tutorial runs Apache SparkML jobs with Dataproc and BigQuery to exemplify a common machine learning pipeline use case: data ingestion and cleaning, feature engineering, modeling, and model evaluation."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "XoEqT2Y4DJmf"
-      },
-      "source": [
-        "### Dataset\n",
-        "\n",
-        "The [NYC TLC (Taxi and Limousine Commission) Trips](https://console.cloud.google.com/marketplace/product/city-of-new-york/nyc-tlc-trips) (New York taxi and limosine trips data) and [NYC Citi Bike Trips](https://console.cloud.google.com/marketplace/product/city-of-new-york/nyc-citi-bike) (NYC public bicycle sharing system data) datasets are available in [BigQuery Public Datasets](https://cloud.google.com/bigquery/public-data). BigQuery provides free querying of up to 1TB of data each month."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -78,7 +74,13 @@
         "\n",
         "This tutorial runs an Apache SparkML job that fetches data from the BigQuery dataset, performs exploratory data analysis, cleans the data, executes feature engineering, trains the model, evaluates the model, outputs results, and saves the model to a Cloud Storage bucket.\n",
         "\n",
-        "This notebook tutorial performs the following steps:\n",
+        "This tutorial uses the following Google Cloud ML services:\n",
+        "\n",
+        "- `Dataproc`\n",
+        "- `BigQuery`\n",
+        "- `Vertex AI Training`\n",
+        "\n",
+        "The steps performed are:\n",
         "\n",
         "- Sets up a Google Cloud project and Dataproc cluster.\n",
         "- Creates a Cloud Storage bucket and a BigQuery dataset.\n",
@@ -94,6 +96,17 @@
         "- Deletes the resources created for the tutorial."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "XoEqT2Y4DJmf"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The [NYC TLC (Taxi and Limousine Commission) Trips](https://console.cloud.google.com/marketplace/product/city-of-new-york/nyc-tlc-trips) (New York taxi and limosine trips data) and [NYC Citi Bike Trips](https://console.cloud.google.com/marketplace/product/city-of-new-york/nyc-citi-bike) (NYC public bicycle sharing system data) datasets are available in [BigQuery Public Datasets](https://cloud.google.com/bigquery/public-data). BigQuery provides free querying of up to 1TB of data each month."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {

notebooks/official/workbench/subscriber_churn_prediction/telecom-subscriber-churn-prediction.ipynb

@@ -44,7 +44,7 @@
         "        </a>\n",
         "  </td>\n",
         "  <td>\n",
-        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/subscriber_churn_prediction/telecom-subscriber-churn-prediction.ipynb\">\n",
+        "<a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/workbench/subscriber_churn_prediction/telecom-subscriber-churn-prediction.ipynb\" target='_blank'>\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",