debug: force use of newest cloud-build

* Use sample dataset from regional bucket

* retrigger checks

community-content/pytorch_efficient_training/README.md

@@ -15,15 +15,19 @@ pip install -r requirements.txt
 * resnet_dp.py - Train ResNet-50 on single node multiple GPUs with `DataParallel` strategy.
 * resnet_ddp.py - Train ResNet-50 on single node multiple GPUs with `DistributedDataParallel` strategy.
 * resnet_ddp_wds.py - Train ResNet-50 on single node multiple GPUs with `DistributedDataParallel` strategy and `Webdataset`.
+* resnet_fsdp.py - Train ResNet-50 on single node multiple GPUs with `FullyShardedDataParallel` strategy.
+* resnet_fsdp_wds.py - Train ResNet-50 on single node multiple GPUs with `FullyShardedDataParallel` strategy and `Webdataset`.
 * shard_imagenet.py - Shard ImagNet individual files into `tar` files.
 
 ## Benchmark
 
 When run the benchmark on Nvidia T4 GPUs using ImageNet validation dataset, you can get the result like:
-Strategy              | Seconds/Epoch - Local Data | Seconds/Epoch - Cloud Data
---------------------- | -------------------------- | --------------------------
-On 1 GPU              | 489                        | 804 (2x slower)
-On 4 GPUs (DP)        | 157                        | 738 (5x slower)
-On 4 GPUs (DDP)       | 134                        | 432 (3x slower)
-On 4 GPUs (DDP + WDS) | 131                        | 133 (same performance)
+Strategy               | Seconds/Epoch - Local Data | Seconds/Epoch - Cloud Data
+---------------------- | -------------------------- | --------------------------
+On 1 GPU               | 489                        | 804 (2x slower)
+On 4 GPUs (DP)         | 157                        | 738 (5x slower)
+On 4 GPUs (DDP)        | 134                        | 432 (3x slower)
+On 4 GPUs (DDP + WDS)  | 131                        | 133 (same performance)
+On 4 GPUs (FSDP)       | 139                        | 353 (3x slower)
+On 4 GPUs (FSDP + WDS) | 138                        | 135 (same performance)
 

community-content/pytorch_efficient_training/resnet_fsdp.py

@@ -0,0 +1,242 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the \"License\");
+# you may not use this file except in compliance with the License.\n",
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an \"AS IS\" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Train resnet on multiple GPUs with FSDP."""
+
+import argparse
+import functools
+import os
+import time
+
+from PIL import Image
+import torch
+from torch import nn
+import torch.distributed as dist
+from torch.distributed.fsdp import FullyShardedDataParallel as FSDP
+from torch.distributed.fsdp.wrap import size_based_auto_wrap_policy
+import torch.multiprocessing as mp
+import torchmetrics
+import torchvision
+from torchvision.models import resnet50
+
+
+class ImageFolder(torchvision.datasets.ImageFolder):
+  """Class for loading imagenet."""
+
+  def __init__(self, image_list_file, transform=None, target_transform=None):
+    self.samples = self._make_dataset(image_list_file)
+    self.loader = self._loader
+
+    self.imgs = self.samples
+    self.targets = [s[1] for s in self.samples]
+
+    self.transform = transform
+    self.target_transform = target_transform
+
+  def _make_dataset(self, image_list_file):
+    items = []
+    with open(image_list_file, 'r') as f:
+      for line in f:
+        item = line.strip().split(' ')
+        items.append((item[0], int(item[1])))
+    return items
+
+  def _loader(self, image_path):
+    with open(image_path, 'rb') as f:
+      img = Image.open(f)
+      img = img.convert('RGB')
+      return img
+
+
+def train(model, device, dataloader, optimizer):
+  model.train()
+  for image, target in dataloader:
+    image = image.to(device, non_blocking=True)
+    target = target.to(device, non_blocking=True)
+    pred = model(image)
+    # pred.shape (N, C), target.shape (N)
+    loss = nn.functional.cross_entropy(pred, target)
+    optimizer.zero_grad()
+    loss.backward()
+    optimizer.step()
+  return loss
+
+
+def evaluate(model, device, dataloader, metric):
+  model.eval()
+  with torch.no_grad():
+    for image, target in dataloader:
+      image = image.to(device, non_blocking=True)
+      target = target.to(device, non_blocking=True)
+      pred = model(image)
+      metric.update(pred, target)
+  accuracy = metric.compute()
+  metric.reset()
+  return accuracy
+
+
+def worker(gpu, args):
+  """Run training and evaluation."""
+  # Init process group.
+  print(f'Initiating process {gpu}')
+  dist.init_process_group(
+      backend='nccl',
+      init_method='env://',
+      world_size=args.gpus,
+      rank=gpu)
+
+  # Create train dataloader.
+  train_dataset = ImageFolder(
+      image_list_file=args.train_data_path,
+      transform=torchvision.transforms.Compose([
+          torchvision.transforms.RandomResizedCrop(224),
+          torchvision.transforms.RandomHorizontalFlip(),
+          torchvision.transforms.ToTensor(),
+          torchvision.transforms.Normalize(
+              mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
+      ]))
+  train_sampler = torch.utils.data.distributed.DistributedSampler(
+      train_dataset, num_replicas=args.gpus, rank=gpu)
+  train_dataloader = torch.utils.data.DataLoader(
+      dataset=train_dataset,
+      batch_size=args.train_batch_size,
+      shuffle=False,
+      num_workers=args.dataloader_num_workers,
+      pin_memory=True,
+      sampler=train_sampler)
+  if gpu == 0:
+    print(f'Train dataloader | samples: {len(train_dataloader.dataset)}, '
+          f'num workers: {train_dataloader.num_workers}, '
+          f'global batch size: {args.train_batch_size * args.gpus}, '
+          f'batches/epoch: {len(train_dataloader)}')
+
+  # Create eval dataloader.
+  eval_dataset = ImageFolder(
+      image_list_file=args.eval_data_path,
+      transform=torchvision.transforms.Compose([
+          torchvision.transforms.Resize(256),
+          torchvision.transforms.CenterCrop(224),
+          torchvision.transforms.ToTensor(),
+          torchvision.transforms.Normalize(
+              mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
+      ]))
+  eval_sampler = torch.utils.data.distributed.DistributedSampler(
+      eval_dataset, num_replicas=args.gpus, rank=gpu)
+  eval_dataloader = torch.utils.data.DataLoader(
+      dataset=eval_dataset,
+      batch_size=args.eval_batch_size,
+      shuffle=False,
+      num_workers=args.dataloader_num_workers,
+      pin_memory=True,
+      drop_last=True,
+      sampler=eval_sampler)
+  if gpu == 0:
+    print(f'Eval dataloader | samples: {len(eval_dataloader.dataset)}, '
+          f'num workers: {eval_dataloader.num_workers}, '
+          f'batch size: {args.eval_batch_size}, '
+          f'batches/epoch: {len(eval_dataloader)}')
+
+  # Wrap policy.
+  my_auto_wrap_policy = functools.partial(
+      size_based_auto_wrap_policy, min_num_params=100)
+  torch.cuda.set_device(gpu)
+
+  # Create model.
+  model = resnet50(weights=None)
+  model.to(args.device)
+  model = FSDP(model, auto_wrap_policy=my_auto_wrap_policy)
+
+  # Optimizer.
+  optimizer = torch.optim.SGD(model.parameters(), 0.1)
+
+  # Main loop.
+  metric = torchmetrics.classification.Accuracy(top_k=1).to(args.device)
+  for epoch in range(1, args.epochs + 1):
+    if gpu == 0:
+      print(f'Running epoch {epoch}')
+    train_sampler.set_epoch(epoch)
+
+    start = time.time()
+    train(model, args.device, train_dataloader, optimizer)
+    end = time.time()
+    if gpu == 0:
+      print(f'Training finished in {(end - start):>0.3f} seconds')
+
+    start = time.time()
+    evaluate(model, args.device, eval_dataloader, metric)
+    end = time.time()
+    if gpu == 0:
+      print(f'Evaluation finished in {(end - start):>0.3f} seconds')
+
+  if gpu == 0:
+    print('Done')
+  dist.destroy_process_group()
+
+
+def create_args():
+  """Create main args."""
+  parser = argparse.ArgumentParser(
+      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+  parser.add_argument(
+      '--gpus',
+      default=4,
+      type=int,
+      help='number of gpus to use')
+  parser.add_argument(
+      '--epochs',
+      default=2,
+      type=int,
+      help='number of total epochs to run')
+  parser.add_argument(
+      '--dataloader_num_workers',
+      default=2,
+      type=int,
+      help='number of workders for dataloader')
+  parser.add_argument(
+      '--train_data_path',
+      default='',
+      type=str,
+      help='path to training data')
+  parser.add_argument(
+      '--train_batch_size',
+      default=32,
+      type=int,
+      help='batch size for training per gpu')
+  parser.add_argument(
+      '--eval_data_path',
+      default='',
+      type=str,
+      help='path to evaluation data')
+  parser.add_argument(
+      '--eval_batch_size',
+      default=32,
+      type=int,
+      help='batch size for evaluation per gpu')
+  args = parser.parse_args()
+  return args
+
+
+def main():
+  args = create_args()
+
+  os.environ['MASTER_ADDR'] = 'localhost'
+  os.environ['MASTER_PORT'] = '8888'
+
+  args.device = 'cuda' if torch.cuda.is_available() else 'cpu'
+  print(f'Launch job on {args.gpus} GPUs with FSDP')
+  mp.spawn(worker, nprocs=args.gpus, args=(args,))
+
+
+if __name__ == '__main__':
+  main()

community-content/pytorch_efficient_training/resnet_fsdp_wds.py

@@ -0,0 +1,240 @@
+"""Train resnet on multiple GPUs with DDP."""
+
+import argparse
+import functools
+import itertools
+import math
+import os
+import time
+
+import torch
+from torch import nn
+import torch.distributed as dist
+from torch.distributed.fsdp import FullyShardedDataParallel as FSDP
+from torch.distributed.fsdp.wrap import size_based_auto_wrap_policy
+import torch.multiprocessing as mp
+import torchmetrics
+from torchvision.models import resnet50
+from torchvision.transforms import transforms
+import webdataset as wds
+
+
+def wds_split(src, rank, world_size):
+  """Shards split function for webdataset."""
+  # The context of caller of this function is within multiple processes
+  # (by DDP world_size) and multiple workers (by dataloader_num_workers).
+  # So we totally have (world_size * num_workers) workers for processing data.
+  # NOTE: Raw data should be sharded to enough shards to make sure one process
+  # can handle at least one shard, otherwise the process may hang.
+  worker_id = 0
+  num_workers = 1
+  worker_info = torch.utils.data.get_worker_info()
+  if worker_info:
+    worker_id = worker_info.id
+    num_workers = worker_info.num_workers
+  for s in itertools.islice(src, rank * num_workers + worker_id, None,
+                            world_size * num_workers):
+    yield s
+
+
+def identity(x):
+  return x
+
+
+def create_wds_dataloader(rank, args, mode):
+  """Create webdataset dataset and dataloader."""
+  if mode == 'train':
+    transform = transforms.Compose([
+        transforms.RandomResizedCrop(224),
+        transforms.RandomHorizontalFlip(),
+        transforms.ToTensor(),
+        transforms.Normalize(
+            mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
+    ])
+    data_path = args.train_data_path
+    data_size = args.train_data_size
+    batch_size_local = args.train_batch_size
+    batch_size_global = args.train_batch_size * args.gpus
+    # Since webdataset disallows partial batch, we pad the last batch for train.
+    batches = int(math.ceil(data_size / batch_size_global))
+  else:
+    transform = transforms.Compose([
+        transforms.Resize(256),
+        transforms.CenterCrop(224),
+        transforms.ToTensor(),
+        transforms.Normalize(
+            mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
+    ])
+    data_path = args.eval_data_path
+    data_size = args.eval_data_size
+    batch_size_local = args.eval_batch_size
+    batch_size_global = args.eval_batch_size * args.gpus
+    # Since webdataset disallows partial batch, we drop the last batch for eval.
+    batches = int(data_size / batch_size_global)
+
+  dataset = wds.DataPipeline(
+      wds.SimpleShardList(data_path),
+      functools.partial(wds_split, rank=rank, world_size=args.gpus),
+      wds.tarfile_to_samples(),
+      wds.decode('pil'),
+      wds.to_tuple('jpg;png;jpeg cls'),
+      wds.map_tuple(transform, identity),
+      wds.batched(batch_size_local, partial=False),
+  )
+  num_workers = args.dataloader_num_workers
+  dataloader = wds.WebLoader(
+      dataset=dataset,
+      batch_size=None,
+      shuffle=False,
+      num_workers=num_workers,
+      persistent_workers=True if num_workers > 0 else False,
+      pin_memory=True).repeat(nbatches=batches)
+  print(f'{mode} dataloader | samples: {data_size}, '
+        f'num_workers: {num_workers}, '
+        f'local batch size: {batch_size_local}, '
+        f'global batch size: {batch_size_global}, '
+        f'batches: {batches}')
+  return dataloader
+
+
+def train(model, device, dataloader, optimizer):
+  model.train()
+  for image, target in dataloader:
+    image = image.to(device, non_blocking=True)
+    target = target.to(device, non_blocking=True)
+    pred = model(image)
+    # pred.shape (N, C), target.shape (N)
+    loss = nn.functional.cross_entropy(pred, target)
+    optimizer.zero_grad()
+    loss.backward()
+    optimizer.step()
+  return loss
+
+
+def evaluate(model, device, dataloader, metric):
+  model.eval()
+  with torch.no_grad():
+    for image, target in dataloader:
+      image = image.to(device, non_blocking=True)
+      target = target.to(device, non_blocking=True)
+      pred = model(image)
+      metric.update(pred, target)
+  accuracy = metric.compute()
+  metric.reset()
+  return accuracy
+
+
+def worker(gpu, args):
+  """Run training and evaluation."""
+  # Init process group.
+  print(f'Initiating process {gpu}')
+  dist.init_process_group(
+      backend='nccl',
+      init_method='env://',
+      world_size=args.gpus,
+      rank=gpu)
+
+  # Create dataloader.
+  train_dataloader = create_wds_dataloader(gpu, args, 'train')
+  eval_dataloader = create_wds_dataloader(gpu, args, 'eval')
+
+  # Wrap policy.
+  my_auto_wrap_policy = functools.partial(
+      size_based_auto_wrap_policy, min_num_params=100)
+  torch.cuda.set_device(gpu)
+
+  # Create model.
+  model = resnet50(weights=None)
+  model.to(args.device)
+  model = FSDP(model, auto_wrap_policy=my_auto_wrap_policy)
+
+  # Optimizer.
+  optimizer = torch.optim.SGD(model.parameters(), 0.1)
+
+  # Main loop.
+  metric = torchmetrics.classification.Accuracy(top_k=1).to(args.device)
+  for epoch in range(1, args.epochs + 1):
+    if gpu == 0:
+      print(f'Running epoch {epoch}')
+
+    start = time.time()
+    train(model, args.device, train_dataloader, optimizer)
+    end = time.time()
+    if gpu == 0:
+      print(f'Training finished in {(end - start):>0.3f} seconds')
+
+    start = time.time()
+    evaluate(model, args.device, eval_dataloader, metric)
+    end = time.time()
+    if gpu == 0:
+      print(f'Evaluation finished in {(end - start):>0.3f} seconds')
+
+  if gpu == 0:
+    print('Done')
+
+
+def create_args():
+  """Create main args."""
+  parser = argparse.ArgumentParser(
+      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+  parser.add_argument(
+      '--gpus',
+      default=4,
+      type=int,
+      help='number of gpus to use')
+  parser.add_argument(
+      '--epochs',
+      default=2,
+      type=int,
+      help='number of total epochs to run')
+  parser.add_argument(
+      '--dataloader_num_workers',
+      default=2,
+      type=int,
+      help='number of workders for dataloader')
+  parser.add_argument(
+      '--train_data_path',
+      default='',
+      type=str,
+      help='path to training data')
+  parser.add_argument(
+      '--train_batch_size',
+      default=32,
+      type=int,
+      help='batch size for training per gpu')
+  parser.add_argument(
+      '--train_data_size',
+      default=50000,
+      type=int,
+      help='data size for training')
+  parser.add_argument(
+      '--eval_data_path',
+      default='',
+      type=str,
+      help='path to evaluation data')
+  parser.add_argument(
+      '--eval_batch_size',
+      default=32,
+      type=int,
+      help='batch size for evaluation per gpu')
+  parser.add_argument(
+      '--eval_data_size',
+      default=50000,
+      type=int,
+      help='data size for evaluation')
+  args = parser.parse_args()
+  return args
+
+
+def main():
+  args = create_args()
+  os.environ['MASTER_ADDR'] = 'localhost'
+  os.environ['MASTER_PORT'] = '8888'
+
+  args.device = 'cuda' if torch.cuda.is_available() else 'cpu'
+  print(f'Launch job on {args.gpus} GPUs with FSDP')
+  mp.spawn(worker, nprocs=args.gpus, args=(args,))
+
+
+if __name__ == '__main__':
+  main()

notebooks/community/ml_ops/stage5/get_started_with_vertex_endpoint_and_shared_vm.ipynb

@@ -159,9 +159,9 @@
         "\n",
         "# Install the packages\n",
         "\n",
-        "! pip3 install --upgrade google-cloud-aiplatform $USER_FLAG -q\n",
-        "! pip3 install --upgrade tensorflow $USER_FLAG -q\n",
-        "! pip3 install --upgrade tensorflow-hub $USER_FLAG -q"
+        "! pip3 install --upgrade google-cloud-aiplatform \\\n",
+        "                         tensorflow \\\n",
+        "                         tensorflow-hub $USER_FLAG -q"
       ]
     },
     {
@@ -307,22 +307,29 @@
         "id": "timestamp"
       },
       "source": [
-        "#### Timestamp\n",
+        "#### UUID\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append the timestamp onto the name of resources you create in this tutorial."
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "timestamp"
+        "id": "84Vdv7R-QEH6"
       },
       "outputs": [],
       "source": [
-        "from datetime import datetime\n",
+        "import random\n",
+        "import string\n",
         "\n",
-        "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
+        "\n",
+        "# Generate a uuid of a specifed length(default=8)\n",
+        "def generate_uuid(length: int = 8) -> str:\n",
+        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
+        "\n",
+        "\n",
+        "UUID = generate_uuid()"
       ]
     },
     {
@@ -421,7 +428,7 @@
       "outputs": [],
       "source": [
         "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://[your-bucket-name]\":\n",
-        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n",
+        "    BUCKET_NAME = PROJECT_ID + \"aip-\" + UUID\n",
         "    BUCKET_URI = \"gs://\" + BUCKET_NAME"
       ]
     },
@@ -523,7 +530,7 @@
         "\n",
         "Setup up the following constants for Vertex AI:\n",
         "\n",
-        "- `API_ENDPOINT`: The Vertex AI API service endpoint for `Endpoint` services."
+        "- `API_ENDPOINT`: The Vertex AI API service endpoint."
       ]
     },
     {
@@ -538,46 +545,10 @@
         "API_ENDPOINT = \"{}-aiplatform.googleapis.com\".format(REGION)\n",
         "\n",
         "# Vertex location root path for your dataset, model and endpoint resources\n",
-        "PARENT = \"projects/\" + PROJECT_ID + \"/locations/\" + REGION"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "clients:metadata"
-      },
-      "source": [
-        "## Set up clients\n",
+        "PARENT = \"projects/\" + PROJECT_ID + \"/locations/\" + REGION\n",
         "\n",
-        "The Vertex  works as a client/server model. On your side (the Python script) you will create a client that sends requests and receives responses from the Vertex AI server.\n",
-        "\n",
-        "You will use different clients in this tutorial for different steps in the workflow. So set them all up upfront.\n",
-        "\n",
-        "- Endpoint Service for creating endpoints, and deploying models to endpoints."
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "clients:metadata"
-      },
-      "outputs": [],
-      "source": [
         "# client options same for all services\n",
-        "client_options = {\"api_endpoint\": API_ENDPOINT}\n",
-        "\n",
-        "\n",
-        "def create_endpoint_client():\n",
-        "    client = aip_beta.EndpointServiceClient(client_options=client_options)\n",
-        "    return client\n",
-        "\n",
-        "\n",
-        "clients = {}\n",
-        "clients[\"endpoint\"] = create_endpoint_client()\n",
-        "\n",
-        "for client in clients.items():\n",
-        "    print(client)"
+        "client_options = {\"api_endpoint\": API_ENDPOINT}"
       ]
     },
     {
@@ -592,7 +563,7 @@
         "\n",
         "Set the variables `DEPLOY_GPU/DEPLOY_NGPU` to use a container image supporting a GPU and the number of GPUs allocated to the virtual machine (VM) instance. For example, to use a GPU container image with 4 Nvidia Telsa K80 GPUs allocated to each VM, you would specify:\n",
         "\n",
-        "    (aip.AcceleratorType.NVIDIA_TESLA_K80, 4)\n",
+        "    (aip.gapic.AcceleratorType.NVIDIA_TESLA_K80, 4)\n",
         "\n",
         "\n",
         "Otherwise specify `(None, None)` to use a container image to run on a CPU.\n",
@@ -902,7 +873,7 @@
       "outputs": [],
       "source": [
         "model_icn = aiplatform.Model.upload(\n",
-        "    display_name=\"icn_\" + TIMESTAMP,\n",
+        "    display_name=\"icn_\" + UUID,\n",
         "    artifact_uri=MODEL_ICN_DIR,\n",
         "    serving_container_image_uri=DEPLOY_IMAGE,\n",
         ")\n",
@@ -1013,7 +984,7 @@
       "outputs": [],
       "source": [
         "model_use = aiplatform.Model.upload(\n",
-        "    display_name=\"icn_\" + TIMESTAMP,\n",
+        "    display_name=\"icn_\" + UUID,\n",
         "    artifact_uri=MODEL_USE_DIR,\n",
         "    serving_container_image_uri=DEPLOY_IMAGE,\n",
         ")\n",
@@ -1029,64 +1000,52 @@
       "source": [
         "## Creating a deployment resource pool\n",
         "\n",
-        "Currently, creating deploynent resource pools is only supported via the REST-based API (e.g., CURL).\n",
+        "Currently, creating deploynent resource pools is only supported via the REST-based API (e.g., CURL) and GAPIC APIs (Python).\n",
         "\n",
-        "Use `CreateDeploymentResourcePool` API to create a resource pool, with the following configuration:\n",
+        "Use `create_deployment_resource_pool` API to create a resource pool, with the following configuration:\n",
         "\n",
         "- `dedicated_resources`: Compute (HW) resources to allocate for the shared vm.\n",
         "- `min_replica_count`: Auto-scaling, the minimum number of compute nodes.\n",
         "- `max_replica_count`: Auto-scaling, the maximum number of compute nodes.\n",
         "\n",
-        "Learn more about [Deployment Resource Pools]()."
+        "Learn more about [Deployment Resource Pools](https://googleapis.dev/python/aiplatform/latest/aiplatform_v1beta1/deployment_resource_pool_service.html)."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "YiBmoiWYcMQt"
+        "id": "90c51b6cf34a"
       },
       "outputs": [],
       "source": [
-        "DEPLOYMENT_RESOURCE_POOL_ID = \"shared-vm\"  # @param {type: \"string\"}"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "0CHPJ4h-Slgs"
-      },
-      "outputs": [],
-      "source": [
-        "import json\n",
-        "import pprint\n",
-        "pp = pprint.PrettyPrinter(indent=4)\n",
-        "\n",
+        "DEPLOYMENT_RESOURCE_POOL_ID = f\"shared-vm-{UUID}\"  # @param {type: \"string\"}\n",
         "MIN_NODES = 1\n",
         "MAX_NODES = 2\n",
         "\n",
-        "CREATE_RP_PAYLOAD = {\n",
-        "  \"deployment_resource_pool\":{\n",
-        "    \"dedicated_resources\":{\n",
-        "      \"machine_spec\":{\n",
-        "        \"machine_type\": DEPLOY_COMPUTE\n",
-        "      },\n",
-        "      \"min_replica_count\": MIN_NODES, \n",
-        "      \"max_replica_count\": MAX_NODES\n",
-        "    }\n",
-        "  },\n",
-        "  \"deployment_resource_pool_id\":DEPLOYMENT_RESOURCE_POOL_ID\n",
-        "}\n",
-        "CREATE_RP_REQUEST=json.dumps(CREATE_RP_PAYLOAD)\n",
-        "pp.pprint(\"CREATE_RP_REQUEST: \" + CREATE_RP_REQUEST)\n",
+        "# Initialize request argument(s)\n",
+        "deployment_resource_pool = aip_beta.DeploymentResourcePool()\n",
+        "deployment_resource_pool.dedicated_resources.min_replica_count = MIN_NODES\n",
+        "deployment_resource_pool.dedicated_resources.max_replica_count = MAX_NODES\n",
+        "deployment_resource_pool.dedicated_resources.machine_spec.machine_type = DEPLOY_COMPUTE\n",
         "\n",
-        "! curl \\\n",
-        "-X POST \\\n",
-        "-H \"Authorization: Bearer $(gcloud auth print-access-token)\" \\\n",
-        "-H \"Content-Type: application/json\" \\\n",
-        "https://{REGION}-aiplatform.googleapis.com/v1beta1/projects/{PROJECT_ID}/locations/{REGION}/deploymentResourcePools \\\n",
-        "-d '{CREATE_RP_REQUEST}'"
+        "request = aip_beta.CreateDeploymentResourcePoolRequest(\n",
+        "    parent=f\"projects/{PROJECT_ID}/locations/{REGION}\",\n",
+        "    deployment_resource_pool=deployment_resource_pool,\n",
+        "    deployment_resource_pool_id=DEPLOYMENT_RESOURCE_POOL_ID,\n",
+        ")\n",
+        "\n",
+        "pool_client = aip_beta.services.deployment_resource_pool_service.DeploymentResourcePoolServiceClient(\n",
+        "    client_options=client_options\n",
+        ")\n",
+        "\n",
+        "op = pool_client.create_deployment_resource_pool(request=request)\n",
+        "print(op)\n",
+        "\n",
+        "result = op.result()\n",
+        "print(result)\n",
+        "\n",
+        "deployment_pool_id = result.name"
       ]
     },
     {
@@ -1099,21 +1058,19 @@
         "\n",
         "Use `GetDeploymentResourcePool` API to check out the deploynent resource pool that you created. \n",
         "\n",
-        "Learn more about [Get Deployment Resource Pool](https://source.corp.google.com/piper///depot/google3/google/cloud/aiplatform/master/deployment_resource_pool_service.proto;l=75?q=deployment_resource_pool&sq=package:piper%20file:%2F%2Fdepot%2Fgoogle3%20-file:google3%2Fexperimental)."
+        "Learn more about [Get Deployment Resource Pool](https://googleapis.dev/python/aiplatform/latest/aiplatform_v1beta1/deployment_resource_pool_service.html)."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "6wTLyhPraFah"
+        "id": "b740253903c0"
       },
       "outputs": [],
       "source": [
-        "! curl -X GET \\\n",
-        "-H \"Authorization: Bearer $(gcloud auth print-access-token)\" \\\n",
-        "-H \"Content-Type: application/json\" \\\n",
-        "https://{REGION}-aiplatform.googleapis.com/v1beta1/projects/{PROJECT_ID}/locations/{REGION}/deploymentResourcePools/{DEPLOYMENT_RESOURCE_POOL_ID}"
+        "response = pool_client.get_deployment_resource_pool(name=deployment_pool_id)\n",
+        "print(response)"
       ]
     },
     {
@@ -1126,21 +1083,22 @@
         "\n",
         "Use `ListDeploymentResourcePools` API to list all the deployment resource pools. \n",
         "\n",
-        "Learn more about [Listing Deployment Resource Pools](https://source.corp.google.com/piper///depot/google3/google/cloud/aiplatform/master/deployment_resource_pool_service.proto;l=101?q=deployment_resource_pool&sq=package:piper%20file:%2F%2Fdepot%2Fgoogle3%20-file:google3%2Fexperimental)."
+        "Learn more about [Listing Deployment Resource Pools](https://googleapis.dev/python/aiplatform/latest/aiplatform_v1beta1/deployment_resource_pool_service.html)."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "Pxls4sNnaltU"
+        "id": "3ebfd007bff2"
       },
       "outputs": [],
       "source": [
-        "! curl -X GET \\\n",
-        "-H \"Authorization: Bearer $(gcloud auth print-access-token)\" \\\n",
-        "-H \"Content-Type: application/json\" \\\n",
-        "https://{REGION}-aiplatform.googleapis.com/v1beta1/projects/{PROJECT_ID}/locations/{REGION}/deploymentResourcePools"
+        "pools = pool_client.list_deployment_resource_pools(\n",
+        "    parent=f\"projects/{PROJECT_ID}/locations/{REGION}\"\n",
+        ")\n",
+        "for pool in pools:\n",
+        "    print(pool)"
       ]
     },
     {
@@ -1170,11 +1128,11 @@
       },
       "outputs": [],
       "source": [
-        "endpoint_icn = aiplatform.Endpoint.create(display_name=\"icn_\" + TIMESTAMP)\n",
+        "endpoint_icn = aiplatform.Endpoint.create(display_name=\"icn_\" + UUID)\n",
         "\n",
         "print(endpoint_icn)\n",
         "\n",
-        "endpoint_use = aiplatform.Endpoint.create(display_name=\"use_\" + TIMESTAMP)\n",
+        "endpoint_use = aiplatform.Endpoint.create(display_name=\"use_\" + UUID)\n",
         "\n",
         "print(endpoint_use)"
       ]
@@ -1204,6 +1162,12 @@
       },
       "outputs": [],
       "source": [
+        "import json\n",
+        "import pprint\n",
+        "\n",
+        "pp = pprint.PrettyPrinter(indent=4)\n",
+        "\n",
+        "\n",
         "SHARED_RESOURCE = \"projects/{project_id}/locations/{region}/deploymentResourcePools/{deployment_resource_pool_id}\".format(\n",
         "    project_id=PROJECT_ID,\n",
         "    region=REGION,\n",
@@ -1363,18 +1327,27 @@
         "    time.sleep(30)"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "52248c450776"
+      },
+      "source": [
+        "### Get deployment details for the endpoint\n",
+        "\n",
+        "List the deployed models on the endpoint."
+      ]
+    },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "86a659bf60f0"
+        "id": "3b768614e7c6"
       },
       "outputs": [],
       "source": [
-        "! curl -X GET \\\n",
-        " -H \"Authorization: Bearer $(gcloud auth print-access-token)\" \\\n",
-        " -H \"Content-Type: application/json\" \\\n",
-        "https://{REGION}-aiplatform.googleapis.com/v1/projects/759209241365/locations/us-central1/endpoints/2259566763823857664"
+        "print(endpoint_icn.list_models())\n",
+        "print(endpoint_use.list_models())"
       ]
     },
     {
@@ -1557,21 +1530,19 @@
       "source": [
         "#### Delete the `DeploymentResourcePool`\n",
         "\n",
-        "The method 'delete()' will delete your deployment resource pool."
+        "The method 'delete_deployment_resource_pool()' will delete your deployment resource pool."
       ]
     },
     {
       "cell_type": "code",
       "execution_count": null,
       "metadata": {
-        "id": "ac40cc1d594a"
+        "id": "b76a4de1e57e"
       },
       "outputs": [],
       "source": [
-        "! curl -X DELETE \\\n",
-        "-H \"Authorization: Bearer $(gcloud auth print-access-token)\" \\\n",
-        "-H \"Content-Type: application/json\" \\\n",
-        "https://{REGION}-aiplatform.googleapis.com/v1beta1/projects/{PROJECT_ID}/locations/{REGION}/deploymentResourcePools/{DEPLOYMENT_RESOURCE_POOL_ID}"
+        "response = pool_client.delete_deployment_resource_pool(name=deployment_pool_id)\n",
+        "print(response)"
       ]
     },
     {

notebooks/notebook_template_review.py

@@ -380,7 +380,6 @@ class NotebookRule(ABC):
         pass
 
 
-
 class CopyrightRule(NotebookRule):
     def validate(self, notebook: Notebook) -> bool:
         """
@@ -401,16 +400,6 @@ class NoticesRule(NotebookRule):
         if cell['source'][0].startswith('This notebook'):
             notebook.pop()
         return True
-    
-class TestEnvRule(NotebookRule):
-    def validate(self, notebook: Notebook) -> bool:
-        """
-        Parse the (optional) test in which environment cell
-        """
-        cell = notebook.peek()
-        if cell['source'][0].startswith('**_NOTE_**: This notebook has been tested'):
-            notebook.pop()
-        return True
 
 
 class TitleRule(NotebookRule): 
@@ -496,6 +485,28 @@ class LinksRule(NotebookRule):
         return ret
 
 
+class TableRule(NotebookRule):
+    def validate(self, notebook: Notebook) -> bool:
+        """
+        Parse the (optional) table of contents cell
+        """
+        cell = notebook.peek()
+        if cell['source'][0].startswith('## Table of contents'):
+            notebook.pop()
+        return True
+
+
+class TestEnvRule(NotebookRule):
+    def validate(self, notebook: Notebook) -> bool:
+        """
+        Parse the (optional) test in which environment cell
+        """
+        cell = notebook.peek()
+        if cell['source'][0].startswith('**_NOTE_**: This notebook has been tested'):
+            notebook.pop()
+        return True
+
+
 class OverviewRule(NotebookRule):
     def validate(self, notebook: Notebook) -> bool: 
         """
@@ -1050,6 +1061,7 @@ notices = NoticesRule()
 title = TitleRule()
 links = LinksRule()
 testenv = TestEnvRule()
+table = TableRule()
 overview = OverviewRule()
 objective = ObjectiveRule()
 recommendations = RecommendationsRule()
@@ -1065,7 +1077,7 @@ enableapis = EnableAPIsRule()
 setupproject = SetupProjectRule()
 
  # Cell Validation
-rules = [ copyright, notices, title, links, testenv, overview, objective,
+rules = [ copyright, notices, title, links, testenv, table, overview, objective,
           recommendations, dataset, costs, setuplocal, helpers,
           installation, restart, versions, beforebegin, enableapis,
           setupproject

notebooks/official/CODEOWNERS

@@ -42,3 +42,4 @@
 /sdk/SDK_FBProphet_Forecasting_Online.ipynb @brianchunkang
 /automl/sdk_automl_forecasting_hierarchical_batch.ipynb @ivanmkc
 /prediction/custom_batch_prediction_feature_filter.ipynb @soheilazangeneh
+/feature_store/feature_store_streaming_ingestion_sdk.ipynb @soheilazangeneh

notebooks/official/automl/automl-tabular-classification.ipynb

@@ -29,7 +29,7 @@
         "id": "JAPoU8Sm5E6e"
       },
       "source": [
-        "# Vertex AI SDK for Python: AutoML Tabular Training and Prediction\n",
+        "# Vertex AI SDK for Python: AutoML Tabular training and prediction\n",
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
@@ -63,7 +63,7 @@
         "\n",
         "This tutorial demonstrates how to use the Vertex AI Python client library to train and deploy a tabular classification model for online prediction.\n",
         "\n",
-        "**Note**: you may incur charges for training, prediction, storage, or usage of other GCP products in connection with testing this SDK."
+        "**Note**: you may incur charges for training, prediction, storage, or usage of other Google Cloud products in connection with testing this SDK."
       ]
     },
     {
@@ -76,6 +76,11 @@
         "\n",
         "In this tutorial, you learn how to train and make predictions on an AutoML model based on a tabular dataset. Alternatively, you can train and make predictions on models by using the `gcloud` command-line tool or by using the online Cloud Console.\n",
         "\n",
+        "This tutorial uses the following Google Cloud ML services and resources:\n",
+        "\n",
+        "- Vertex AI\n",
+        "- AutoML Tabular\n",
+        "\n",
         "The steps performed include the following:\n",
         "\n",
         "- Create a Vertex AI model training job.\n",
@@ -122,7 +127,9 @@
         "id": "install_aip"
       },
       "source": [
-        "## Installation"
+        "## Installation\n",
+        "\n",
+        "Install the packages required for executing this notebook."
       ]
     },
     {
@@ -135,55 +142,20 @@
       "source": [
         "import os\n",
         "\n",
-        "# The Google Cloud Notebook product has specific requirements\n",
-        "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
+        "# The Vertex AI Workbench Notebook product has specific requirements\n",
+        "IS_WORKBENCH_NOTEBOOK = os.getenv(\"DL_ANACONDA_HOME\") and not os.getenv(\"VIRTUAL_ENV\")\n",
+        "IS_USER_MANAGED_WORKBENCH_NOTEBOOK = os.path.exists(\n",
+        "    \"/opt/deeplearning/metadata/env_version\"\n",
+        ")\n",
         "\n",
-        "# Google Cloud Notebook requires dependencies to be installed with '--user'\n",
+        "# Vertex AI Notebook requires dependencies to be installed with '--user'\n",
         "USER_FLAG = \"\"\n",
-        "if IS_GOOGLE_CLOUD_NOTEBOOK:\n",
-        "    USER_FLAG = \"--user\""
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "b03b7f4487ff"
-      },
-      "source": [
-        "Install the latest version of the Vertex AI client library.\n",
+        "if IS_WORKBENCH_NOTEBOOK:\n",
+        "    USER_FLAG = \"--user\"\n",
         "\n",
-        "Run the following command in your virtual environment to install the Vertex SDK for Python:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "d489d38261dd"
-      },
-      "outputs": [],
-      "source": [
-        "! pip install {USER_FLAG} --upgrade google-cloud-aiplatform"
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "install_storage"
-      },
-      "source": [
-        "Install the Cloud Storage library:"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "qssss-KSlugo"
-      },
-      "outputs": [],
-      "source": [
-        "! pip install {USER_FLAG} --upgrade google-cloud-storage"
+        "# Install the packagesimport os\n",
+        "! pip3 install {USER_FLAG} -q --upgrade google-cloud-aiplatform \\\n",
+        "                                        google-cloud-storage"
       ]
     },
     {

notebooks/official/automl/automl_tabular_on_vertex_pipelines.ipynb

@@ -29,7 +29,7 @@
         "id": "mThXALJl9Yue"
       },
       "source": [
-        "# Tabular Workflow: AutoML Tabular Pipeline\n",
+        "# AutoML Tabular Workflow pipelines\n",
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
@@ -72,7 +72,12 @@
       "source": [
         "### Objective\n",
         "\n",
-        "In this tutorial, you learn how to create two regression models using [Vertex Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction) downloaded from [Google Cloud Pipeline Components](https://cloud.google.com/vertex-ai/docs/pipelines/components-introduction) (GCPC). These pipelines will be Vertex AI Tabular Workflow pipelines which are maintained by Google. These pipelines will showcase different ways to customize the Vertex Tabular training process.\n",
+        "In this tutorial, you learn how to create two regression models using [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction) downloaded from [Google Cloud Pipeline Components](https://cloud.google.com/vertex-ai/docs/pipelines/components-introduction) (GCPC). These pipelines will be Vertex AI Tabular Workflow pipelines which are maintained by Google. These pipelines will showcase different ways to customize the Vertex Tabular training process.\n",
+        "\n",
+        "This tutorial uses the following Google Cloud ML services:\n",
+        "\n",
+        "- `AutoML Training`\n",
+        "- `Vertex AI Datasets`\n",
         "\n",
         "The steps performed are:\n",
         "\n",

notebooks/official/automl/sdk_automl_forecasting_hierarchical_batch.ipynb

@@ -64,6 +64,32 @@
         "This tutorial demonstrates how to use the Vertex AI SDK for Python to create hierarchical forecasting models using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users)and do batch prediction. Specifically, you predict a fictional store's sales based on historical sales data."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "objective:automl,training,online_prediction"
+      },
+      "source": [
+        "### Objective\n",
+        "\n",
+        "In this tutorial, you create an AutoML hierarchical forecasting model and deploy it for batch prediction using the Vertex AI SDK for Python. You can alternatively create and deploy models using the `gcloud` command-line tool or batch using the Cloud Console.\n",
+        "The rationale for a hierarchical forecasting model is to minimize the error for a given group of sales data. In this tutorial, you will be minimizing the error for sale predictions at the \"product\" level.\n",
+        "\n",
+        "This tutorial uses the following Google Cloud ML services:\n",
+        "\n",
+        "- `AutoML Training`\n",
+        "- `Vertex AI Datasets`\n",
+        "\n",
+        "The steps performed include:\n",
+        "\n",
+        "- Create a Vertex AI `TimeSeriesDataset` resource.\n",
+        "- Train the model.\n",
+        "- View the model evaluation.\n",
+        "- Deploy the `Model` resource to a serving `Endpoint` resource.\n",
+        "- Make a prediction.\n",
+        "- Undeploy the `Model`."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -90,27 +116,6 @@
         "Link to data: gs://cloud-samples-data/vertex-ai/structured_data/forecasting/synthetic_sales_data.csv"
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "objective:automl,training,online_prediction"
-      },
-      "source": [
-        "### Objective\n",
-        "\n",
-        "In this tutorial, you create an AutoML hierarchical forecasting model and deploy it for batch prediction using the Vertex AI SDK for Python. You can alternatively create and deploy models using the `gcloud` command-line tool or batch using the Cloud Console.\n",
-        "The rationale for a hierarchical forecasting model is to minimize the error for a given group of sales data. In this tutorial, you will be minimizing the error for sale predictions at the \"product\" level.\n",
-        "\n",
-        "The steps performed include:\n",
-        "\n",
-        "- Create a Vertex AI `TimeSeriesDataset` resource.\n",
-        "- Train the model.\n",
-        "- View the model evaluation.\n",
-        "- Deploy the `Model` resource to a serving `Endpoint` resource.\n",
-        "- Make a prediction.\n",
-        "- Undeploy the `Model`."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {

notebooks/official/automl/sdk_automl_image_object_detection_batch.ipynb

@@ -66,17 +66,6 @@
         "This tutorial demonstrates how to use the Vertex AI SDK to create image object detection models and do batch prediction using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) model."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "dataset:salads,iod"
-      },
-      "source": [
-        "### Dataset\n",
-        "\n",
-        "The dataset used for this tutorial is the Salads category of the [OpenImages dataset](https://www.tensorflow.org/datasets/catalog/open_images_v4) from [TensorFlow Datasets](https://www.tensorflow.org/datasets/catalog/overview). This dataset does not require any feature engineering. The version of the dataset you will use in this tutorial is stored in a public Cloud Storage bucket. The trained model predicts the bounding box locations and the corresponding type of salad items in an image from a class of five items: salad, seafood, tomato, baked goods, or cheese."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -87,6 +76,11 @@
         "\n",
         "In this tutorial, you create an AutoML image object detection model from a Python script, and then do a batch prediction using the Vertex AI SDK. You can alternatively create and deploy models using the `gcloud` command-line tool or online using the Cloud Console.\n",
         "\n",
+        "This tutorial uses the following Google Cloud ML services:\n",
+        "\n",
+        "- `AutoML Training`\n",
+        "- `Vertex AI Datasets`\n",
+        "\n",
         "The steps performed include:\n",
         "\n",
         "- Create a Vertex `Dataset` resource.\n",
@@ -101,6 +95,17 @@
         "* Batch Prediction Service: Does a queued (batch) prediction for the entire set of instances in the background and stores the results in a Cloud Storage bucket when ready."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dataset:salads,iod"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The dataset used for this tutorial is the Salads category of the [OpenImages dataset](https://www.tensorflow.org/datasets/catalog/open_images_v4) from [TensorFlow Datasets](https://www.tensorflow.org/datasets/catalog/overview). This dataset does not require any feature engineering. The version of the dataset you will use in this tutorial is stored in a public Cloud Storage bucket. The trained model predicts the bounding box locations and the corresponding type of salad items in an image from a class of five items: salad, seafood, tomato, baked goods, or cheese."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {

notebooks/official/automl/sdk_automl_tabular_regression_batch_bq.ipynb

@@ -74,7 +74,7 @@
         "\n",
         "In this tutorial, you learn how to create an AutoML tabular regression model and deploy it for batch prediction using the Vertex AI SDK for Python. You can alternatively create and deploy models using the `gcloud` command-line tool or batch using the Cloud Console.\n",
         "\n",
-        "This tutorial uses the following Google Cloud ML services and resources:\n",
+        "This tutorial uses the following Google Cloud ML services:\n",
         "\n",
         "- Vertex AI Datasets (Tabular)\n",
         "- Vertex AI Training (AutoML Tabular Training)\n",

notebooks/official/automl/sdk_automl_text_entity_extraction_online.ipynb

@@ -73,7 +73,12 @@
       "source": [
         "### Objective\n",
         "\n",
-        "In this tutorial, you learn how to create an AutoML text entity extraction model and deploy for online prediction from a Python script using the Vertex SDK. You can alternatively create and deploy models using the `gcloud` command-line tool or online using the Cloud Console.\n",
+        "In this tutorial, you learn how to create an AutoML text entity extraction model and deploy for online prediction from a Python script using the Vertex AI SDK. You can alternatively create and deploy models using the `gcloud` command-line tool or online using the Cloud Console.\n",
+        "\n",
+        "This tutorial uses the following Google Cloud ML services:\n",
+        "\n",
+        "- `AutoML Training`\n",
+        "- `Vertex AI Datasets`\n",
         "\n",
         "The steps performed include:\n",
         "\n",

notebooks/official/experiments/build_model_experimentation_lineage_with_prebuild_code.ipynb

@@ -72,7 +72,22 @@
       "source": [
         "### Objective\n",
         "\n",
-        "In this notebook, you learn how to integrate preprocessing code in a Vertex AI experiments. Also you build the experiment lineage lets you record, analyze, debug, and audit metadata and artifacts produced along your ML journey."
+        "In this notebook, you learn how to integrate preprocessing code in a Vertex AI experiments. Also you build the experiment lineage lets you record, analyze, debug, and audit metadata and artifacts produced along your ML journey.\n",
+        "\n",
+        "This tutorial uses the following Google Cloud ML services and resources:\n",
+        "\n",
+        "- Vertex ML Metadata\n",
+        "- Vertex AI Experiments\n",
+        "\n",
+        "The steps performed include:\n",
+        "\n",
+        "- Execute module for preprocessing data\n",
+        "  - Create a dataset artifact\n",
+        "  - Log parameters\n",
+        "-  Execute module for training the model\n",
+        "  - Log parameters\n",
+        "  - Create model artifact\n",
+        "  - Assign tracking lineage to dataset, model and parameters"
       ]
     },
     {

notebooks/official/experiments/comparing_pipeline_runs.ipynb

@@ -61,7 +61,7 @@
       "source": [
         "## Overview\n",
         "\n",
-        "Depending on the model life cycle of your data science team, you would like to experiment and track training Pipeline runs and its associated parameters. Then, you would to compare runs of these Pipelines to each others in order to figure out which is the best configuration generates the model you will register in the Vertex AI Model Registry."
+        "Depending on the model life cycle of your data science team, you would like to experiment and track training pipeline runs and its associated parameters. Then, you would to compare runs of these Pipelines to each others in order to figure out which is the best configuration generates the model you will register in the Vertex AI Model Registry."
       ]
     },
     {
@@ -74,6 +74,11 @@
         "\n",
         "In this notebook, you learn how to use `Vertex AI Experiments` to log a pipeline job and compare different pipeline jobs.\n",
         "\n",
+        "This tutorial uses the following Google Cloud ML services and resources:\n",
+        "\n",
+        "- Vertex AI Pipelines\n",
+        "- Vertex AI Experiments\n",
+        "\n",
         "The steps performed include:\n",
         "\n",
         "* Formalize a training component\n",

notebooks/official/explainable_ai/sdk_custom_tabular_regression_online_explain_get_metadata.ipynb

@@ -62,7 +62,7 @@
         "## Overview\n",
         "\n",
         "\n",
-        "This tutorial demonstrates how to use the Vertex SDK to train and deploy a custom tabular regression model for online prediction with explanation."
+        "This tutorial demonstrates how to use the Vertex AI SDK to train and deploy a custom tabular regression model for online prediction with explanation."
       ]
     },
     {
@@ -73,7 +73,16 @@
       "source": [
         "### Objective\n",
         "\n",
-        "In this tutorial, you learn how to create a custom model from a Python script in a Google prebuilt Docker container using the Vertex SDK, and then do a prediction with explanations on the deployed model by sending data. You can alternatively create custom models using `gcloud` command-line tool or online using Cloud Console.\n",
+        "In this tutorial, you learn how to create a custom model from a Python script in a Google prebuilt Docker container using the Vertex AI SDK, and then do a prediction with explanations on the deployed model by sending data. You can alternatively create custom models using `gcloud` command-line tool or online using Cloud Console.\n",
+        "\n",
+        "This tutorial uses the following Google Cloud ML services:\n",
+        "\n",
+        "- `Vertex AI Training`\n",
+        "- `Vertex AI Online Prediction`\n",
+        "- `Vertex Explainable AI`\n",
+        "- `Vertex AI Model` resource\n",
+        "- `Vertex AI Endpoint` resource\n",
+        "\n",
         "\n",
         "The steps performed include:\n",
         "\n",

notebooks/official/feature_store/feature_store_streaming_ingestion_sdk.ipynb

@@ -0,0 +1,774 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ur8xi4C7S06n"
+      },
+      "outputs": [],
+      "source": [
+        "# Copyright 2022 Google LLC\n",
+        "#\n",
+        "# Licensed under the Apache License, Version 2.0 (the \"License\");\n",
+        "# you may not use this file except in compliance with the License.\n",
+        "# You may obtain a copy of the License at\n",
+        "#\n",
+        "#     https://www.apache.org/licenses/LICENSE-2.0\n",
+        "#\n",
+        "# Unless required by applicable law or agreed to in writing, software\n",
+        "# distributed under the License is distributed on an \"AS IS\" BASIS,\n",
+        "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
+        "# See the License for the specific language governing permissions and\n",
+        "# limitations under the License."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "JAPoU8Sm5E6e"
+      },
+      "source": [
+        "# Feature Store: Streaming ingestion SDK\n",
+        "\n",
+        "<table align=\"left\">\n",
+        "\n",
+        "  <td>\n",
+        "    <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/feature_store/feature_store_streaming_ingestion_sdk.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/feature_store/feature_store_streaming_ingestion_sdk.ipynb\">\n",
+        "      <img src=\"https://cloud.google.com/ml-engine/images/github-logo-32px.png\" alt=\"GitHub logo\">\n",
+        "      View on GitHub\n",
+        "    </a>\n",
+        "  </td>\n",
+        "  <td>\n",
+        "    <a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/feature_store/feature_store_streaming_ingestion_sdk.ipynb\">\n",
+        "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
+        "      Open in Vertex AI Workbench\n",
+        "    </a>\n",
+        "  </td>\n",
+        "</table>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "24743cf4a1e1"
+      },
+      "source": [
+        "**_NOTE_**: This notebook has been tested in the following environment:\n",
+        "\n",
+        "* Python version = 3.9"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tvgnzT1CKxrO"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "This notebook demonstrates how to use Vertex AI Feature Store's streaming ingestion at the SDK layer."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "d975e698c9a4"
+      },
+      "source": [
+        "### Objective\n",
+        "\n",
+        "In this tutorial, you learn how to ingest features from a `Pandas DataFrame` into your Vertex AI Feature Store using `write_feature_values` method from the Vertex AI SDK.\n",
+        "\n",
+        "This tutorial uses the following Google Cloud ML services and resources:\n",
+        "\n",
+        "- Vertex AI Feature Store\n",
+        "\n",
+        "\n",
+        "The steps performed include:\n",
+        "\n",
+        "- Create `Feature Store`\n",
+        "- Create new `Entity Type` for your `Feature Store`\n",
+        "- Ingest feature values from `Pandas DataFrame` into `Feature Store`'s `Entity Types`."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "08d289fa873f"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The dataset used for this notebook is the penguins dataset from [BigQuery public datasets](https://cloud.google.com/bigquery/public-data). This dataset has the following features: `culmen_length_mm`, `culmen_depth_mm`, `flipper_length_mm`, `body_mass_g`, `species`, and `sex`."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "aed92deeb4a0"
+      },
+      "source": [
+        "### Costs\n",
+        "\n",
+        "This tutorial uses billable components of Google Cloud:\n",
+        "\n",
+        "* Vertex AI\n",
+        "\n",
+        "Learn about [Vertex AI\n",
+        "pricing](https://cloud.google.com/vertex-ai/pricing) and use the [Pricing\n",
+        "Calculator](https://cloud.google.com/products/calculator/)\n",
+        "to generate a cost estimate based on your projected usage.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "i7EUnXsZhAGF"
+      },
+      "source": [
+        "## Installation\n",
+        "\n",
+        "Install the following packages required to execute this notebook."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "2b4ef9b72d43"
+      },
+      "outputs": [],
+      "source": [
+        "# Install the packages\n",
+        "! pip3 install --upgrade google-cloud-aiplatform\\\n",
+        "                         google-cloud-bigquery\\\n",
+        "                         numpy\\\n",
+        "                         pandas\\\n",
+        "                         pyarrow -q"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "58707a750154"
+      },
+      "source": [
+        "### Colab only: Uncomment the following cell to restart the kernel."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "f200f10a1da3"
+      },
+      "outputs": [],
+      "source": [
+        "# Automatically restart kernel after installs so that your environment can access the new packages\n",
+        "# import IPython\n",
+        "\n",
+        "# app = IPython.Application.instance()\n",
+        "# app.kernel.do_shutdown(True)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "BF1j6f9HApxa"
+      },
+      "source": [
+        "## Before you begin\n",
+        "\n",
+        "### Set up your Google Cloud project\n",
+        "\n",
+        "**The following steps are required, regardless of your notebook environment.**\n",
+        "\n",
+        "1. [Select or create a Google Cloud project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
+        "\n",
+        "2. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
+        "\n",
+        "3. [Enable the Vertex AI API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com).\n",
+        "\n",
+        "4. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "WReHDGG5g0XY"
+      },
+      "source": [
+        "#### Set your project ID\n",
+        "\n",
+        "**If you don't know your project ID**, try the following:\n",
+        "* Run `gcloud config list`.\n",
+        "* Run `gcloud projects list`.\n",
+        "* See the support page: [Locate the project ID](https://support.google.com/googleapi/answer/7014113)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "oM1iC_MfAts1"
+      },
+      "outputs": [],
+      "source": [
+        "PROJECT_ID = \"[your-project-id]\"  # @param {type:\"string\"}\n",
+        "\n",
+        "# Set the project id\n",
+        "! gcloud config set project {PROJECT_ID}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "region"
+      },
+      "source": [
+        "#### Region\n",
+        "\n",
+        "You can also change the `REGION` variable used by Vertex AI. Learn more about [Vertex AI regions](https://cloud.google.com/vertex-ai/docs/general/locations)."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "kljmKgilI_de"
+      },
+      "outputs": [],
+      "source": [
+        "REGION = \"us-central1\"  # @param {type: \"string\"}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "sBCra4QMA2wR"
+      },
+      "source": [
+        "### Authenticate your Google Cloud account\n",
+        "\n",
+        "Depending on your Jupyter environment, you may have to manually authenticate. Follow the relevant instructions below."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "74ccc9e52986"
+      },
+      "source": [
+        "**1. Vertex AI Workbench**\n",
+        "* Do nothing as you are already authenticated."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "de775a3773ba"
+      },
+      "source": [
+        "**2. Local JupyterLab instance, uncomment and run:**"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "254614fa0c46"
+      },
+      "outputs": [],
+      "source": [
+        "# ! gcloud auth login"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ef21552ccea8"
+      },
+      "source": [
+        "**3. Colab, uncomment and run:**"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "603adbbf0532"
+      },
+      "outputs": [],
+      "source": [
+        "# from google.colab import auth\n",
+        "# auth.authenticate_user()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "f6b2ccc891ed"
+      },
+      "source": [
+        "**4. Service account or other**\n",
+        "* See how to grant Cloud Storage permissions to your service account at https://cloud.google.com/storage/docs/gsutil/commands/iam#ch-examples."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "EsCYkJ4IU-z4"
+      },
+      "source": [
+        "### UUID\n",
+        "\n",
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "4jWj2DSTU9my"
+      },
+      "outputs": [],
+      "source": [
+        "import random\n",
+        "import string\n",
+        "\n",
+        "\n",
+        "# Generate a uuid of a specifed length(default=8)\n",
+        "def generate_uuid(length: int = 8) -> str:\n",
+        "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
+        "\n",
+        "\n",
+        "UUID = generate_uuid()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "960505627ddf"
+      },
+      "source": [
+        "### Import libraries"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "PyQmSRbKA8r-"
+      },
+      "outputs": [],
+      "source": [
+        "import numpy as np\n",
+        "import pandas as pd\n",
+        "from google.cloud import aiplatform, bigquery"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "init_aip:mbsdk,all"
+      },
+      "source": [
+        "### Initialize Vertex AI SDK for Python\n",
+        "\n",
+        "Initialize the Vertex AI SDK for Python for your project."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "0ep8KuQhI_df"
+      },
+      "outputs": [],
+      "source": [
+        "aiplatform.init(project=PROJECT_ID, location=REGION)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "k5XsEiAuEWUJ"
+      },
+      "source": [
+        "## Download and prepare the data"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "rOd7Ixa1pqBY"
+      },
+      "outputs": [],
+      "source": [
+        "def download_bq_table(bq_table_uri: str) -> pd.DataFrame:\n",
+        "    # Remove bq:// prefix if present\n",
+        "    prefix = \"bq://\"\n",
+        "    if bq_table_uri.startswith(prefix):\n",
+        "        bq_table_uri = bq_table_uri[len(prefix) :]\n",
+        "\n",
+        "    table = bigquery.TableReference.from_string(bq_table_uri)\n",
+        "\n",
+        "    # Create a BigQuery client\n",
+        "    bqclient = bigquery.Client(project=PROJECT_ID)\n",
+        "\n",
+        "    # Download the table rows\n",
+        "    rows = bqclient.list_rows(\n",
+        "        table,\n",
+        "    )\n",
+        "    return rows.to_dataframe()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "SdX_m1Uppkfu"
+      },
+      "outputs": [],
+      "source": [
+        "BQ_SOURCE = \"bq://bigquery-public-data.ml_datasets.penguins\"\n",
+        "\n",
+        "# Download penguins BigQuery table\n",
+        "penguins_df = download_bq_table(BQ_SOURCE)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "QuQe6mSbFbhm"
+      },
+      "source": [
+        "### Prepare the data\n",
+        "\n",
+        "Feature values to be written to the Feature Store can take the form of a list of `WriteFeatureValuesPayload` objects, a Python `dict` of the form\n",
+        "\n",
+        "`{entity_id : {feature_id : feature_value}, ...},`\n",
+        "\n",
+        "or a pandas `Dataframe`, where the `index` column holds the unique entity ID strings and each remaining column represents a feature.  In this notebook, since you use a pandas `DataFrame` for ingesting features we convert the index column data type to `string` to be used as `Entity ID`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cljxzJ3bqDer"
+      },
+      "outputs": [],
+      "source": [
+        "# Prepare the data\n",
+        "penguins_df.index = penguins_df.index.map(str)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "GSxrSdSY2ovn"
+      },
+      "outputs": [],
+      "source": [
+        "# Remove null values\n",
+        "NA_VALUES = [\"NA\", \".\"]\n",
+        "penguins_df = penguins_df.replace(to_replace=NA_VALUES, value=np.NaN).dropna()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "vgn4oQmSqdKI"
+      },
+      "source": [
+        "## Create Feature Store and define schemas\n",
+        "\n",
+        "Vertex AI Feature Store organizes resources hierarchically in the following order:\n",
+        "\n",
+        "`Featurestore -> EntityType -> Feature`\n",
+        "\n",
+        "You must create these resources before you can ingest data into Vertex AI Feature Store.\n",
+        "\n",
+        "Learn more about [Vertex AI Feature Store](https://cloud.google.com/vertex-ai/docs/featurestore)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "yaHwdbGjZWTq"
+      },
+      "source": [
+        "### Create a Feature Store\n",
+        "\n",
+        "You create a Feature Store using `aiplatform.Featurestore.create` with the following parameters:\n",
+        "\n",
+        "* `featurestore_id (str)`: The ID to use for this Featurestore, which will become the final component of the Featurestore's resource name. The value must be unique within the project and location.\n",
+        "* `online_store_fixed_node_count`: Configuration for online serving resources.\n",
+        "* `project`: Project to create EntityType in. If not set, project set in `aiplatform.init` is used.\n",
+        "* `location`: Location to create EntityType in. If not set, location set in `aiplatform.init` is used.\n",
+        "* `sync`:  Whether to execute this creation synchronously."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "cImsONglqfxO"
+      },
+      "outputs": [],
+      "source": [
+        "FEATURESTORE_ID = f\"penguins_{UUID}\"\n",
+        "\n",
+        "penguins_feature_store = aiplatform.Featurestore.create(\n",
+        "    featurestore_id=FEATURESTORE_ID,\n",
+        "    online_store_fixed_node_count=1,\n",
+        "    project=PROJECT_ID,\n",
+        "    location=REGION,\n",
+        "    sync=True,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "UfXgSD1VdzKb"
+      },
+      "source": [
+        "##### Verify that the Feature Store is created\n",
+        "Check if the Feature Store was successfully created by running the following code block."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "oud1OdfQd52r"
+      },
+      "outputs": [],
+      "source": [
+        "fs = aiplatform.Featurestore(\n",
+        "    featurestore_name=FEATURESTORE_ID,\n",
+        "    project=PROJECT_ID,\n",
+        "    location=REGION,\n",
+        ")\n",
+        "print(fs.gca_resource)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ep74rSlJWF3c"
+      },
+      "source": [
+        "### Create an EntityType\n",
+        "\n",
+        "An entity type is a collection of semantically related features. You define your own entity types, based on the concepts that are relevant to your use case. For example, a movie service might have the entity types `movie` and `user`, which group related features that correspond to movies or users.\n",
+        "\n",
+        "Here, you create an entity type entity type named `penguin_entity_type` using `create_entity_type` with the following parameters:\n",
+        "* `entity_type_id (str)`: The ID to use for the EntityType, which will become the final component of the EntityType's resource name. The value must be unique within a Feature Store.\n",
+        "* `description`: Description of the EntityType."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "zNzr-FlEr3tI"
+      },
+      "outputs": [],
+      "source": [
+        "ENTITY_TYPE_ID = f\"penguin_entity_type_{UUID}\"\n",
+        "\n",
+        "# Create penguin entity type\n",
+        "penguins_entity_type = penguins_feature_store.create_entity_type(\n",
+        "    entity_type_id=ENTITY_TYPE_ID,\n",
+        "    description=\"Penguins entity type\",\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "CquSdTp7duVw"
+      },
+      "source": [
+        "##### Verify that the EntityType is created\n",
+        "Check if the Entity Type was successfully created by running the following code block."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "76ocr_hJsG-t"
+      },
+      "outputs": [],
+      "source": [
+        "entity_type = penguins_feature_store.get_entity_type(entity_type_id=ENTITY_TYPE_ID)\n",
+        "\n",
+        "print(entity_type.gca_resource)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "2vYV2UUFehwZ"
+      },
+      "source": [
+        "### Create Features\n",
+        "A feature is a measurable property or attribute of an entity type. For example, `penguin` entity type has features such as `flipper_length_mm`, and `body_mass_g`. Features can be created within each entity type.\n",
+        "\n",
+        "When you create a feature, you specify its value type such as `DOUBLE`, and `STRING`. This value determines what value types you can ingest for a particular feature.\n",
+        "\n",
+        "Learn more about [Feature Value Types](https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.featurestores.entityTypes.features)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "WQ5EsPPbsSuE"
+      },
+      "outputs": [],
+      "source": [
+        "penguins_feature_configs = {\n",
+        "    \"species\": {\n",
+        "        \"value_type\": \"STRING\",\n",
+        "    },\n",
+        "    \"island\": {\n",
+        "        \"value_type\": \"STRING\",\n",
+        "    },\n",
+        "    \"culmen_length_mm\": {\n",
+        "        \"value_type\": \"DOUBLE\",\n",
+        "    },\n",
+        "    \"culmen_depth_mm\": {\n",
+        "        \"value_type\": \"DOUBLE\",\n",
+        "    },\n",
+        "    \"flipper_length_mm\": {\n",
+        "        \"value_type\": \"DOUBLE\",\n",
+        "    },\n",
+        "    \"body_mass_g\": {\"value_type\": \"DOUBLE\"},\n",
+        "    \"sex\": {\"value_type\": \"STRING\"},\n",
+        "}"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "AKRXJCPijM8w"
+      },
+      "source": [
+        "You can create features either using `create_feature` or `batch_create_features`. Here, for convinience, you have added all feature configs in one variabel, so we use `batch_create_features`."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "tXOI1Onhs46x"
+      },
+      "outputs": [],
+      "source": [
+        "penguin_features = penguins_entity_type.batch_create_features(\n",
+        "    feature_configs=penguins_feature_configs,\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "WBx26pZItUN4"
+      },
+      "source": [
+        "### Write features to the Feature Store\n",
+        "Use the `write_feature_values` API to write a feature to the Feature Store with the following parameter:\n",
+        "\n",
+        "* `instances`: Feature values to be written to the Feature Store that can take the form of a list of WriteFeatureValuesPayload objects, a Python dict, or a pandas Dataframe.\n",
+        "\n",
+        "This streaming ingestion feature has been introduced to the Vertex AI SDK under the **preview** namespace. Here, you pass the pandas `Dataframe` you created from penguins dataset as `instances` parameter.\n",
+        "\n",
+        "Learn more about [Streaming ingestion API](https://github.com/googleapis/python-aiplatform/blob/e6933503d2d3a0f8a8f7ef8c178ed50a69ac2268/google/cloud/aiplatform/preview/featurestore/entity_type.py#L36)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "iUGI-ftltXqE"
+      },
+      "outputs": [],
+      "source": [
+        "penguins_entity_type.preview.write_feature_values(instances=penguins_df)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "STq67KHO3q_e"
+      },
+      "source": [
+        "## Read back written features\n",
+        "\n",
+        "Wait a few seconds for the write to propagate, then do an online read to confirm the write was successful."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "lwoMnze43r9G"
+      },
+      "outputs": [],
+      "source": [
+        "ENTITY_IDS = [str(x) for x in range(100)]\n",
+        "penguins_entity_type.read(entity_ids=ENTITY_IDS)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "TpV-iwP9qw9c"
+      },
+      "source": [
+        "## Cleaning up\n",
+        "\n",
+        "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
+        "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
+        "\n",
+        "Otherwise, you can delete the individual resources you created in this tutorial:"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "sx_vKniMq9ZX"
+      },
+      "outputs": [],
+      "source": [
+        "penguins_feature_store.delete(force=True)"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "name": "feature_store_streaming_ingestion_sdk.ipynb",
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

notebooks/official/matching_engine/intro-swivel.ipynb

@@ -81,6 +81,12 @@
         "\n",
         "In this notebook, you learn how to train custom embeddings using Vertex AI Pipelines and deploy the model for serving. \n",
         "\n",
+        "This tutorial uses the following Google Cloud ML services:\n",
+        "\n",
+        "- `Vertex AI Pipelines`\n",
+        "- `Vertex AI Training`\n",
+        "- `Swivel builtin algorithm`\n",
+        "\n",
         "The steps performed include:\n",
         "\n",
         "1. **Setup**: Importing the required libraries and setting your global variables.\n",

notebooks/official/matching_engine/sdk_matching_engine_for_indexing.ipynb

@@ -60,7 +60,7 @@
       "source": [
         "## Overview\n",
         "\n",
-        "This example demonstrates how to use the GCP ANN Service. It is a high scale, low latency solution, to find similar vectors (or more specifically \"embeddings\") for a large corpus. Moreover, it is a fully managed offering, further reducing operational overhead. It is built upon [Approximate Nearest Neighbor (ANN) technology](https://ai.googleblog.com/2020/07/announcing-scann-efficient-vector.html) developed by Google Research."
+        "This example demonstrates how to use the Vertex AI ANN Service. It is a high scale, low latency solution, to find similar vectors (or more specifically \"embeddings\") for a large corpus. Moreover, it is a fully managed offering, further reducing operational overhead. It is built upon [Approximate Nearest Neighbor (ANN) technology](https://ai.googleblog.com/2020/07/announcing-scann-efficient-vector.html) developed by Google Research."
       ]
     },
     {
@@ -73,6 +73,10 @@
         "\n",
         "In this notebook, you learn how to create Approximate Nearest Neighbor (ANN) Index, query against indexes, and validate the performance of the index. \n",
         "\n",
+        "This tutorial uses the following Google Cloud ML services:\n",
+        "\n",
+        "- `Vertex AI Matching Engine`\n",
+        "\n",
         "The steps performed include:\n",
         "\n",
         "* Create ANN Index and Brute Force Index\n",

notebooks/official/matching_engine/two-tower-model-introduction.ipynb

@@ -29,7 +29,7 @@
         "id": "JAPoU8Sm5E6e"
       },
       "source": [
-        "# Introduction to builtin Two-towers embedding algorithm\n",
+        "# Introduction to builtin Two-Towers embedding algorithm\n",
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
@@ -73,7 +73,13 @@
       "source": [
         "### Objective\n",
         "\n",
-        "In this notebook, you learn how to run the two-tower model.\n",
+        "In this notebook, you learn how to run the Two-Tower model.\n",
+        "\n",
+        "This tutorial uses the following Google Cloud ML services:\n",
+        "\n",
+        "- `Vertex AI Training`\n",
+        "- `Vertex AI Prediction`\n",
+        "- `Two-Tower builtin algorithm`\n",
         "\n",
         "The steps performed include:\n",
         "1. **Setup**: Importing the required libraries and setting your global variables.\n",

notebooks/official/ml_metadata/vertex-pipelines-ml-metadata.ipynb

@@ -74,6 +74,10 @@
         "\n",
         "In this notebook, you learn how to track artifacts and metrics with `Vertex ML Metadata` in `Vertex AI Pipeline` runs.\n",
         "\n",
+        "This tutorial uses the following Google Cloud ML services and resources:\n",
+        "\n",
+        "- Vertex AI Pipelines\n",
+        "- Vertex ML Metadata\n",
         "\n",
         "The steps performed include:\n",
         "\n",

notebooks/official/tabnet/ai-explanations-tabnet-algorithm.ipynb

@@ -76,6 +76,13 @@
         "\n",
         "In this tutorial, you learn how to provide a sample plotting tool to visualize the output of TabNet, which is helpful in explaining the algorithm. \n",
         "\n",
+        "\n",
+        "\n",
+        "This tutorial uses the following Google Cloud ML services and resources:\n",
+        "\n",
+        "- Vertex Explainable AI\n",
+        "- TabNet builtin algorithm\n",
+        "\n",
         "The steps performed are:\n",
         "* Setup the the project.\n",
         "* Download the prediction data of pretrain model onf Syn2 data.\n",

notebooks/official/tabnet/tabnet_vertex_tutorial.ipynb

@@ -78,6 +78,13 @@
         "\n",
         "In this notebook, you learn how to run TabNet model on Vertex AI.\n",
         "\n",
+        "This tutorial uses the following Google Cloud ML services and resources:\n",
+        "\n",
+        "- Vertex AI Training\n",
+        "- Vertex AI Hyperparameter Tuning\n",
+        "- TabNet builtin algorithm\n",
+        "- BigQuery\n",
+        "\n",
         "The steps performed are:\n",
         "1. **Setup**: Importing the required libraries and setting your global variables.\n",
         "2. **Configure parameters**: Setting the appropriate parameter values for the training job.\n",

notebooks/official/tabular_workflows/tabnet_on_vertex_pipelines.ipynb

@@ -97,7 +97,7 @@
         "### Dataset\n",
         "\n",
         "The dataset you will be using is [Bank Marketing](https://archive.ics.uci.edu/ml/datasets/bank+marketing).\n",
-        "The data is for direct marketing campaigns (phone calls) of a Portuguese banking institution. The binary classification goal is to predict if a client subscribe a term deposit. For this notebook, you randomly selected 90% of the rows in the original dataset and saved them in a train.csv file hosted on Cloud Storage. To download the file, click [here](https://storage.googleapis.com/cloud-samples-data/vertex-ai/tabular-workflows/datasets/bank-marketing/train.csv)."
+        "The data is for direct marketing campaigns (phone calls) of a Portuguese banking institution. The binary classification goal is to predict if a client subscribe a term deposit. For this notebook, you randomly selected 90% of the rows in the original dataset and saved them in a train.csv file hosted on Cloud Storage. To download the file, click [here](https://storage.googleapis.com/cloud-samples-data-us-central1/vertex-ai/tabular-workflows/datasets/bank-marketing/train.csv)."
       ]
     },
     {
@@ -746,7 +746,7 @@
       },
       "outputs": [],
       "source": [
-        "data_source_csv_filenames = \"gs://cloud-samples-data/vertex-ai/tabular-workflows/datasets/bank-marketing/train.csv\"\n",
+        "data_source_csv_filenames = \"gs://cloud-samples-data-us-central1/vertex-ai/tabular-workflows/datasets/bank-marketing/train.csv\"\n",
         "data_source_bigquery_table_path = (\n",
         "    None  # @param {type:\"string\"}, format: bq://bq_project.bq_dataset.bq_table\n",
         ")"

notebooks/official/tabular_workflows/wide_and_deep_on_vertex_pipelines.ipynb

@@ -97,7 +97,7 @@
         "### Dataset\n",
         "\n",
         "The dataset you will be using is [Bank Marketing](https://archive.ics.uci.edu/ml/datasets/bank+marketing).\n",
-        "The data is for direct marketing campaigns (phone calls) of a Portuguese banking institution. The binary classification goal is to predict if a client will subscribe a term deposit. For this notebook, we randomly selected 90% of the rows in the original dataset and saved them in a train.csv file hosted on Cloud Storage. To download the file, click [here](https://storage.googleapis.com/cloud-samples-data/vertex-ai/tabular-workflows/datasets/bank-marketing/train.csv)."
+        "The data is for direct marketing campaigns (phone calls) of a Portuguese banking institution. The binary classification goal is to predict if a client will subscribe a term deposit. For this notebook, we randomly selected 90% of the rows in the original dataset and saved them in a train.csv file hosted on Cloud Storage. To download the file, click [here](https://storage.googleapis.com/cloud-samples-data-us-central1/vertex-ai/tabular-workflows/datasets/bank-marketing/train.csv)."
       ]
     },
     {
@@ -667,7 +667,7 @@
       },
       "outputs": [],
       "source": [
-        "data_source_csv_filenames = \"gs://cloud-samples-data/vertex-ai/tabular-workflows/datasets/bank-marketing/train.csv\"\n",
+        "data_source_csv_filenames = \"gs://cloud-samples-data-us-central1/vertex-ai/tabular-workflows/datasets/bank-marketing/train.csv\"\n",
         "data_source_bigquery_table_path = (\n",
         "    None  # @param {type:\"string\"}, format: bq://bq_project.bq_dataset.bq_table\n",
         ")"

notebooks/official/tensorboard/tensorboard_custom_training_with_custom_container.ipynb

@@ -29,7 +29,7 @@
         "id": "ed2pOXQMb8fY"
       },
       "source": [
-        "# Vertex AI TensorBoard Custom Training with Custom Container\n",
+        "# Vertex AI TensorBoard Custom Training with custom container\n",
         "\n",
         "<table align=\"left\">\n",
         "  <td>\n",
@@ -102,6 +102,11 @@
         "\n",
         "In this tutorial, you learn how to create a custom training job using custom containers, and monitor your training process on Vertex AI TensorBoard in near real time.\n",
         "\n",
+        "This tutorial uses the following Google Cloud ML services and resources:\n",
+        "\n",
+        "- Vertex AI Training\n",
+        "- Vertex AI TensorBoard\n",
+        "\n",
         "The steps performed include:\n",
         "\n",
         "* Create docker repository & config.\n",

notebooks/official/tensorboard/tensorboard_custom_training_with_prebuilt_container.ipynb

@@ -102,6 +102,11 @@
         "\n",
         "In this tutorial, you learn how to create a custom training job using prebuilt containers, and monitor your training process on Vertex AI TensorBoard in near real time.\n",
         "\n",
+        "This tutorial uses the following Google Cloud ML services and resources:\n",
+        "\n",
+        "- Vertex AI Training\n",
+        "- Vertex AI TensorBoard\n",
+        "\n",
         "The steps performed include:\n",
         "\n",
         "* Setup service account and Google Cloud Storage buckets.\n",

notebooks/official/tensorboard/tensorboard_profiler_custom_training.ipynb

@@ -29,7 +29,7 @@
         "id": "l2mMvIUG9meX"
       },
       "source": [
-        "# Profile model training performance using Profiler\n",
+        "# Profile model training performance using Vertex AI TensorBoard Profiler\n",
         "\n",
         "<table align=\"left\">\n",
         "\n",
@@ -49,7 +49,7 @@
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
         "    </a>\n",
-        "  </td>                                                                                               \n",
+        "  </td>\n",
         "</table>"
       ]
     },
@@ -331,22 +331,16 @@
         "! gcloud projects add-iam-policy-binding $PROJECT_ID \\\n",
         "        --member=\"serviceAccount:$SERVICE_ACCOUNT\" \\\n",
         "        --role=\"roles/storage.admin\" \\\n",
-        "        --quiet"
-      ]
-    },
-    {
-      "cell_type": "code",
-      "execution_count": null,
-      "metadata": {
-        "id": "S_8_5jm-Gk6w"
-      },
-      "outputs": [],
-      "source": [
+        "        --quiet\n",
+        "\n",
         "# Grant AI Platform permission.\n",
         "! gcloud projects add-iam-policy-binding $PROJECT_ID \\\n",
         "        --member=\"serviceAccount:$SERVICE_ACCOUNT\" \\\n",
         "        --role=\"roles/aiplatform.user\" \\\n",
-        "        --quiet"
+        "        --quiet\n",
+        "\n",
+        "! gcloud projects get-iam-policy $PROJECT_ID \\\n",
+        "        --filter=bindings.members:serviceAccount:$SERVICE_ACCOUNT"
       ]
     },
     {
@@ -768,7 +762,7 @@
         "WORKDIR /root\n",
         "\n",
         "# Installs additional packages as you need.\n",
-        "RUN pip3 install google-cloud-aiplatform[cloud_profiler]\n",
+        "RUN pip3 install google-cloud-aiplatform[cloud_profiler]>=1.19.1\n",
         "\n",
         "# Copies the trainer code to the docker image.\n",
         "RUN mkdir /root/trainer\n",
@@ -798,7 +792,7 @@
         "IMAGE_NAME = \"tensorboard-custom-container\"\n",
         "IMAGE_URI = f\"{REGION}-docker.pkg.dev/{PROJECT_ID}/{DOCKER_REPOSITORY}/{IMAGE_NAME}\"\n",
         "\n",
-        "! gcloud builds submit --project {PROJECT_ID} --region={REGION} --tag {IMAGE_URI} --timeout=60m --quiet"
+        "! gcloud builds submit --project {PROJECT_ID} --region={REGION} --tag {IMAGE_URI} --timeout=3600s --quiet"
       ]
     },
     {

notebooks/official/tensorboard/tensorboard_vertex_ai_pipelines_integration.ipynb

@@ -101,6 +101,12 @@
         "\n",
         "In this tutorial, you learn how to create a training pipeline using the KFP SDK, execute the pipeline in Vertex AI Pipelines, and monitor your training process on Vertex AI TensorBoard in near real time.\n",
         "\n",
+        "This tutorial uses the following Google Cloud ML services and resources:\n",
+        "\n",
+        "- Vertex AI Training\n",
+        "- Vertex AI TensorBoard\n",
+        "- Vertex AI Pipelines\n",
+        "\n",
         "The steps performed include:\n",
         "\n",
         "* Setup a service account and Google Cloud Storage buckets.\n",

notebooks/official/workbench/README.md

@@ -0,0 +1,200 @@
+
+### sentiment_analysis 
+
+
+[Sentiment Analysis using AutoML Natural Language and Vertex AI](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/sentiment_analysis/Sentiment_Analysis.ipynb)
+
+```
+<a name="section-2"></a>
+
+
+
+```
+
+
+### exploratory_data_analysis 
+
+
+[Interactive exploratory analysis of BigQuery data in a notebook](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/exploratory_data_analysis/explore_data_in_bigquery_with_workbench.ipynb)
+
+```
+Learn about various ways to explore and gain insights from BigQuery data in a Jupyter notebook environment.
+
+The steps performed include:
+
+- Using Python & SQL to query public data in BigQuery
+- Exploring the dataset using BigQuery INFORMATION_SCHEMA
+- Creating interactive elements to help explore interesting parts of the data
+- Doing some exploratory correlation and time series analysis
+- Creating static and interactive outputs (data tables and plots) in the notebook
+- Saving some outputs to Cloud Storage
+
+```
+
+
+### demand_forecasting 
+
+
+[Forecasting retail demand with Vertex AI and BigQuery ML](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/demand_forecasting/forecasting-retail-demand.ipynb)
+
+```
+Learn how to build ARIMA (Autoregressive integrated moving average) model from BigQuery ML on retail data
+
+The steps performed include:
+
+* Explore data
+* Model with BigQuery and the ARIMA model
+* Evaluate the model
+* Evaluate the model results using BigQuery ML (on training data)
+* Evalute the model results - MAE, MAPE, MSE, RMSE (on test data)
+* Use the executor feature
+
+```
+
+
+### predictive_maintainance 
+
+
+[Predictive Maintenance using Vertex AI](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/predictive_maintainance/predictive_maintenance_usecase.ipynb)
+
+```
+<a name="section-2"></a>
+
+
+
+```
+
+
+### subscriber_churn_prediction 
+
+
+[Telecom subscriber churn prediction on Vertex AI](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/subscriber_churn_prediction/telecom-subscriber-churn-prediction.ipynb)
+
+```
+This tutorial shows you how to do exploratory data analysis, preprocess data, train, deploy and get predictions from a churn prediction model on a tabular churn dataset.
+
+The steps performed include:
+
+- Load data from a Cloud Storage path
+- Perform exploratory data analysis (EDA)
+- Preprocess the data
+- Train a scikit-learn model
+- Evaluate the scikit-learn model
+- Save the model to a Cloud Storage path
+- Create a model and an endpoint in Vertex AI
+- Deploy the trained model to an endpoint
+- Generate predictions and explanations on test data from the hosted model
+- Undeploy the model resource
+
+```
+
+
+### spark 
+
+
+[SparkML with Dataproc and BigQuery](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/spark/spark_ml.ipynb)
+
+```
+This tutorial runs an Apache SparkML job that fetches data from the BigQuery dataset, performs exploratory data analysis, cleans the data, executes feature engineering, trains the model, evaluates the model, outputs results, and saves the model to a Cloud Storage bucket.
+
+
+
+```
+
+
+[Digest and analyze data from BigQuery with Dataproc](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/spark/spark_bigquery.ipynb)
+
+```
+This notebook tutorial runs an Apache Spark job that fetches data from the BigQuery "GitHub Activity Data" dataset, queries the data, and then writes the results back to BigQuery.
+
+
+
+```
+
+
+### ads_targetting 
+
+
+[Train a multi-class classification model for ads-targeting](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/ads_targetting/training-multi-class-classification-model-for-ads-targeting-usecase.ipynb)
+
+```
+Learn how to collect data from BigQuery, preprocess it, and train a multi-class classification model on an e-commerce dataset.
+
+The steps performed include:
+
+- Fetch the required data from BigQuery
+- Preprocess the data
+- Train a TensorFlow (>=2.4) classification model
+- Evaluate the loss for the trained model
+- Automate the notebook execution using the executor feature
+- Save the model to a Cloud Storage path
+- Clean up the created resources
+
+```
+
+
+### inventory-prediction 
+
+
+[Inventory prediction on ecommerce data using Vertex AI](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/inventory-prediction/inventory_prediction.ipynb)
+
+```
+This tutorial shows you how to do exploratory data analysis, preprocess data, train model, evaluate model, deploy model, configure What-If Tool.
+
+The steps performed include:
+
+* Load the dataset from BigQuery using the "BigQuery in Notebooks" integration.
+* Analyze the dataset.
+* Preprocess the features in the dataset.
+* Build a random forest classifier model that predicts whether a product will get sold in the next 60 days.
+* Evaluate the model.
+* Deploy the model using Vertex AI.
+* Configure and test with the What-If Tool.
+
+```
+
+
+### fraud_detection 
+
+
+[Build a fraud detection model on Vertex AI](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/fraud_detection/fraud-detection-model.ipynb)
+
+```
+This tutorial demonstrates data analysis and model-building using a synthetic financial dataset.
+
+The steps performed include:
+
+- Installation of required libraries
+- Reading the dataset from a Cloud Storage bucket
+- Performing exploratory analysis on the dataset
+- Preprocessing the dataset
+- Training a random forest model using scikit-learn
+- Saving the model to a Cloud Storage bucket
+- Creating a Vertex AI model resource and deploying to an endpoint
+- Running the What-If Tool on test data
+- Un-deploying the model and cleaning up the model resources
+
+```
+
+
+### chicago_taxi_fare_prediction 
+
+
+[Taxi fare prediction using the Chicago Taxi Trips dataset](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/chicago_taxi_fare_prediction/chicago_taxi_fare_prediction.ipynb)
+
+```
+The goal of this notebook is to provide an overview on the latest Vertex AI features like **Explainable AI** and **BigQuery in Notebooks** by trying to solve a taxi fare prediction problem.
+
+The steps performed include:
+
+- Loading the dataset using "BigQuery in Notebooks".
+- Performing exploratory data analysis on the dataset.
+- Feature selection and preprocessing.
+- Building a linear regression model using scikit-learn.
+- Configuring the model for Vertex Explainable AI.
+- Deploying the model to Vertex AI.
+- Testing the deployed model.
+- Clean up.
+
+```
+

notebooks/official/workbench/exploratory_data_analysis/explore_data_in_bigquery_with_workbench.ipynb

@@ -56,13 +56,20 @@
     {
       "cell_type": "markdown",
       "metadata": {
-        "id": "tvgnzT1CKxrO"
+        "id": "780762457db0"
       },
       "source": [
         "## Overview\n",
         "\n",
-        "This notebook is written for data analysts and data scientists who have data in BigQuery and want to perform exploratory data analysis to gather insights from that data in an interactive environment.\n",
-        "\n",
+        "This notebook is written for data analysts and data scientists who have data in BigQuery and want to perform exploratory data analysis to gather insights from that data in an interactive environment."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tvgnzT1CKxrO"
+      },
+      "source": [
         "### Objective\n",
         "\n",
         "In this tutorial, you learn about various ways to explore and gain insights from BigQuery data in a Jupyter notebook environment.\n",
@@ -80,12 +87,26 @@
         "- Creating interactive elements to help explore interesting parts of the data\n",
         "- Doing some exploratory correlation and time series analysis\n",
         "- Creating static and interactive outputs (data tables and plots) in the notebook\n",
-        "- Saving some outputs to Cloud Storage\n",
-        "\n",
+        "- Saving some outputs to Cloud Storage"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "65aa4ba05101"
+      },
+      "source": [
         "### Dataset\n",
         "\n",
-        "The dataset, [available publicly on BigQuery](https://console.cloud.google.com/bigquery?project=bigquery-public-data&d=ga4_obfuscated_sample_ecommerce&p=bigquery-public-data&page=dataset), comes from obfuscated [Google Analytics 4 (GA4) data](https://developers.google.com/analytics/bigquery/web-ecommerce-demo-dataset) from the Google Merchandise Store.\n",
-        "\n",
+        "The dataset, [available publicly on BigQuery](https://console.cloud.google.com/bigquery?project=bigquery-public-data&d=ga4_obfuscated_sample_ecommerce&p=bigquery-public-data&page=dataset), comes from obfuscated [Google Analytics 4 (GA4) data](https://developers.google.com/analytics/bigquery/web-ecommerce-demo-dataset) from the Google Merchandise Store."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "fd44a67414d5"
+      },
+      "source": [
         "### Costs \n",
         "\n",
         "This tutorial uses billable components of Google Cloud:\n",

notebooks/official/workbench/gaming_churn_prediction/churn_prediction_for_game_developers.ipynb

@@ -29,7 +29,7 @@
         "id": "py8EYwG_91Pn"
       },
       "source": [
-        "# Churn prediction for game developers using Google Analytics 4 (GA4) and BigQuery ML\n",
+        "# Churn prediction for game developers using Google Analytics 4 and BigQuery ML\n",
         "\n",
         "<table align=\"left\">\n",
         "\n",
@@ -99,7 +99,7 @@
         "id": "sH0CZGku0BPp"
       },
       "source": [
-        "## Objective\n",
+        "### Objective\n",
         "<a name=\"section-2\"></a>\n",
         "\n",
         "In this tutorial, you learn how to train, evaluate a propensity model in BigQuery ML.\n",
@@ -122,7 +122,7 @@
         "id": "b07fc9940120"
       },
       "source": [
-        "## Dataset\n",
+        "### Dataset\n",
         "<a name=\"section-3\"></a>\n",
         "\n",
         "This notebook uses [this public BigQuery dataset](https://console.cloud.google.com/bigquery?p=firebase-public-project&d=analytics_153293282&t=events_20181003&page=table), which contains raw event data from a real mobile gaming app called Flood It! ([Android app](https://play.google.com/store/apps/details?id=com.labpixies.flood), [iOS app](https://itunes.apple.com/us/app/flood-it!/id476943146?mt=8)). The [data schema](https://support.google.com/analytics/answer/7029846) originates from Google Analytics for Firebase, but is the same schema as [Google Analytics 4](https://support.google.com/analytics/answer/9358801); the techniques in this notebook can be applied to either Google Analytics for Firebase or Google Analytics 4 data.\n",
@@ -138,7 +138,7 @@
         "id": "589ffe790261"
       },
       "source": [
-        "## Costs\n",
+        "### Costs\n",
         "<a name=\"section-4\"></a>\n",
         "\n",
         "This tutorial uses the following billable components of Google Cloud:\n",

notebooks/official/workbench/predictive_maintainance/predictive_maintenance_usecase.ipynb

@@ -93,7 +93,12 @@
         "### Objective\n",
         "<a name=\"section-2\"></a>\n",
         "\n",
-        "The objectives of this notebook include:\n",
+        "This tutorial uses the following Google Cloud ML services:\n",
+        "\n",
+        "- `Vertex AI Training`\n",
+        "- `Vertex AI Model Evaluation`\n",
+        "\n",
+        "The steps performed are:\n",
         "\n",
         "- Loading the required dataset from a Cloud Storage bucket.\n",
         "- Analyzing the fields present in the dataset.\n",

notebooks/official/workbench/pricing_optimization/pricing-optimization.ipynb

@@ -94,7 +94,7 @@
         "id": "71f69cfdff2b"
       },
       "source": [
-        "## Objective\n",
+        "### Objective\n",
         "<a name=\"section-2\"></a>\n",
         "\n",
         "The objective of this notebook is to build a pricing optimization model using BigQuery ML. The following steps have been followed:  \n",
@@ -122,7 +122,7 @@
         "id": "d20422a5c34d"
       },
       "source": [
-        "## Dataset\n",
+        "### Dataset\n",
         "<a name=\"section-3\"></a>\n",
         "\n",
         "The dataset used in this notebook is a part of the [CDM Pricing dataset](https://github.com/trifacta/trifacta-google-cloud/blob/main/design-pattern-pricing-optimization/CDM_Pricing_large_table.csv), which consists of product sales information on specified dates."
@@ -134,7 +134,7 @@
         "id": "c05bcd30859d"
       },
       "source": [
-        "## Costs\n",
+        "### Costs\n",
         "<a name=\"section-4\"></a>\n",
         "\n",
         "This tutorial uses the following billable components of Google Cloud:\n",

notebooks/official/workbench/sentiment_analysis/Sentiment_Analysis.ipynb

@@ -97,13 +97,20 @@
         "### Objective\n",
         "<a name=\"section-2\"></a>\n",
         "\n",
-        "The objectives of this notebook include:\n",
+        "This tutorial uses the following Google Cloud ML services:\n",
+        "\n",
+        "- `Vertex AI Dataset`\n",
+        "- `Vertex AI Model`\n",
+        "- `AutoML training`\n",
+        "- `Vertex AI Prediction`\n",
+        "\n",
+        "The steps performed are:\n",
         "\n",
         "- Loading the required data. \n",
         "- Preprocessing the data.\n",
         "- Selecting the required data for the model.\n",
         "- Loading the dataset into Vertex AI managed datasets.\n",
-        "- Training a sentiment model using AutoML Natural Language.\n",
+        "- Training a sentiment model using AutoML Text training.\n",
         "- Evaluating the model.\n",
         "- Deploying the model on Vertex AI.\n",
         "- Getting predictions.\n",

notebooks/official/workbench/spark/spark_bigquery.ipynb

@@ -29,6 +29,8 @@
         "id": "JAPoU8Sm5E6e"
       },
       "source": [
+        "# Digest and analyze data from BigQuery with Dataproc\n",
+        "\n",
         "<table align=\"left\">\n",
         "\n",
         "  <td>\n",
@@ -38,6 +40,11 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
+        "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/spark/spark_bigquery.ipynb\">\n",
+        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "        </a>\n",
+        "  </td>\n",
+        "  <td>\n",
         "<a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/workbench/spark/spark_bigquery.ipynb\" target='_blank'>\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
@@ -59,17 +66,6 @@
         "To run this notebook, click the link `Open in Vertex AI Workbench` above."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "tvgnzT1CKxrO"
-      },
-      "source": [
-        "### Dataset\n",
-        "\n",
-        "The [GitHub Activity Data](https://console.cloud.google.com/marketplace/product/github/github-repos) dataset is available in [BigQuery Public Datasets](https://cloud.google.com/bigquery/public-data), and provides free querying of up to 1TB of data each month. It contains data on two different types of repositories: \"polyglot\" repos, which support multiple programming language files, and \"monoglot\" repos, which support one programming language."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -80,7 +76,12 @@
         "\n",
         "This notebook tutorial runs an Apache Spark job that fetches data from the BigQuery \"GitHub Activity Data\" dataset, queries the data, and then writes the results back to BigQuery. This job sequence represents a common data engineering use case: ingesting, transforming, and querying data, and then writing the output to a database. It also demonstrates how to submit an Apache Spark job to Dataproc.\n",
         "\n",
-        "This notebook tutorial performs the following steps:\n",
+        "This tutorial uses the following Google Cloud ML services:\n",
+        "\n",
+        "- `Dataproc`\n",
+        "- `BigQuery`\n",
+        "\n",
+        "The steps performed are:\n",
         "\n",
         "- Setting up a Google Cloud project and Dataproc cluster.\n",
         "- Configuring the spark-bigquery-connector.\n",
@@ -93,6 +94,17 @@
         "- Deleting the resources created for this notebook tutorial."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "tvgnzT1CKxrO"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The [GitHub Activity Data](https://console.cloud.google.com/marketplace/product/github/github-repos) dataset is available in [BigQuery Public Datasets](https://cloud.google.com/bigquery/public-data), and provides free querying of up to 1TB of data each month. It contains data on two different types of repositories: \"polyglot\" repos, which support multiple programming language files, and \"monoglot\" repos, which support one programming language."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {

notebooks/official/workbench/spark/spark_ml.ipynb

@@ -29,6 +29,8 @@
         "id": "XoEqT2Y4DJmf"
       },
       "source": [
+        "# SparkML with Dataproc and BigQuery\n",
+        "\n",
         "<table align=\"left\">\n",
         "\n",
         "  <td>\n",
@@ -38,6 +40,11 @@
         "    </a>\n",
         "  </td>\n",
         "  <td>\n",
+        "        <a href=\"https://colab.research.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/workbench/spark/spark_ml.ipynb\">\n",
+        "        <img src=\"https://cloud.google.com/ml-engine/images/colab-logo-32px.png\" alt=\"Colab logo\"> Run in Colab\n",
+        "        </a>\n",
+        "  </td>\n",
+        "  <td>\n",
         "<a href=\"https://console.cloud.google.com/vertex-ai/workbench/deploy-notebook?download_url=https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/main/notebooks/official/workbench/spark/spark_ml.ipynb\" target='_blank'>\n",
         "      <img src=\"https://lh3.googleusercontent.com/UiNooY4LUgW_oTvpsNhPpQzsstV5W8F7rYgxgGBD85cWJoLmrOzhVs_ksK_vgx40SHs7jCqkTkCk=e14-rj-sc0xffffff-h130-w32\" alt=\"Vertex AI logo\">\n",
         "      Open in Vertex AI Workbench\n",
@@ -57,17 +64,6 @@
         "This notebook tutorial runs Apache SparkML jobs with Dataproc and BigQuery to exemplify a common machine learning pipeline use case: data ingestion and cleaning, feature engineering, modeling, and model evaluation."
       ]
     },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "XoEqT2Y4DJmf"
-      },
-      "source": [
-        "### Dataset\n",
-        "\n",
-        "The [NYC TLC (Taxi and Limousine Commission) Trips](https://console.cloud.google.com/marketplace/product/city-of-new-york/nyc-tlc-trips) (New York taxi and limosine trips data) and [NYC Citi Bike Trips](https://console.cloud.google.com/marketplace/product/city-of-new-york/nyc-citi-bike) (NYC public bicycle sharing system data) datasets are available in [BigQuery Public Datasets](https://cloud.google.com/bigquery/public-data). BigQuery provides free querying of up to 1TB of data each month."
-      ]
-    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -78,7 +74,13 @@
         "\n",
         "This tutorial runs an Apache SparkML job that fetches data from the BigQuery dataset, performs exploratory data analysis, cleans the data, executes feature engineering, trains the model, evaluates the model, outputs results, and saves the model to a Cloud Storage bucket.\n",
         "\n",
-        "This notebook tutorial performs the following steps:\n",
+        "This tutorial uses the following Google Cloud ML services:\n",
+        "\n",
+        "- `Dataproc`\n",
+        "- `BigQuery`\n",
+        "- `Vertex AI Training`\n",
+        "\n",
+        "The steps performed are:\n",
         "\n",
         "- Sets up a Google Cloud project and Dataproc cluster.\n",
         "- Creates a Cloud Storage bucket and a BigQuery dataset.\n",
@@ -94,6 +96,17 @@
         "- Deletes the resources created for the tutorial."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "XoEqT2Y4DJmf"
+      },
+      "source": [
+        "### Dataset\n",
+        "\n",
+        "The [NYC TLC (Taxi and Limousine Commission) Trips](https://console.cloud.google.com/marketplace/product/city-of-new-york/nyc-tlc-trips) (New York taxi and limosine trips data) and [NYC Citi Bike Trips](https://console.cloud.google.com/marketplace/product/city-of-new-york/nyc-citi-bike) (NYC public bicycle sharing system data) datasets are available in [BigQuery Public Datasets](https://cloud.google.com/bigquery/public-data). BigQuery provides free querying of up to 1TB of data each month."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {