Update automl-tabular-classification.ipynb

* fix: alpha sort

* fix: alpha sort

* fix: alpha sort

.cloud-build/execute_changed_notebooks_helper.py

@@ -245,7 +245,7 @@ def process_and_execute_notebook(
         result.logs_bucket = operation_metadata.build.logs_bucket
 
         # Block and wait for the result
-        operation_result = operation.result()
+        operation_result = operation.result(timeout=timeout_in_seconds)
 
         result.duration = datetime.datetime.now() - time_start
         result.is_pass = True

.cloud-build/requirements.txt

@@ -10,4 +10,5 @@ google-cloud-aiplatform
 google-cloud-storage
 google-cloud-build
 ratemate
-GitPython
+GitPython
+google-api-core==2.10

community-content/CODEOWNERS

@@ -8,3 +8,4 @@
 /cpr-examples @samthrasher
 /Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines @Ark-kun
 /pipeline_components @Ark-kun
+/pipeline_components/image_ml_model_training @lakeyk

community-content/Train_tabular_models_with_many_frameworks_and_import_to_Vertex_AI_using_Pipelines/Train_tabular_classification_logistic_regression_model_using_Scikit_learn_and_import_to_Vertex_AI/pipeline.py

@@ -6,8 +6,8 @@ download_from_gcs_op = components.load_component_from_url("https://raw.githubuse
 select_columns_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Select_columns/in_CSV_format/component.yaml")
 fill_all_missing_values_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Fill_all_missing_values/in_CSV_format/component.yaml")
 binarize_column_using_Pandas_on_CSV_data_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/pandas/Binarize_column/in_CSV_format/component.yaml")
-train_logistic_regression_model_using_scikit_learn_from_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/ML_frameworks/Scikit_learn/Train_logistic_regression_model/from_CSV/component.yaml")
-upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/1f5cf6e06409b704064b2086c0a705e4e6b4fcde/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/component.yaml")
+train_logistic_regression_model_using_scikit_learn_from_CSV_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/1f5cf6e06409b704064b2086c0a705e4e6b4fcde/community-content/pipeline_components/ML_frameworks/Scikit_learn/Train_logistic_regression_model/from_CSV/component.yaml")
+upload_Scikit_learn_pickle_model_to_Google_Cloud_Vertex_AI_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_Scikit-learn_pickle_model/component.yaml")
 deploy_model_to_endpoint_op = components.load_component_from_url("https://raw.githubusercontent.com/GoogleCloudPlatform/vertex-ai-samples/399405402d95f4a011e2d2e967c96f8508ba5688/community-content/pipeline_components/google-cloud/Vertex_AI/Models/Deploy_to_endpoint/component.yaml")
 
 # %% Pipeline definition

community-content/pipeline_components/image_ml_model_training/load_image_classification_model/component.yaml

@@ -0,0 +1,112 @@
+name: Load image classification model from tfhub
+description: |
+  Loads specified model from TFHub, creates layer to receive additional (3 channel) imagery data.
+  Args:
+    class_names (Sequence[str]):
+        Sequence of strings of categories for classification corresponding to input data.
+    loaded_model_path (str):
+        Output path for the loaded model.
+    image_size_path (str):
+        Output path for the model expected image size.
+    model_name (Optional[str]):
+        Name of the pre-trained image classification model to load from TFHub.
+        Eligible model_name:
+        - efficientnetv2-s
+        - efficientnetv2-m
+        - efficientnetv2-l
+        - efficientnetv2-s-21k
+        - efficientnetv2-m-21k
+        - efficientnetv2-l-21k
+        - efficientnetv2-xl-21k
+        - efficientnetv2-b0-21k
+        - efficientnetv2-b1-21k
+        - efficientnetv2-b2-21k
+        - efficientnetv2-b3-21k
+        - efficientnetv2-s-21k-ft1k
+        - efficientnetv2-m-21k-ft1k
+        - efficientnetv2-l-21k-ft1k
+        - efficientnetv2-xl-21k-ft1k
+        - efficientnetv2-b0-21k-ft1k
+        - efficientnetv2-b1-21k-ft1k
+        - efficientnetv2-b2-21k-ft1k
+        - efficientnetv2-b3-21k-ft1k
+        - efficientnetv2-b0
+        - efficientnetv2-b1
+        - efficientnetv2-b2
+        - efficientnetv2-b3
+        - efficientnet_b0
+        - efficientnet_b1
+        - efficientnet_b2
+        - efficientnet_b3
+        - efficientnet_b4
+        - efficientnet_b5
+        - efficientnet_b6
+        - efficientnet_b7
+        - bit_s-r50x1
+        - inception_v3
+        - inception_resnet_v2
+        - resnet_v1_50
+        - resnet_v1_101
+        - resnet_v1_152
+        - resnet_v2_50
+        - resnet_v2_101
+        - resnet_v2_152
+        - nasnet_large
+        - nasnet_mobile
+        - pnasnet_large
+        - mobilenet_v2_100_224
+        - mobilenet_v2_130_224
+        - mobilenet_v2_140_224
+        - mobilenet_v3_small_100_224
+        - mobilenet_v3_small_075_224
+        - mobilenet_v3_large_100_224
+        - mobilenet_v3_large_075_224
+    dropout_rate (Optional[float]):
+        Fraction of input units to drop in the last layer. Value should be between 0.0 and 1.0.
+    trainable (Optional[bool]):
+        If true fine tuning will be performed on entire Hub model. If false only additional
+        layers will be trained.
+    l2_regularization_penalty (Optional[float]):
+        l2 regularization penalty.
+inputs:
+- {name: class_names, type: 'typing.List[str]', description: List of class names corresponding
+    to the input image data}
+- {name: model_name, type: String, description: Name of the TFHub model to load, default: efficientnetv2-xl-21k,
+  optional: true}
+- {name: dropout_rate, type: Float, description: Dropout rate, default: '0.2', optional: true}
+- name: trainable
+  type: Boolean
+  description: True if fine tuning should be performed
+  default: "True"
+  optional: true
+- {name: l2_regularization_penalty, type: Float, description: Regularization penalty,
+  default: '0.0001', optional: true}
+outputs:
+- {name: loaded_model_path, type: TensorflowSavedModel, description: Output path for
+    the loaded model}
+- {name: image_size_path, type: HeightWidth}
+implementation:
+  container:
+    image: us-docker.pkg.dev/vertex-ai/ready-to-go-image-classification/image-components:v0.1
+    # command is a list of strings (command-line arguments).
+    # The YAML language has two syntaxes for lists and you can use either of them.
+    # Here we use the "flow syntax" - comma-separated strings inside square brackets.
+    command: [
+      python3,
+      # Path of the program inside the container
+      /pipelines/component/src/loading_component.py,
+      --loaded-model-path,
+      {outputPath: loaded_model_path},
+      --class-names,
+      {inputValue: class_names},
+      --model-name,
+      {inputValue: model_name},
+      --dropout-rate,
+      {inputValue: dropout_rate},
+      --trainable,
+      {inputValue: trainable},
+      --l2-regularization-penalty,
+      {inputValue: l2_regularization_penalty},
+      --image-size-path,
+      {outputPath: image_size_path},
+    ]

community-content/pipeline_components/image_ml_model_training/preprocess_image_data/component.yaml

@@ -0,0 +1,57 @@
+name: Preprocess image data
+description: |
+  Preprocess the image data and split between train and validation.
+  Args:
+    input_data_path (str):
+        Input path for the TFRecord image data. Data will be formatted as 'label' (encoded image
+        label), and 'image_raw' (the binary string of the image data).
+    height_width_path (str):
+        Path to square height and width to resize images to. File should contain single float value.
+        Value is dependent on training model.
+    preprocessed_training_data_path (str):
+        Output path for the TFRecord training data. Data will be formatted as 'label' (encoded image
+        label), and 'image_raw' (the binary string of the image data).
+    preprocessed_validation_data_path (str):
+        Output path for the TFRecord validation data. Data will be formatted as 'label' (encoded
+        image label), and 'image_raw' (the binary string of the image data).
+    validation_split (Optional[float]):
+        Fraction of data that will make up validation dataset. Value should be between 0.0 and 1.0.
+    seed (Optional[int]):
+        The global random seed to ensure the system gets a unique random sequence
+        that is deterministic (https://www.tensorflow.org/api_docs/python/tf/random/set_seed).
+inputs:
+- {name: input_data_path, type: ImageDatasetTFRecord, description: 'Input path for
+    the TFRecord image data,'}
+- {name: height_width_path, type: HeightWidth, description: 'Path to square height and width to
+    resize images to,'}
+- {name: validation_split, type: Float, description: 'Fraction of data that will make
+    up validation dataset,', default: '0.2', optional: true}
+- {name: seed, type: Integer, description: Random seed, default: '0', optional: true}
+outputs:
+- {name: preprocessed_training_data_path, type: ImageDatasetTFRecord, description: 'Output
+    path for the training data,'}
+- {name: preprocessed_validation_data_path, type: ImageDatasetTFRecord, description: 'Output
+    path for the validation data,'}
+implementation:
+  container:
+    image: us-docker.pkg.dev/vertex-ai/ready-to-go-image-classification/image-components:v0.1
+    # command is a list of strings (command-line arguments).
+    # The YAML language has two syntaxes for lists and you can use either of them.
+    # Here we use the "flow syntax" - comma-separated strings inside square brackets.
+    command: [
+      python3,
+      # Path of the program inside the container
+      /pipelines/component/src/preprocessing_component.py,
+      --input-data-path,
+      {inputPath: input_data_path},
+      --height-width-path,
+      {inputPath: height_width_path},
+      --validation-split,
+      {inputValue: validation_split},
+      --seed,
+      {inputValue: seed},
+      --preprocessed-training-data-path,
+      {outputPath: preprocessed_training_data_path},
+      --preprocessed-validation-data-path,
+      {outputPath: preprocessed_validation_data_path},
+    ]

community-content/pipeline_components/image_ml_model_training/train_image_classification_model/component.yaml

@@ -0,0 +1,90 @@
+name: Train tensorflow image classification model
+description: |
+  Creates a trained image classification TensorFlow model.
+  Args:
+    preprocessed_training_data_path (str):
+        Input path to the TFRecord training data. Data will be formatted as 'label' (encoded image
+        label), and 'image_raw' (the binary string of the image data).
+    preprocessed_validation_data_path (str):
+        Input path to the TFRecord validation data. Data will be formatted as 'label' (encoded
+        image label), and 'image_raw' (the binary string of the image data).
+    model_path (str):
+        Input path to the loaded pre-trained model.
+    trained_model_path (str):
+        Output path to save the trained model to.
+    optimizer_name (Optional[str]):
+        Name of the tf.keras optimizer. Available optimizers are listed at
+        https://keras.io/api/optimizers/
+    optimizer_parameters (Optional[Dict[str, str]]):
+        Optimizer parameters.
+    loss_function_name (Optional[str]):
+        Name of the loss function.
+    loss_function_parameters (Optional[Dict[str, str]]):
+        Loss function parameters.
+    number_of_epochs (Optional[int]):
+        Number of training iterations over data.
+    metric_names (Optional[Sequence[str]]):
+        List of tf.keras.metrics to be evaluated by the model during training and testing. Available
+        metrics are listed at https://keras.io/api/metrics/.
+    seed Optional(int):
+        The global random seed to ensure the system gets a unique random sequence
+        that is deterministic (https://www.tensorflow.org/api_docs/python/tf/random/set_seed).
+inputs:
+- {name: preprocessed_training_data_path, type: ImageDatasetTFRecord, description: 'Input
+    path for the training data,'}
+- {name: preprocessed_validation_data_path, type: ImageDatasetTFRecord, description: 'Input
+    path for the validation data,'}
+- {name: model_path, type: TensorflowSavedModel, description: 'Input path for the
+    model,'}
+- {name: optimizer_name, type: String, description: 'Name of the optimizer,', default: SGD,
+  optional: true}
+- {name: optimizer_parameters, type: 'typing.Dict[str, str]', description: 'Optimizer
+    parameters,', default: '{}', optional: true}
+- {name: loss_function_name, type: String, description: 'Name of the loss function,',
+  default: CategoricalCrossentropy, optional: true}
+- {name: loss_function_parameters, type: 'typing.Dict[str, str]', description: 'Loss
+    function parameters,', default: '{}', optional: true}
+- {name: number_of_epochs, type: Integer, description: 'Number of epochs,', default: '10',
+  optional: true}
+- {name: metric_names, type: 'typing.List[str]', description: 'List of metrics to
+    use,', default: '["accuracy"]', optional: true}
+- {name: seed, type: Integer, description: 'Random seed,', default: '0', optional: true}
+- {name: batch_size, type: Integer, description: Batch size, default: '16', optional: true}
+outputs:
+- {name: trained_model_path, type: TensorflowSavedModel, description: 'Output path
+    for the saved model,'}
+implementation:
+  container:
+    image: us-docker.pkg.dev/vertex-ai/ready-to-go-image-classification/image-components:v0.1
+    # command is a list of strings (command-line arguments).
+    # The YAML language has two syntaxes for lists and you can use either of them.
+    # Here we use the "flow syntax" - comma-separated strings inside square brackets.
+    command: [
+      python3,
+      # Path of the program inside the container
+      /pipelines/component/src/training_component.py,
+      --preprocessed-training-data-path,
+      {inputPath: preprocessed_training_data_path},
+      --preprocessed-validation-data-path,
+      {inputPath: preprocessed_validation_data_path},
+      --model-path,
+      {inputPath: model_path},
+      --trained-model-path,
+      {outputPath: trained_model_path},
+      --optimizer-name,
+      {inputValue: optimizer_name},
+      --loss-function-name,
+      {inputValue: loss_function_name},
+      --number-of-epochs,
+      {inputValue: number_of_epochs},
+      --seed,
+      {inputValue: seed},
+      --batch-size,
+      {inputValue: batch_size},
+      --metric-names,
+      {inputValue: metric_names},
+      --optimizer-parameters,
+      {inputValue: optimizer_parameters},
+      --loss-function-parameters,
+      {inputValue: loss_function_parameters},
+    ]

community-content/pipeline_components/image_ml_model_training/transcode_tfrecord_image_dataset_from_csv/component.yaml

@@ -0,0 +1,37 @@
+name: Transcode imagedataset tfrecord from csv
+description: |
+  Transcodes CSV Data into TFRecord file of TFExamples.
+  Args:
+    csv_image_data_path (str):
+        Path to the CSV image data. Data must include 'image_filepath' (Path to image file) and
+        'image_label' (output for a prediction) fields.
+    class_names (Sequence[str]):
+        Sequence of strings of categories for classification corresponding to input data.
+    tfrecord_image_data_path (str):
+        Output path for the TFRecord image data. Data will be formatted as 'label' (encoded image
+        label), and 'image_raw' (the binary string of the image data).
+inputs:
+- {name: csv_image_data_path, type: ImageDatasetCSV, description: Input path for the
+    CSV image data}
+- {name: class_names, type: 'typing.List[str]', description: List of class names corresponding
+    to the input image data}
+outputs:
+- {name: tfrecord_image_data_path, type: ImageDatasetTFRecord, description: Output
+    path for the TFRecord image data}
+implementation:
+  container:
+    image: us-docker.pkg.dev/vertex-ai/ready-to-go-image-classification/image-components:v0.1
+    # command is a list of strings (command-line arguments).
+    # The YAML language has two syntaxes for lists and you can use either of them.
+    # Here we use the "flow syntax" - comma-separated strings inside square brackets.
+    command: [
+      python3,
+      # Path of the program inside the container
+      /pipelines/component/src/transcoding_csv_component.py,
+      --csv-image-data-path,
+      {inputPath: csv_image_data_path},
+      --tfrecord-image-data-path,
+      {outputPath: tfrecord_image_data_path},
+      --class-names,
+      {inputValue: class_names},
+    ]

community-content/pipeline_components/image_ml_model_training/transcode_tfrecord_image_dataset_from_jsonl/component.yaml

@@ -0,0 +1,39 @@
+name: Transcode imagedataset tfrecord from jsonlines
+description: |
+  Transcodes JSONL Data into TFRecord file of TFExamples.
+  Args:
+    jsonl_image_data_path (str):
+        Input path for the JSONL image data
+        Path to the JSONL image data. Each line corresponds to a JSON input describing an image.
+        Schema follows AutoML image classification JSONL format
+        https://cloud.google.com/vertex-ai/docs/image-data/classification/prepare-data#json-lines.
+    class_names (Sequence[str]):
+        Sequence of strings of categories for classification corresponding to input data.
+    tfrecord_image_data_path (str):
+        Output path for the TFRecord image data. Data will be formatted as 'label' (encoded image
+        label), and 'image_raw' (the binary string of the image data).
+inputs:
+- {name: jsonl_image_data_path, type: ImageDatasetJsonLines, description: Input path
+    for the JSONL image data}
+- {name: class_names, type: 'typing.List[str]', description: List of class names corresponding
+    to the input image data}
+outputs:
+- {name: tfrecord_image_data_path, type: ImageDatasetTFRecord, description: Output
+    path for the TFRecord image data}
+implementation:
+  container:
+    image: us-docker.pkg.dev/vertex-ai/ready-to-go-image-classification/image-components:v0.1
+    # command is a list of strings (command-line arguments).
+    # The YAML language has two syntaxes for lists and you can use either of them.
+    # Here we use the "flow syntax" - comma-separated strings inside square brackets.
+    command: [
+      python3,
+      # Path of the program inside the container
+      /pipelines/component/src/transcoding_jsonl_component.py,
+      --jsonl-image-data-path,
+      {inputPath: jsonl_image_data_path},
+      --tfrecord-image-data-path,
+      {outputPath: tfrecord_image_data_path},
+      --class-names,
+      {inputValue: class_names},
+    ]

notebooks/community/sdk/sdk_pytorch_torchrun_custom_container_training_imagenet.ipynb

@@ -567,7 +567,7 @@
       "outputs": [],
       "source": [
         "%%writefile trainer/Dockerfile\n",
-        "FROM gcr.io/deeplearning-platform-release/pytorch-gpu.1-12\n",
+        "FROM gcr.io/deeplearning-platform-release/pytorch-gpu.1-13:m102\n",
         "\n",
         "RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - && \\\n",
         "    # Install reduction server plugin on GPU containers. google-fast-socket is\n",
@@ -588,16 +588,31 @@
         "RUN apt-get update -y && \\\n",
         "    apt-get install -y curl gnupg telnet nano net-tools iputils-ping\n",
         "\n",
+        "# Set ETCD version\n",
+        "ARG ETCD_VER=v2.3.0\n",
+        "# Choose either URL\n",
+        "ARG GOOGLE_URL=https://storage.googleapis.com/etcd\n",
+        "ARG GITHUB_URL=https://github.com/etcd-io/etcd/releases/download\n",
+        "# Set ETCD URL to download from\n",
+        "ARG DOWNLOAD_URL=$GOOGLE_URL\n",
         "\n",
+        "# Install ETCD\n",
+        "RUN mkdir -p /tmp/etcd-download-test && \\\n",
+        "    curl -L ${DOWNLOAD_URL}/${ETCD_VER}/etcd-${ETCD_VER}-linux-amd64.tar.gz -o /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz && \\\n",
+        "    tar xzvf /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz -C /tmp/etcd-download-test --strip-components=1 && \\\n",
+        "    rm -f /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz\n",
+        "\n",
+        "# Copy training application code\n",
         "COPY . /trainer\n",
         "\n",
         "WORKDIR /trainer\n",
         "\n",
+        "# Install dependencies\n",
         "RUN pip install -r requirements.txt\n",
         "\n",
         "RUN chmod 777 main.sh\n",
         "\n",
-        "# download data to the container\n",
+        "# Download data to the container\n",
         "RUN wget -q -P /trainer/data https://image-net.org/data/tiny-imagenet-200.zip\n",
         "RUN unzip -q /trainer/data/tiny-imagenet-200.zip\n",
         "RUN rm /trainer/data/tiny-imagenet-200.zip\n",
@@ -614,8 +629,8 @@
       "outputs": [],
       "source": [
         "%%writefile trainer/requirements.txt\n",
-        "torch==1.12.0\n",
-        "torchvision==0.13.0\n",
+        "torch==1.13.0\n",
+        "torchvision==0.14.0\n",
         "tensorboard==2.5.0\n",
         "protobuf==3.20.*\n",
         "python-etcd\n",
@@ -675,30 +690,17 @@
         "setup_etcd() {\n",
         "    HOST_IP=$1\n",
         "    # Start a local instane of ETCD v2 \n",
-        "    ETCD_VER=v2.3.0 #v3.5.6\n",
         "    export ETCD_ENABLE_V2=true\n",
         "    export ETCDCTL_API=2\n",
         "\n",
-        "    # choose either URL\n",
-        "    GOOGLE_URL=https://storage.googleapis.com/etcd\n",
-        "    GITHUB_URL=https://github.com/etcd-io/etcd/releases/download\n",
-        "    DOWNLOAD_URL=${GOOGLE_URL}\n",
-        "\n",
-        "    rm -f /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz\n",
-        "    rm -rf /tmp/etcd-download-test && mkdir -p /tmp/etcd-download-test\n",
-        "\n",
-        "    curl -L ${DOWNLOAD_URL}/${ETCD_VER}/etcd-${ETCD_VER}-linux-amd64.tar.gz -o /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz\n",
-        "    tar xzvf /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz -C /tmp/etcd-download-test --strip-components=1\n",
-        "    rm -f /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz\n",
-        "\n",
         "    /tmp/etcd-download-test/etcd --name s1 --data-dir /tmp/etcd-download-test/s1  \\\n",
         "    --listen-client-urls http://0.0.0.0:2379 --advertise-client-urls http://$HOST_IP:2379 \\\n",
         "    --listen-peer-urls http://0.0.0.0:2380 --initial-advertise-peer-urls http://$HOST_IP:2380 \\\n",
         "    --initial-cluster s1=http://$HOST_IP:2380 --initial-cluster-token tkn \\\n",
         "    --initial-cluster-state new &> /tmp/etcd-download-test/node.log &\n",
         "\n",
-        "    sudo /tmp/etcd-download-test/etcd --version\n",
-        "    sudo /tmp/etcd-download-test/etcdctl --version\n",
+        "    /tmp/etcd-download-test/etcd --version\n",
+        "    /tmp/etcd-download-test/etcdctl --version\n",
         "}\n",
         "\n",
         "\n",

notebooks/notebook_template_review.py

@@ -61,6 +61,8 @@ parser.add_argument('--uses', dest='uses', action='store_true',
                     default=False, help='Output uses (resources)')
 parser.add_argument('--steps', dest='steps', action='store_true', 
                     default=False, help='Ouput steps')
+parser.add_argument('--linkback', dest='linkback', action='store_true', 
+                    default=False, help='Ouput linkback')
 parser.add_argument('--web', dest='web', action='store_true', 
                     default=False, help='Output format in HTML')
 parser.add_argument('--repo', dest='repo', action='store_true', 
@@ -109,13 +111,14 @@ class ErrorCode(Enum):
     #   Costs cell required
     #     Check for required Vertex and optional BQ and Dataflow
     ERROR_OVERVIEW_NOTFOUND = 10,
-    ERROR_OBJECTIVE_NOTFOUND = 11,
-    ERROR_OBJECTIVE_MISSING_DESC = 12,
-    ERROR_OBJECTIVE_MISSING_USES = 13,
-    ERROR_OBJECTIVE_MISSING_STEPS = 14,
-    ERROR_DATASET_NOTFOUND = 15,
-    ERROR_COSTS_NOTFOUND = 16,
-    ERROR_COSTS_MISSING = 17,
+    ERROR_LINKBACK_NOTFOUND = 11,
+    ERROR_OBJECTIVE_NOTFOUND = 12,
+    ERROR_OBJECTIVE_MISSING_DESC = 13,
+    ERROR_OBJECTIVE_MISSING_USES = 14,
+    ERROR_OBJECTIVE_MISSING_STEPS = 15,
+    ERROR_DATASET_NOTFOUND = 16,
+    ERROR_COSTS_NOTFOUND = 17,
+    ERROR_COSTS_MISSING = 18,
 
     # Installation cell
     #   Installation cell required
@@ -126,34 +129,34 @@ class ErrorCode(Enum):
     #   option {USER_FLAG} required
     #   installation code cell not match template
     #   all packages must be installed as a single pip3
-    ERROR_INSTALLATION_NOTFOUND = 18,
-    ERROR_INSTALLATION_HEADING = 19,
-    ERROR_INSTALLATION_CODE_NOTFOUND = 20,
-    ERROR_INSTALLATION_PIP3 = 21,
-    ERROR_INSTALLATION_QUIET = 22,
-    ERROR_INSTALLATION_USER_FLAG = 23,
-    ERROR_INSTALLATION_CODE_TEMPLATE = 24,
-    ERROR_INSTALLATION_SINGLE_PIP3 = 25,
+    ERROR_INSTALLATION_NOTFOUND = 19,
+    ERROR_INSTALLATION_HEADING = 20,
+    ERROR_INSTALLATION_CODE_NOTFOUND = 21,
+    ERROR_INSTALLATION_PIP3 = 22,
+    ERROR_INSTALLATION_QUIET = 23,
+    ERROR_INSTALLATION_USER_FLAG = 24,
+    ERROR_INSTALLATION_CODE_TEMPLATE = 25,
+    ERROR_INSTALLATION_SINGLE_PIP3 = 26,
 
     # Restart kernel cell
     #    Restart code cell required
     #    Restart code cell not found
-    ERROR_RESTART_NOTFOUND = 23,
-    ERROR_RESTART_CODE_NOTFOUND = 24,
+    ERROR_RESTART_NOTFOUND = 27,
+    ERROR_RESTART_CODE_NOTFOUND = 28,
 
     # Before you begin cell
     #    Before you begin cell required
     #    Before you begin cell incomplete
-    ERROR_BEFOREBEGIN_NOTFOUND = 25,
-    ERROR_BEFOREBEGIN_INCOMPLETE = 26,
+    ERROR_BEFOREBEGIN_NOTFOUND = 29,
+    ERROR_BEFOREBEGIN_INCOMPLETE = 30,
 
     # Set Project ID
     #    Set project ID cell required
     #    Set project ID code cell not found
     #    Set project ID not match template
-    ERROR_PROJECTID_NOTFOUND = 27,
-    ERROR_PROJECTID_CODE_NOTFOUND = 28,
-    ERROR_PROJECTID_TEMPLATE = 29,
+    ERROR_PROJECTID_NOTFOUND = 31,
+    ERROR_PROJECTID_CODE_NOTFOUND = 32,
+    ERROR_PROJECTID_TEMPLATE = 33,
 
     # Technical Writer Rules
     ERROR_TWRULE_TODO = 51,
@@ -182,7 +185,21 @@ def parse_dir(directory: str) -> int:
     """
     exit_code = 0
     
+    sorted_entries = []
     entries = os.scandir(directory)
+    for entry in entries:
+
+        inserted = False
+        for ix in range(len(sorted_entries)):
+            if entry.name < sorted_entries[ix].name:
+                sorted_entries.insert(ix, entry)
+                inserted = True
+                break
+        
+        if not inserted:
+            sorted_entries.append(entry)
+    
+    entries = sorted_entries
     for entry in entries:
         if entry.is_dir():
             if entry.name[0] == '.':
@@ -191,13 +208,65 @@ def parse_dir(directory: str) -> int:
                 continue
             exit_code += parse_dir(entry.path)
         elif entry.name.endswith('.ipynb'):
-            exit_code += parse_notebook(entry.path, tag=directory.split('/')[-1], linkback=None, rules=rules)
+            tag = directory.split('/')[-1]
+            if tag == 'automl':
+                tag = 'AutoML'
+            elif tag == 'bigquery_ml':
+                tag = 'BigQuery ML'
+            elif tag == 'custom':
+                tag = 'Vertex AI Training'
+            elif tag == 'experiments':
+                tag = 'Vertex AI Experiments'
+            elif tag == 'explainable_ai':
+                tag = 'Vertex Explainable AI'
+            elif tag == 'feature_store':
+                tag = 'Vertex AI Feature Store'
+            elif tag == 'matching_engine':
+                tag = 'Vertex AI Matching Engine'
+            elif tag == 'migration':
+                tag = 'CAIP to Vertex AI migration'
+            elif tag == 'ml_metadata':
+                tag = 'Vertex ML Metadata'
+            elif tag == 'model_evaluation':
+                tag = 'Vertex AI Model Evaluation'
+            elif tag == 'model_monitoring':
+                tag = 'Vertex AI Model Monitoring'
+            elif tag == 'model_registry':
+                tag = 'Vertex AI Model Registry'
+            elif tag == 'pipelines':
+                tag = 'Vertex AI Pipelines'
+            elif tag == 'prediction':
+                tag = 'Vertex AI Prediction'
+            elif tag == 'pytorch':
+                tag = 'Vertex AI Training'
+            elif tag == 'reduction_server':
+                tag = 'Vertex AI Reduction Server'
+            elif tag == 'sdk':
+                tag = 'Vertex AI SDK'
+            elif tag == 'structured_data':
+                tag = 'AutoML / BQML'
+            elif tag == 'tabnet':
+                tag = 'Vertex AI TabNet'
+            elif tag == 'tabular_workflows':
+                tag = 'AutoML Tabular Workflows'
+            elif tag == 'tensorboard':
+                tag = 'Vertex AI TensorBoard'
+            elif tag == 'training':
+                tag = 'Vertex AI Training'
+            elif tag == 'vizier':
+                tag = 'Vertex AI Vizier'
+                
+            # special case
+            if 'workbench' in directory:
+                tag = 'Vertex AI Workbench'
+                
+            exit_code += parse_notebook(entry.path, tags=[tag], linkback=None, rules=rules)
             
     return exit_code
 
 
 def parse_notebook(path: str,
-                   tag: str,
+                   tags: List,
                    linkback: str,
                    rules: List) -> int:
     """
@@ -205,8 +274,9 @@ def parse_notebook(path: str,
         and notebook authoring requirements.
         
             path: The path to the notebook.
-            tag: The associated tag
+            tags: The associated tags
             linkback: A link back to the web docs
+            rules: The cell rules to apply
             
         Returns the number of errors
     """
@@ -218,9 +288,20 @@ def parse_notebook(path: str,
     
     # Automatic Index Generation
     if objective.desc != '':
+        if overview.linkbacks:
+            linkbacks = overview.linkbacks
+        else:
+            if linkback:
+                linkbacks = [linkback]
+            else:
+                linkbacks = []
+
+        if overview.tags:
+            tags = overview.tags
+                
         add_index(path, 
-                  tag, 
-                  linkback,
+                  tags, 
+                  linkbacks,
                   title.title, 
                   objective.desc, 
                   objective.uses, 
@@ -512,9 +593,23 @@ class OverviewRule(NotebookRule):
         """
         Parse the overview cell
         """
+        self.linkbacks = []
+        self.tags = []
+        
         cell = notebook.get()
         if not cell['source'][0].startswith("## Overview"):
             return notebook.report_error(ErrorCode.ERROR_OVERVIEW_NOTFOUND, "Overview section not found")
+        
+        last_line = cell['source'][-1]
+        if last_line.startswith('Learn more about ['):
+            for more in last_line.split('[')[1:]:
+                tag = more.split(']')[0]
+                linkback = more.split('(')[1].split(')')[0]
+                self.tags.append(tag)
+                self.linkbacks.append(linkback)
+        else:
+            return notebook.report_error(ErrorCode.ERROR_LINKBACK_NOTFOUND, "Linkback missing in overview section")
+                
         return True
 
 
@@ -542,6 +637,10 @@ class ObjectiveRule(NotebookRule):
         in_steps = False
     
         for line in cell['source'][1:]:
+            # TOC anchor
+            if line.startswith('<a name='):
+                continue
+                
             if line.startswith('This tutorial uses'):
                 in_desc = False
                 in_steps = False
@@ -578,10 +677,14 @@ class ObjectiveRule(NotebookRule):
                         # check for italic font setting
                         if ch == '*' and sline[1] != ' ':
                             in_steps = False
+                        # special case
+                        elif sline.startswith('* Prediction Service'):
+                            in_steps = False
                         else:
                             self.steps += line
                     elif ch == '#':
                         in_steps = False
+
             
         if self.desc == '':
             ret = notebook.report_error(ErrorCode.ERROR_OBJECTIVE_MISSING_DESC, "Objective section missing desc")
@@ -607,7 +710,7 @@ class ObjectiveRule(NotebookRule):
             ret = notebook.report_error(ErrorCode.ERROR_OBJECTIVE_MISSING_STEPS, "Objective section missing steps list")
             
         notebook.costs = self.costs
-        ret = True
+        return ret
 
 
 class RecommendationsRule(NotebookRule):
@@ -972,8 +1075,8 @@ class TextTWRule(TextRule):
 
 
 def add_index(path: str, 
-              tag: str, 
-              linkback: str,
+              tags: List, 
+              linkbacks: List,
               title : str, 
               desc: str, 
               uses: str, 
@@ -986,15 +1089,15 @@ def add_index(path: str,
     Add a discoverability index for this notebook
     
         path: The path to the notebook
-        tag: The tag (if any) for the notebook
+        tags: The tags (if any) for the notebook
         title: The H1 title for the notebook
-        desc:
-        uses:
-        steps:
-        git_link:
-        colab_link:
-        workbench_link:
-        linkback:
+        desc: The notebook description
+        uses: The resources/services used by the notebook
+        steps: The steps specified by the notebook
+        git_link: The link to the notebook in the git repo
+        colab_link: Link to launch notebook in Colab
+        workbench_link: Link to launch notebook in Workbench
+        linkbacks: The linkbacks per tag
     """
     global last_tag
     
@@ -1004,43 +1107,53 @@ def add_index(path: str,
     title = title.split(':')[-1].strip()
     title = title[0].upper() + title[1:]
     if args.web:
-        title = title.replace('`', '')
+        title = replace_cl(title.replace('`', ''))
         
         print('    <tr>')
         print('        <td>')
-        tags = tag.split(',')
         for tag in tags:
+            tag = replace_cl(tag)
             print(f'            {tag.strip()}<br/>\n')
         print('        </td>')
         print('        <td>')
-        print(f'            {title}<br/>\n')
+        print(f'            <b>{title}</b><br/>\n')
         if args.desc:
-            desc = desc.replace('`', '')
+            desc = replace_cl(desc.replace('`', ''))
+            print('<br/>')
             print(f'            {desc}<br/>\n')
-        if linkback:
-            text = ''
-            for tag in tags:
-                text += tag.strip() + ' '
-                
-            print(f'            Learn more about <a src="https://cloud.google.com/{linkback}">{text}</a><br/>\n')
+            
+        if args.steps:
+            steps = replace_cl(steps.replace('\n', '<br/>').replace('-', '&nbsp;&nbsp;-').replace('**', '').replace('*', '&nbsp;&nbsp;-').replace('`', ''))
+            print('<br/>' + steps +  '<br/>')
+            
+        if args.linkback and linkbacks:
+            num = len(tags)
+            for _ in range(num):
+                if linkbacks[_].startswith("vertex-ai"):
+                    print(f'<br/>            Learn more about <a href="https://cloud.google.com/{linkbacks[_]}" target="_blank">{replace_cl(tags[_])}</a>.\n')
+                else:
+                    print(f'<br/>            Learn more about <a href="{linkbacks[_]}" target="_blank">{replace_cl(tags[_])}</a>.\n')
+                    
         print('        </td>')
         print('        <td>')
         if colab_link:
-            print(f'            <a src="{colab_link}">Colab</a><br/>\n')
+            print(f'            <a href="{colab_link}" target="_blank">Colab</a><br/>\n')
         if git_link:
-            print(f'            <a src="{git_link}">GitHub</a><br/>\n')
+            print(f'            <a href="{git_link}" target="_blank">GitHub</a><br/>\n')
         if workbench_link:
-            print(f'            <a src="{workbench_link}">Vertex AI Workbench</a><br/>\n')
+            print(f'            <a href="{workbench_link}" target="_blank">Vertex AI Workbench</a><br/>\n')
         print('        </td>')
         print('    </tr>\n')
     elif args.repo:
-        tags = tag.split(',')
-        if tags != last_tag and tag != '':
-            last_tag = tags
-            flat_list = ''
-            for item in tags:
-                flat_list += item.replace("'", '') + ' '
-            print(f"\n### {flat_list}\n")
+        try:
+            if tags != last_tag and tag != '':
+                last_tag = tags
+                flat_list = ''
+                for item in tags:
+                    flat_list += item.replace("'", '') + ' '
+                print(f"\n### {flat_list}\n")
+        except:
+            pass
         print(f"\n[{title}]({git_link})\n")
     
         print("```")
@@ -1052,7 +1165,80 @@ def add_index(path: str,
 
         if args.steps:
             print(steps.rstrip() + '\n')
+            
         print("```\n")
+            
+        if args.linkback and linkbacks:
+            num = len(tags)
+            for _ in range(num):
+                if linkbacks[_].startswith("vertex-ai"):
+                    print(f'&nbsp;&nbsp;&nbsp;Learn more about [{tags[_]}]({linkbacks[_]}).\n')
+                else:
+                    print(f'&nbsp;&nbsp;&nbsp;Learn more about [{tags[_]}]({linkbacks[_]}).\n')
+
+def replace_cl(text : str ) -> str:
+    '''
+    Replace product names with CL substitution variables
+    '''
+    substitutions = {
+        #'AutoML Tabular Workflow': '{{automl_name}} Tabular Workflow',
+        #'AutoML Tables': '{{automl_tables_name}}',
+        #'AutoML Tabular': '{{automl_tables_name}}',
+        #'AutoML Vision': '{automl_vision_name}}',
+        #'AutoML Image': '{automl_vision_name}}',
+        'AutoML': '{{automl_name}}',
+        
+        'BigQuery ML': '{{bigqueryml_name}}',
+        'BQML': '{{bigqueryml_name}}',
+        'BigQuery': '{{bigquery_name}}',
+        'BQ': '{{bigquery_name}}',
+        
+        'Vertex Dataset': '{{vertex_ai_name}} Dataset',
+        'Vertex Model': '{{vertex_ai_name}} Model',
+        'Vertex Endpoint': '{{vertex_ai_name}} Endpoint',
+        'Vertex Model Registry': '{{vertex_model_registry_name}}',
+        'Vertex AI Model Registry': '{{vertex_model_registry_name}}',
+        'Vertex Training': '{{vertex_training_name}}',
+        'Vertex AI Training': '{{vertex_training_name}}',
+        'Vertex Prediction': '{{vertex_prediction_name}}',
+        'Vertex AI Prediction': '{{vertex_prediction_name}}',
+        'Vertex TensorBoard': '{{vertex_tensorboard_name}}',
+        'Vertex AI TensorBoard': '{{vertex_tensorboard_name}}',
+        'Vertex ML Metadata': '{{vertex_metadata_name}}',
+        'Vertex Pipelines': '{{vertex_pipelines_name}}',
+        'Vertex AI Pipelines': '{{vertex_pipelines_name}}',
+        'Vertex AI Data Labeling': '{{vertex_data_labeling_name}}',
+        'Vertex AI Experiments': '{{vertex_experiments_name}}',
+        'Vertex Experiments': '{{vertex_experiments_name}}',
+        'Vertex AI Matching Engine': '{vertex_matching_engine_name}}',
+        'Vertex Matching Engine': '{vertex_matching_engine_name}}',
+        'Vertex Model Monitoring': '{{vertex_model_monitoring_name}}',
+        'Vertex AI Model Monitoring': '{{vertex_model_monitoring_name}}',
+        'Vertex Feature Store': '{{vertex_featurestore_name}}',
+        'Vertex AI Feature Store': '{{vertex_featurestore_name}}',
+        'Vertex Vizier': '{{vertex_vizier_name}}',
+        'Vertex AI Vizier': '{{vertex_vizier_name}}',
+        'Vertex Explainable AI': '{{vertex_xai_name}}',
+        'NAS': '{{vertex_nas_name}',
+        'Vertex AI Neural Architectural Search': '{{vertex_nas_name}}',
+        'Vertex Workbench': '{{vertex_workbench_name}}',
+        'Vertex AI Workbench': '{{vertex_workbench_name}}',
+        'Vertex AI Edge Manager': '{{vertex_edge_manager_name}}',
+        'Vertex SDK': '{{vertex_sdk_name}}',
+        'Vertex AI SDK': '{{vertex_sdk_name}}',
+        'Vertex AI': '{{vertex_ai_name}}',
+        
+        'Cloud Storage': '{{storage_name}}',
+        'TensorFlow Enterprise': '{{tf4gcp_name}}',
+        'TensorFlow': '{{tensorflow_name}}',
+    }
+    
+    for key, value in substitutions.items():
+        if key in text:
+            text = text.replace(key, value)
+            
+    return text
+
 
 
 # Instantiate the rules
@@ -1084,21 +1270,27 @@ rules = [ copyright, notices, title, links, testenv, table, overview, objective,
 ]
 
 if args.web:
+    print('<style>')
+    print('table, th, td {')
+    print('  border: 1px solid black;')
+    print('  padding-left:10px')
+    print('}')
+    print('</style>')
     print('<table>')
-    print('    <th>Vertex AI Feature</th>')
+    print('    <th width="180px">Services</th>')
     print('    <th>Description</th>')
-    print('    <th>Open in</th>')
+    print('    <th width="80px">Open in</th>')
 
 if args.notebook_dir:
     if not os.path.isdir(args.notebook_dir):
-        print("Error: not a directory:", args.notebook_dir)
+        print(f"Error: not a directory: {args.notebook_dir}", file=sys.stderr)
         exit(1)
     exit_code = parse_dir(args.notebook_dir)
 elif args.notebook:
     if not os.path.isfile(args.notebook):
-        print("Error: not a notebook:", args.notebook)
+        print(f"Error: not a notebook: {args.notebook}", file=sys.stderr)
         exit(1)
-    exit_code = parse_notebook(args.notebook, tag='', linkback=None, rules=rules)
+    exit_code = parse_notebook(args.notebook, tags=[], linkback=None, rules=rules)
 elif args.notebook_file:
     if not os.path.isfile(args.notebook_file):
         print("Error: file does not exist", args.notebook_file)
@@ -1111,15 +1303,15 @@ elif args.notebook_file:
                 if heading:
                     heading = False
                 else:
-                    tag = row[0]
+                    tags = row[0].split(',')
                     notebook = row[1]
                     try:
                         linkback = row[2]
                     except:
                         linkback = None
-                    exit_code += parse_notebook(notebook, tag=tag, linkback=linkback, rules=rules)
+                    exit_code += parse_notebook(notebook, tags=tags, linkback=linkback, rules=rules)
 else:
-    print("Error: must specify a directory or notebook")
+    print("Error: must specify a directory or notebook", file=sys.stderr)
     exit(1)
 
 if args.web:

notebooks/official/automl/README.md

@@ -1,4 +1,5 @@
-[AutoML Tabular Training and Prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/automl-tabular-classification.ipynb)
+
+[AutoML Tabular training and prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/automl-tabular-classification.ipynb)
 
 ```
 Learn how to train and make predictions on an AutoML model based on a tabular dataset.
@@ -13,6 +14,8 @@ The steps performed include the following:
 
 ```
 
+   Learn more about [Tabular classification](https://cloud.google.com/vertex-ai/docs/tabular-data/classification-regression/overview).
+
 
 [Create, train, and deploy an AutoML text classification model](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/automl-text-classification.ipynb)
 
@@ -31,97 +34,7 @@ The steps performed include:
 
 ```
 
-
-[AutoML training video classification model for batch prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_video_classification_batch.ipynb)
-
-```
-Learn how to create an AutoML video classification model from a Python script, and then do a batch prediction using the Vertex AI SDK.
-
-The steps performed include:
-
-- Create a Vertex `Dataset` resource.
-- Train the model.
-- View the model evaluation.
-- Make a batch prediction.
-
-```
-
-
-[AutoML training text entity extraction model for online prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_text_entity_extraction_online.ipynb)
-
-```
-Learn how to create an AutoML text entity extraction model and deploy for online prediction from a Python script using the Vertex SDK.
-
-The steps performed include:
-
-- Create a Vertex `Dataset` resource.
-- Train the model.
-- View the model evaluation.
-- Deploy the `Model` resource to a serving `Endpoint` resource.
-- Make a prediction.
-- Undeploy the `Model`.
-
-```
-
-
-[AutoML tabular forecasting model for batch prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_tabular_forecasting_batch.ipynb)
-
-```
-Learn how to create an `AutoML` tabular forecasting model from a Python script, and then do a batch prediction using the Vertex AI SDK.
-
-The steps performed include:
-
-- Create a `Vertex AI Dataset` resource.
-- Train an `AutoML` tabular forecasting `Model` resource.
-- Obtain the evaluation metrics for the `Model` resource.
-- Make a batch prediction.
-
-```
-
-
-[AutoML training video action recognition model for batch prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_video_action_recognition_batch.ipynb)
-
-```
-Learn how to create an AutoML video action recognition model from a Python script, and then do a batch prediction using the Vertex AI SDK.
-
-The steps performed include:
-
-- Create a `Vertex AI Dataset` resource.
-- Train the model.
-- View the model evaluation.
-- Make a batch prediction.
-```
-
-
-[AutoML Tabular Pipeline](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/automl_tabular_on_vertex_pipelines.ipynb)
-
-```
-Learn how to create two regression models using [Vertex Pipelines](https://cloud.
-
-The steps performed are:
-
-- Create a training pipeline that reduces the search space from the default to save time.
-- Create a training pipeline that reuses the architecture search results from the previous pipeline to save time.
-
-```
-
-
-[Training an AutoML text sentiment analysis model for online predictions](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_text_sentiment_analysis_online.ipynb)
-
-```
-Learn how to create an AutoML text sentiment analysis model and deploy it for online predictions from a Python script using the Vertex AI SDK.
-
-The steps performed include:
-
-- Create a `Vertex AI Dataset` resource.
-- Create a training job for the AutoML model on the dataset.
-- View the model evaluation metrics.
-- Deploy the `Vertex AI Model` resource to a serving `Vertex AI Endpoint`.
-- Make a prediction request to the deployed model.
-- Undeploy the model from endpoint.
-- Perform clean up process.
-
-```
+   Learn more about [AutoML Text classification](https://cloud.google.com/vertex-ai/docs/text-data/classification/train-model).
 
 
 [Compare Vertex AI Forecasting and BigQuery ML ARIMA_PLUS](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/automl_forecasting_bqml_arima_plus_comparison.ipynb)
@@ -141,6 +54,97 @@ The steps performed are:
 
 ```
 
+   Learn more about [AutoML Forecasting](https://cloud.google.com/vertex-ai/docs/tabular-data/forecasting/overview).
+
+   Learn more about [BQML Forecasting](https://cloud.google.com/vertex-ai/docs/tabular-data/forecasting-arima/overview).
+
+
+[AutoML Tabular Workflow pipelines](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/automl_tabular_on_vertex_pipelines.ipynb)
+
+```
+Learn how to create two regression models using [Vertex AI Pipelines](https://cloud.
+
+The steps performed are:
+
+- Create a training pipeline that reduces the search space from the default to save time.
+- Create a training pipeline that reuses the architecture search results from the previous pipeline to save time.
+
+```
+
+   Learn more about [AutoML Tabular Workflows](https://cloud.google.com/vertex-ai/docs/tabular-data/tabular-workflows/e2e-automl).
+
+
+[AutoML training hierarchical forecasting for batch prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_forecasting_hierarchical_batch.ipynb)
+
+```
+In this tutorial, you create an AutoML hierarchical forecasting model and deploy it for batch prediction using the Vertex AI SDK for Python.
+
+The steps performed include:
+
+- Create a Vertex AI `TimeSeriesDataset` resource.
+- Train the model.
+- View the model evaluation.
+- Deploy the `Model` resource to a serving `Endpoint` resource.
+- Make a prediction.
+- Undeploy the `Model`.
+
+```
+
+   Learn more about [AutoML Forecasting](https://cloud.google.com/vertex-ai/docs/tabular-data/forecasting/overview).
+
+
+[AutoML training image object detection model for batch prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_image_object_detection_batch.ipynb)
+
+```
+In this tutorial, you create an AutoML image object detection model from a Python script, and then do a batch prediction using the Vertex AI SDK.
+
+The steps performed include:
+
+- Create a Vertex `Dataset` resource.
+- Train the model.
+- View the model evaluation.
+- Make a batch prediction.
+
+```
+
+   Learn more about [AutoML Image](https://cloud.google.com/vertex-ai/docs/image-data/object-detection/train-model).
+
+
+[AutoML tabular forecasting model for batch prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_tabular_forecasting_batch.ipynb)
+
+```
+Learn how to create an `AutoML` tabular forecasting model from a Python script, and then do a batch prediction using the Vertex AI SDK.
+
+The steps performed include:
+
+- Create a `Vertex AI Dataset` resource.
+- Train an `AutoML` tabular forecasting `Model` resource.
+- Obtain the evaluation metrics for the `Model` resource.
+- Make a batch prediction.
+
+```
+
+   Learn more about [AutoML Forecasting](https://cloud.google.com/vertex-ai/docs/tabular-data/forecasting/tutorials-samples).
+
+
+[AutoML training tabular regression model for batch prediction using BigQuery](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_tabular_regression_batch_bq.ipynb)
+
+```
+Learn how to create an AutoML tabular regression model and deploy it for batch prediction using the Vertex AI SDK for Python.
+
+The steps performed include:
+
+- Create a Vertex AI `Dataset` resource.
+- Train the model.
+- View the model evaluation.
+- Deploy the `Model` resource to a serving `Endpoint` resource.
+- Make a prediction.
+- Undeploy the `Model`.
+
+```
+
+   Learn more about [AutoML Tabular](https://cloud.google.com/vertex-ai/docs/training-overview#tabular_data).
+
 
 [AutoML training tabular regression model for online prediction using BigQuery](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_tabular_regression_online_bq.ipynb)
 
@@ -158,6 +162,81 @@ The steps performed include:
 
 ```
 
+   Learn more about [AutoML Tabular](https://cloud.google.com/vertex-ai/docs/training-overview#tabular_data).
+
+
+[AutoML training text entity extraction model for online prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_text_entity_extraction_online.ipynb)
+
+```
+Learn how to create an AutoML text entity extraction model and deploy for online prediction from a Python script using the Vertex AI SDK.
+
+The steps performed include:
+
+- Create a Vertex `Dataset` resource.
+- Train the model.
+- View the model evaluation.
+- Deploy the `Model` resource to a serving `Endpoint` resource.
+- Make a prediction.
+- Undeploy the `Model`.
+
+```
+
+   Learn more about [AutoML Text](https://cloud.google.com/vertex-ai/docs/text-data/entity-extraction/train-model).
+
+
+[Training an AutoML text sentiment analysis model for online predictions](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_text_sentiment_analysis_online.ipynb)
+
+```
+Learn how to create an AutoML text sentiment analysis model and deploy it for online predictions from a Python script using the Vertex AI SDK.
+
+The steps performed include:
+
+- Create a `Vertex AI Dataset` resource.
+- Create a training job for the AutoML model on the dataset.
+- View the model evaluation metrics.
+- Deploy the `Vertex AI Model` resource to a serving `Vertex AI Endpoint`.
+- Make a prediction request to the deployed model.
+- Undeploy the model from endpoint.
+- Perform clean up process.
+
+```
+
+   Learn more about [AutoML Text](https://cloud.google.com/vertex-ai/docs/text-data/sentiment-analysis/train-model).
+
+
+[AutoML training video action recognition model for batch prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_video_action_recognition_batch.ipynb)
+
+```
+Learn how to create an AutoML video action recognition model from a Python script, and then do a batch prediction using the Vertex AI SDK.
+
+The steps performed include:
+
+- Create a `Vertex AI Dataset` resource.
+- Train the model.
+- View the model evaluation.
+- Make a batch prediction.
+
+```
+
+   Learn more about [AutoML Video](https://cloud.google.com/vertex-ai/docs/video-data/action-recognition/train-model).
+
+
+[AutoML training video classification model for batch prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_video_classification_batch.ipynb)
+
+```
+Learn how to create an AutoML video classification model from a Python script, and then do a batch prediction using the Vertex AI SDK.
+
+The steps performed include:
+
+- Create a Vertex `Dataset` resource.
+- Train the model.
+- View the model evaluation.
+- Make a batch prediction.
+
+```
+
+   Learn more about [AutoML Video](https://cloud.google.com/vertex-ai/docs/video-data/classification/train-model).
+
 
 [AutoML training video object tracking model for batch prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_video_object_tracking_batch.ipynb)
 
@@ -170,21 +249,8 @@ The steps performed include:
 - Train the model.
 - View the model evaluation.
 - Make a batch prediction.
-```
-
-
-[AutoML training tabular regression model for batch prediction using BigQuery](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/automl/sdk_automl_tabular_regression_batch_bq.ipynb)
 
 ```
-Learn how to create an AutoML tabular regression model and deploy it for batch prediction using the Vertex AI SDK for Python.
 
-The steps performed include:
+   Learn more about [AutoML Video](https://cloud.google.com/vertex-ai/docs/video-data/object-tracking/train-model).
 
-- Create a Vertex AI `Dataset` resource.
-- Train the model.
-- View the model evaluation.
-- Deploy the `Model` resource to a serving `Endpoint` resource.
-- Make a prediction.
-- Undeploy the `Model`.
-
-```

notebooks/official/automl/automl-tabular-classification.ipynb

@@ -63,7 +63,9 @@
         "\n",
         "This tutorial demonstrates how to use the Vertex AI Python client library to train and deploy a tabular classification model for online prediction.\n",
         "\n",
-        "**Note**: you may incur charges for training, prediction, storage, or usage of other Google Cloud products in connection with testing this SDK."
+        "**Note**: you may incur charges for training, prediction, storage, or usage of other Google Cloud products in connection with testing this SDK.\n",
+        "\n",
+        "Learn more about [Classification for tabular data](https://cloud.google.com/vertex-ai/docs/tabular-data/classification-regression/overview)."
       ]
     },
     {

notebooks/official/automl/automl-text-classification.ipynb

@@ -68,7 +68,9 @@
       "source": [
         "## Overview\n",
         "\n",
-        "This notebook walks you through the major phases of building and using an AutoML text classification model on [Vertex AI](https://cloud.google.com/vertex-ai/docs/). \n"
+        "This notebook walks you through the major phases of building and using an AutoML text classification model on [Vertex AI](https://cloud.google.com/vertex-ai/docs/). \n",
+        "\n",
+        "Learn more about [AutoML Text classification](https://cloud.google.com/vertex-ai/docs/text-data/classification/train-model)."
       ]
     },
     {

notebooks/official/automl/automl_forecasting_bqml_arima_plus_comparison.ipynb

@@ -61,7 +61,9 @@
       "source": [
         "## Overview\n",
         "\n",
-        "In this tutorial, you take on the role of a store planner who must determine how much inventory they will need to order for each of their products and stores for November 2019. You accomplish this by training forecasting models using historical sales data. You start with a baseline model using BigQuery ML (BQML) [ARIMA_PLUS](https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-create-time-series) and then compare it against a [Vertex AI Forecasting](https://cloud.google.com/vertex-ai/docs/tabular-data/forecasting/overview) model."
+        "In this tutorial, you take on the role of a store planner who must determine how much inventory they will need to order for each of their products and stores for November 2019. You accomplish this by training forecasting models using historical sales data. You start with a baseline model using BigQuery ML (BQML) [ARIMA_PLUS](https://cloud.google.com/bigquery-ml/docs/reference/standard-sql/bigqueryml-syntax-create-time-series) and then compare it against a [Vertex AI Forecasting](https://cloud.google.com/vertex-ai/docs/tabular-data/forecasting/overview) model.\n",
+        "\n",
+        "Learn more about [AutoML Forecasting](https://cloud.google.com/vertex-ai/docs/tabular-data/forecasting/overview)  and [BQML Forecasting](https://cloud.google.com/vertex-ai/docs/tabular-data/forecasting-arima/overview)."
       ]
     },
     {

notebooks/official/automl/automl_tabular_on_vertex_pipelines.ipynb

@@ -61,7 +61,9 @@
       "source": [
         "## Overview\n",
         "\n",
-        "In this tutorial, you will use two Vertex AI Tabular Workflows pipelines to train AutoML models using different configurations. You will see how `get_automl_tabular_pipeline_and_parameters` gives you the ability to customize the default AutoML Tabular pipeline, and how `get_skip_architecture_search_pipeline_and_parameters` allows you to reduce the training time and cost for an AutoML model by using the tuning results from a previous pipeline run."
+        "In this tutorial, you will use two Vertex AI Tabular Workflows pipelines to train AutoML models using different configurations. You will see how `get_automl_tabular_pipeline_and_parameters` gives you the ability to customize the default AutoML Tabular pipeline, and how `get_skip_architecture_search_pipeline_and_parameters` allows you to reduce the training time and cost for an AutoML model by using the tuning results from a previous pipeline run.\n",
+        "\n",
+        "Learn more about [AutoML Tabular Workflows](https://cloud.google.com/vertex-ai/docs/tabular-data/tabular-workflows/e2e-automl)."
       ]
     },
     {

notebooks/official/automl/sdk_automl_image_object_detection_batch.ipynb

@@ -63,7 +63,9 @@
         "## Overview\n",
         "\n",
         "\n",
-        "This tutorial demonstrates how to use the Vertex AI SDK to create image object detection models and do batch prediction using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) model."
+        "This tutorial demonstrates how to use the Vertex AI SDK to create image object detection models and do batch prediction using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) model.\n",
+        "\n",
+        "Learn more about [AutoML Image](https://cloud.google.com/vertex-ai/docs/image-data/object-detection/train-model)."
       ]
     },
     {

notebooks/official/automl/sdk_automl_tabular_forecasting_batch.ipynb

@@ -62,7 +62,9 @@
         "## Overview\n",
         "\n",
         "\n",
-        "This tutorial demonstrates how to use the Vertex AI SDK to create tabular forecasting models and do batch prediction using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) model."
+        "This tutorial demonstrates how to use the Vertex AI SDK to create tabular forecasting models and do batch prediction using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) model.\n",
+        "\n",
+        "Learn more about [AutoML Forecasting](https://cloud.google.com/vertex-ai/docs/tabular-data/forecasting/tutorials-samples)."
       ]
     },
     {

notebooks/official/automl/sdk_automl_tabular_regression_batch_bq.ipynb

@@ -61,7 +61,9 @@
         "## Overview\n",
         "\n",
         "\n",
-        "This tutorial demonstrates how to use the Vertex AI SDK for Python to create tabular regression models and do batch prediction using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) model."
+        "This tutorial demonstrates how to use the Vertex AI SDK for Python to create tabular regression models and do batch prediction using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) model.\n",
+        "\n",
+        "Learn more about [AutoML Tabular](https://cloud.google.com/vertex-ai/docs/training-overview#tabular_data)."
       ]
     },
     {

notebooks/official/automl/sdk_automl_tabular_regression_online_bq.ipynb

@@ -62,7 +62,9 @@
         "## Overview\n",
         "\n",
         "\n",
-        "This tutorial demonstrates how to use the Vertex AI SDK for Python to create tabular regression models and do online prediction using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) model."
+        "This tutorial demonstrates how to use the Vertex AI SDK for Python to create tabular regression models and do online prediction using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) model.\n",
+        "\n",
+        "Learn more about [AutoML Tabular](https://cloud.google.com/vertex-ai/docs/training-overview#tabular_data)."
       ]
     },
     {

notebooks/official/automl/sdk_automl_text_entity_extraction_online.ipynb

@@ -62,7 +62,9 @@
         "## Overview\n",
         "\n",
         "\n",
-        "This tutorial demonstrates how to use the Vertex AI SDK for Python to create text entity extraction models and do online prediction using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) model."
+        "This tutorial demonstrates how to use the Vertex AI SDK for Python to create text entity extraction models and do online prediction using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) model.\n",
+        "\n",
+        "Learn more about [AutoML Text](https://cloud.google.com/vertex-ai/docs/text-data/entity-extraction/train-model)."
       ]
     },
     {

notebooks/official/automl/sdk_automl_text_sentiment_analysis_online.ipynb

@@ -61,7 +61,9 @@
       "source": [
         "## Overview\n",
         "\n",
-        "This tutorial demonstrates how to use the Vertex AI SDK to train and deploy an [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) text sentiment analysis model and get online predictions from it."
+        "This tutorial demonstrates how to use the Vertex AI SDK to train and deploy an [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) text sentiment analysis model and get online predictions from it.\n",
+        "\n",
+        "Learn more about [AutoML Text](https://cloud.google.com/vertex-ai/docs/text-data/sentiment-analysis/train-model)."
       ]
     },
     {

notebooks/official/automl/sdk_automl_video_action_recognition_batch.ipynb

@@ -63,7 +63,9 @@
         "## Overview\n",
         "\n",
         "\n",
-        "This tutorial demonstrates how to use the Vertex AI SDK to create video action recognition models and do batch prediction using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) model."
+        "This tutorial demonstrates how to use the Vertex AI SDK to create video action recognition models and do batch prediction using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) model.\n",
+        "\n",
+        "Learn more about [AutoML Video](https://cloud.google.com/vertex-ai/docs/video-data/action-recognition/train-model)."
       ]
     },
     {

notebooks/official/automl/sdk_automl_video_classification_batch.ipynb

@@ -62,7 +62,9 @@
         "## Overview\n",
         "\n",
         "\n",
-        "This tutorial demonstrates how to use the Vertex AI SDK to create video classification models and do batch prediction using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) model."
+        "This tutorial demonstrates how to use the Vertex AI SDK to create video classification models and do batch prediction using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) model.\n",
+        "\n",
+        "Learn more about [AutoML Video](https://cloud.google.com/vertex-ai/docs/video-data/classification/train-model)."
       ]
     },
     {

notebooks/official/automl/sdk_automl_video_object_tracking_batch.ipynb

@@ -62,7 +62,9 @@
         "## Overview\n",
         "\n",
         "\n",
-        "This tutorial demonstrates how to use the Vertex AI SDK for Python to create video object tracking models and do batch prediction using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) model."
+        "This tutorial demonstrates how to use the Vertex AI SDK for Python to create video object tracking models and do batch prediction using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) model.\n",
+        "\n",
+        "Learn more about [AutoML Video](https://cloud.google.com/vertex-ai/docs/video-data/object-tracking/train-model)."
       ]
     },
     {

notebooks/official/bigquery_ml/README.md

@@ -1,3 +1,4 @@
+
 [Online prediction with BigQuery ML](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/bigquery_ml/bqml-online-prediction.ipynb)
 
 ```
@@ -14,3 +15,5 @@ The steps performed include:
 
 ```
 
+   Learn more about [BigQuery ML](https://cloud.google.com/bigquery-ml/docs/introduction).
+

notebooks/official/bigquery_ml/bqml-online-prediction.ipynb

@@ -61,7 +61,9 @@
       "source": [
         "## Overview\n",
         "\n",
-        "This notebook is aimed at data analysts and data scientists who have data in BigQuery, want to train a model using BigQuery ML, register the model to Vertex AI Model Registry, and deploy it to an endpoint for real-time prediction. "
+        "This notebook is aimed at data analysts and data scientists who have data in BigQuery, want to train a model using BigQuery ML, register the model to Vertex AI Model Registry, and deploy it to an endpoint for real-time prediction. \n",
+        "\n",
+        "Learn more about [BigQuery ML](https://cloud.google.com/bigquery-ml/docs/introduction)."
       ]
     },
     {

notebooks/official/custom/README.md

@@ -1,3 +1,24 @@
+
+[Deploying Iris-detection model using FastAPI and Vertex AI custom container serving](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/custom/SDK_Custom_Container_Prediction.ipynb)
+
+```
+Learn how to create, deploy and serve a custom classification model on Vertex AI.
+
+The steps performed include:
+
+- Train a model that uses flower's measurements as input to predict the class of iris.
+- Save the model and its serialized pre-processor.
+- Build a FastAPI server to handle predictions and health checks.
+- Build a custom container with model artifacts.
+- Upload and deploy custom container to Vertex AI Endpoints.
+
+```
+
+   Learn more about [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training).
+
+   Learn more about [Vertex AI Prediction](https://cloud.google.com/vertex-ai/docs/predictions/get-predictions).
+
+
 [Training and deploying a sales forecasting model using FBProphet and Vertex AI](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/custom/SDK_FBProphet_Forecasting_Online.ipynb)
 
 ```
@@ -16,34 +37,9 @@ The steps performed include:
 
 ```
 
+   Learn more about [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training).
 
-[Custom training and batch prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/custom/sdk-custom-image-classification-batch.ipynb)
-
-```
-Learn to use `Vertex AI Training` to create a custom trained model and use `Vertex AI Batch Prediction` to do a batch prediction on the trained model.
-
-The steps performed include:
-
-- Create a `Vertex AI` custom job for training a TensorFlow model.
-- Upload the trained model artifacts as a `Model` resource.
-- Make a batch prediction.
-
-```
-
-
-[Profile model training performance using Profiler](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/custom/custom_training_tensorboard_profiler.ipynb)
-
-```
-Learn how to enable Vertex AI TensorBoard Profiler for custom training jobs.
-
-The steps performed include:
-
-- Setup a service account and a Cloud Storage bucket
-- Create a TensorBoard instance
-- Create and run a custom training job
-- View the TensorBoard Profiler dashboard
-
-```
+   Learn more about [Vertex AI Prediction](https://cloud.google.com/vertex-ai/docs/predictions/get-predictions).
 
 
 [Training a TensorFlow model on BigQuery data](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/custom/custom-tabular-bq-managed-dataset.ipynb)
@@ -61,6 +57,43 @@ The steps performed include:
 
 ```
 
+   Learn more about [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training).
+
+
+[Profile model training performance using Profiler](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/custom/custom_training_tensorboard_profiler.ipynb)
+
+```
+Learn how to enable Vertex AI TensorBoard Profiler for custom training jobs.
+
+The steps performed include:
+
+- Setup a service account and a Cloud Storage bucket
+- Create a TensorBoard instance
+- Create and run a custom training job
+- View the TensorBoard Profiler dashboard
+
+```
+
+   Learn more about [Vertex AI TensorBoard Profiler](https://cloud.google.com/vertex-ai/docs/experiments/tensorboard-profiler).
+
+
+[Custom training and batch prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/custom/sdk-custom-image-classification-batch.ipynb)
+
+```
+Learn to use `Vertex AI Training` to create a custom trained model and use `Vertex AI Batch Prediction` to do a batch prediction on the trained model.
+
+The steps performed include:
+
+- Create a `Vertex AI` custom job for training a TensorFlow model.
+- Upload the trained model artifacts as a `Model` resource.
+- Make a batch prediction.
+
+```
+
+   Learn more about [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training).
+
+   Learn more about [Vertex AI Batch Prediction](https://cloud.google.com/vertex-ai/docs/tabular-data/classification-regression/get-batch-predictions).
+
 
 [Custom training and online prediction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/custom/sdk-custom-image-classification-online.ipynb)
 
@@ -78,19 +111,7 @@ The steps performed include:
 
 ```
 
+   Learn more about [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training).
 
-[Deploying Iris-detection model using FastAPI and Vertex AI custom container serving](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/custom/SDK_Custom_Container_Prediction.ipynb)
-
-```
-Learn how to create, deploy and serve a custom classification model on Vertex AI.
-
-The steps performed include:
-
-- Train a model that uses flower's measurements as input to predict the class of iris.
-- Save the model and its serialized pre-processor.
-- Build a FastAPI server to handle predictions and health checks.
-- Build a custom container with model artifacts.
-- Upload and deploy custom container to Vertex AI Endpoints.
-
-```
+   Learn more about [Vertex AI Prediction](https://cloud.google.com/vertex-ai/docs/predictions/get-predictions).
 

notebooks/official/custom/SDK_Custom_Container_Prediction.ipynb

@@ -58,7 +58,9 @@
       "source": [
         "## Overview\n",
         "\n",
-        "This tutorial walks you through building a custom container to serve a scikit-learn model on Vertex AI. You use the FastAPI Python web server framework to create a prediction and health endpoint. You also incorporate a pre-processor from training pipeline into your online serving application."
+        "This tutorial walks you through building a custom container to serve a scikit-learn model on Vertex AI. You use the FastAPI Python web server framework to create a prediction and health endpoint. You also incorporate a pre-processor from training pipeline into your online serving application.\n",
+        "\n",
+        "Learn more about [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training)  and [Vertex AI Prediction](https://cloud.google.com/vertex-ai/docs/predictions/get-predictions)."
       ]
     },
     {

notebooks/official/custom/SDK_FBProphet_Forecasting_Online.ipynb

@@ -62,7 +62,9 @@
         "\n",
         "This tutorial walks you through building a custom container to serve a facebook prophet model on Vertex AI. You use the FastAPI Python web server framework to create a prediction endpoint. This notebook is a modified version of an example on [serving a scikit-learn model on Vertex AI](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/sdk/SDK_Custom_Container_Prediction.ipynb).\n",
         "\n",
-        "Learn more about serving an FBProphet model from this [article on testdriven.io: Deploying and Hosting a Machine Learning Model with FastAPI and Heroku](https://testdriven.io/blog/fastapi-machine-learning/).\n"
+        "Learn more about serving an FBProphet model from this [article on testdriven.io: Deploying and Hosting a Machine Learning Model with FastAPI and Heroku](https://testdriven.io/blog/fastapi-machine-learning/).\n",
+        "\n",
+        "Learn more about [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training) and [Vertex AI Prediction](https://cloud.google.com/vertex-ai/docs/predictions/get-predictions).\n"
       ]
     },
     {

notebooks/official/custom/custom-tabular-bq-managed-dataset.ipynb

@@ -61,7 +61,9 @@
         "## Overview\n",
         "\n",
         "\n",
-        "This tutorial demonstrates how to use the Vertex AI SDK for Python to train and deploy a custom tabular classification model for online prediction."
+        "This tutorial demonstrates how to use the Vertex AI SDK for Python to train and deploy a custom tabular classification model for online prediction.\n",
+        "\n",
+        "Learn more about [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training)."
       ]
     },
     {

notebooks/official/custom/custom_training_tensorboard_profiler.ipynb

@@ -61,7 +61,9 @@
       "source": [
         "## Overview\n",
         "\n",
-        "Vertex AI TensorBoard Profiler lets you monitor and optimize your model training performance by helping you understand the resource consumption of training operations. This tutorial demonstrates how to enable Vertex AI TensorBoard Profiler so you can debug model training performance for your custom training jobs.\n"
+        "Vertex AI TensorBoard Profiler lets you monitor and optimize your model training performance by helping you understand the resource consumption of training operations. This tutorial demonstrates how to enable Vertex AI TensorBoard Profiler so you can debug model training performance for your custom training jobs.\n",
+        "\n",
+        "Learn more about [Vertex AI TensorBoard Profiler](https://cloud.google.com/vertex-ai/docs/experiments/tensorboard-profiler)."
       ]
     },
     {

notebooks/official/custom/sdk-custom-image-classification-batch.ipynb

@@ -62,7 +62,9 @@
         "## Overview\n",
         "\n",
         "\n",
-        "This tutorial demonstrates how to use the Vertex AI SDK for Python to train and deploy a custom image classification model for batch prediction."
+        "This tutorial demonstrates how to use the Vertex AI SDK for Python to train and deploy a custom image classification model for batch prediction.\n",
+        "\n",
+        "Learn more about [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training)  and [Vertex AI Batch Prediction](https://cloud.google.com/vertex-ai/docs/tabular-data/classification-regression/get-batch-predictions)."
       ]
     },
     {

notebooks/official/custom/sdk-custom-image-classification-online.ipynb

@@ -62,7 +62,9 @@
         "## Overview\n",
         "\n",
         "\n",
-        "This tutorial demonstrates how to use the Vertex AI SDK for Python to train and deploy a custom image classification model for online prediction."
+        "This tutorial demonstrates how to use the Vertex AI SDK for Python to train and deploy a custom image classification model for online prediction.\n",
+        "\n",
+        "Learn more about [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training)  and [Vertex AI Prediction](https://cloud.google.com/vertex-ai/docs/predictions/get-predictions)."
       ]
     },
     {

notebooks/official/datasets/README.md

@@ -0,0 +1,38 @@
+
+[Get started with BigQuery datasets](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/datasets/get_started_bq_datasets.ipynb)
+
+```
+Learn how to use `BigQuery` as a dataset for training with `Vertex AI`.
+
+The steps performed include:
+
+- Create a Vertex AI `Dataset` resource from `BigQuery` table -- compatible for `AutoML` training.
+- Extract a copy of the dataset from `BigQuery` to a CSV file in Cloud Storage -- compatible for `AutoML` or custom training.
+- Select rows from a `BigQuery` dataset into a `pandas` dataframe -- compatible for custom training.
+- Select rows from a `BigQuery` dataset into a `tf.data.Dataset` -- compatible for custom training `TensorFlow` models.
+- Select rows from extracted CSV files into a `tf.data.Dataset` -- compatible for custom training `TensorFlow` models.
+- Create a `BigQuery` dataset from CSV files.
+- Extract data from `BigQuery` table into a `DMatrix` -- compatible for custom training `XGBoost` models.
+
+```
+
+   Learn more about [BigQuery Datasets](https://cloud.google.com/bigquery/docs/datasets-intro).
+
+
+[Get started with Vertex AI Data Labeling](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/datasets/get_started_with_data_labeling.ipynb)
+
+```
+Learn how to use the `Vertex AI Data Labeling` service.
+
+The steps performed include:
+
+- Create a Specialist Pool for data labelers.
+- Create a data labeling job.
+- Submit the data labeling job.
+- List data labeling jobs.
+- Cancel a data labeling job.
+
+```
+
+   Learn more about [Vertex AI Data Labeling](https://cloud.google.com/vertex-ai/docs/datasets/data-labeling-job).
+

notebooks/official/experiments/README.md

@@ -1,17 +1,25 @@
-[Compare pipeline runs with Vertex AI Experiments](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/experiments/comparing_pipeline_runs.ipynb)
+
+[Build Vertex AI Experiment lineage for custom training](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/experiments/build_model_experimentation_lineage_with_prebuild_code.ipynb)
 
 ```
-Learn how to use `Vertex AI Experiments` to log a pipeline job and compare different pipeline jobs.
+Learn how to integrate preprocessing code in a Vertex AI experiments.
 
 The steps performed include:
 
-* Formalize a training component
-* Build a training pipeline
-* Run several Pipeline jobs and log their results
-* Compare different Pipeline jobs
+- Execute module for preprocessing data
+  - Create a dataset artifact
+  - Log parameters
+-  Execute module for training the model
+  - Log parameters
+  - Create model artifact
+  - Assign tracking lineage to dataset, model and parameters
 
 ```
 
+   Learn more about [Vertex AI Experiments](https://cloud.google.com/vertex-ai/docs/experiments/intro-vertex-ai-experiments).
+
+   Learn more about [Vertex ML Metadata](https://cloud.google.com/vertex-ai/docs/ml-metadata).
+
 
 [Track parameters and metrics for locally trained models](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/experiments/comparing_local_trained_models.ipynb)
 
@@ -26,13 +34,24 @@ The steps performed include:
 
 ```
 
+   Learn more about [Vertex AI Experiments](https://cloud.google.com/vertex-ai/docs/experiments/intro-vertex-ai-experiments).
 
-[Build Vertex AI Experiment lineage for custom training](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/experiments/build_model_experimentation_lineage_with_prebuild_code.ipynb)
+
+[Compare pipeline runs with Vertex AI Experiments](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/experiments/comparing_pipeline_runs.ipynb)
 
 ```
-Learn how to integrate preprocessing code in a Vertex AI experiments.
+Learn how to use `Vertex AI Experiments` to log a pipeline job and compare different pipeline jobs.
 
+The steps performed include:
 
+* Formalize a training component
+* Build a training pipeline
+* Run several Pipeline jobs and log their results
+* Compare different Pipeline jobs
 
 ```
 
+   Learn more about [Vertex AI Experiments](https://cloud.google.com/vertex-ai/docs/experiments/intro-vertex-ai-experiments).
+
+   Learn more about [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction).
+

notebooks/official/experiments/build_model_experimentation_lineage_with_prebuild_code.ipynb

@@ -61,7 +61,9 @@
       "source": [
         "## Overview\n",
         "\n",
-        "As a Data Scientist, you want to be able to reuse code path (data preprocessing, feature engineering etc...) that others within your team have written to simplify and standardize all the complex data wrangling. "
+        "As a Data Scientist, you want to be able to reuse code path (data preprocessing, feature engineering etc...) that others within your team have written to simplify and standardize all the complex data wrangling. \n",
+        "\n",
+        "Learn more about [Vertex AI Experiments](https://cloud.google.com/vertex-ai/docs/experiments/intro-vertex-ai-experiments) and [Vertex ML Metadata](https://cloud.google.com/vertex-ai/docs/ml-metadata)."
       ]
     },
     {

notebooks/official/experiments/comparing_local_trained_models.ipynb

@@ -61,7 +61,9 @@
       "source": [
         "## Overview\n",
         "\n",
-        "As a Data Scientist, you probably start running model experiments locally on your notebook. Depending on the framework you use, you would need to track parameters, training time series and evaluation metrics. In this way, you would be able to explain the modelling approach you would choose. \n"
+        "As a Data Scientist, you probably start running model experiments locally on your notebook. Depending on the framework you use, you would need to track parameters, training time series and evaluation metrics. In this way, you would be able to explain the modelling approach you would choose. \n",
+        "\n",
+        "Learn more about [Vertex AI Experiments](https://cloud.google.com/vertex-ai/docs/experiments/intro-vertex-ai-experiments)."
       ]
     },
     {

notebooks/official/experiments/comparing_pipeline_runs.ipynb

@@ -61,7 +61,9 @@
       "source": [
         "## Overview\n",
         "\n",
-        "Depending on the model life cycle of your data science team, you would like to experiment and track training pipeline runs and its associated parameters. Then, you would to compare runs of these Pipelines to each others in order to figure out which is the best configuration generates the model you will register in the Vertex AI Model Registry."
+        "Depending on the model life cycle of your data science team, you would like to experiment and track training pipeline runs and its associated parameters. Then, you would to compare runs of these Pipelines to each others in order to figure out which is the best configuration generates the model you will register in the Vertex AI Model Registry.\n",
+        "\n",
+        "Learn more about [Vertex AI Experiments](https://cloud.google.com/vertex-ai/docs/experiments/intro-vertex-ai-experiments) and [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction)."
       ]
     },
     {

notebooks/official/explainable_ai/README.md

@@ -1,18 +1,3 @@
-[Custom training tabular regression model for batch prediction with explainabilty](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/explainable_ai/sdk_custom_tabular_regression_batch_explain.ipynb)
-
-```
-Learn how to use `Vertex AI Training and Explainable AI` to create a custom image classification model with explanations, and then you learn to use `Vertex AI Batch Prediction` to make a batch prediction request with explanations.
-
-The steps performed include:
-
-- Create a `Vertex AI` custom job for training a TensorFlow model.
-- View the model evaluation for the trained model.
-- Set explanation parameters for when the model is deployed.
-- Upload the trained model artifacts and explanations as a `Model` resource.
-- Make a batch prediction with explanations.
-
-```
-
 
 [AutoML training tabular binary classification model for batch explanation](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/explainable_ai/sdk_automl_tabular_binary_classification_batch_explain.ipynb)
 
@@ -28,6 +13,10 @@ The steps performed include:
 
 ```
 
+   Learn more about [AutoML Tabular](https://cloud.google.com/vertex-ai/docs/tabular-data/overview).
+
+   Learn more about [Vertex Explainable AI](https://cloud.google.com/vertex-ai/docs/explainable-ai/overview).
+
 
 [AutoML training tabular classification model for online explanation](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/explainable_ai/sdk_automl_tabular_classification_online_explain.ipynb)
 
@@ -46,6 +35,30 @@ The steps performed include:
 
 ```
 
+   Learn more about [AutoML Tabular](https://cloud.google.com/vertex-ai/docs/tabular-data/overview).
+
+   Learn more about [Vertex Explainable AI](https://cloud.google.com/vertex-ai/docs/explainable-ai/overview).
+
+
+[Custom training image classification model for batch prediction with explainabilty](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/explainable_ai/sdk_custom_image_classification_batch_explain.ipynb)
+
+```
+Learn to use `Vertex AI Training and Explainable AI` to create a custom image classification model with explanations, and then you learn to use `Vertex AI Batch Prediction` to make a batch prediction request with explanations.
+
+The steps performed include:
+
+- Create a `Vertex AI` custom job for training a TensorFlow model.
+- View the model evaluation for the trained model.
+- Set explanation parameters for when the model is deployed.
+- Upload the trained model artifacts and explanation parameters as a `Model` resource.
+- Make a batch prediction with explanations.
+
+```
+
+   Learn more about [Vertex Explainable AI](https://cloud.google.com/vertex-ai/docs/explainable-ai/overview).
+
+   Learn more about [Vertex AI Batch Prediction](https://cloud.google.com/vertex-ai/docs/tabular-data/classification-regression/get-batch-predictions).
+
 
 [Custom training image classification model for online prediction with explainabilty](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/explainable_ai/sdk_custom_image_classification_online_explain.ipynb)
 
@@ -65,6 +78,30 @@ The steps performed include:
 
 ```
 
+   Learn more about [Vertex Explainable AI](https://cloud.google.com/vertex-ai/docs/explainable-ai/overview).
+
+   Learn more about [Vertex AI Prediction](https://cloud.google.com/vertex-ai/docs/predictions/get-predictions).
+
+
+[Custom training tabular regression model for batch prediction with explainabilty](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/explainable_ai/sdk_custom_tabular_regression_batch_explain.ipynb)
+
+```
+Learn how to use `Vertex AI Training and Explainable AI` to create a custom image classification model with explanations, and then you learn to use `Vertex AI Batch Prediction` to make a batch prediction request with explanations.
+
+The steps performed include:
+
+- Create a `Vertex AI` custom job for training a TensorFlow model.
+- View the model evaluation for the trained model.
+- Set explanation parameters for when the model is deployed.
+- Upload the trained model artifacts and explanations as a `Model` resource.
+- Make a batch prediction with explanations.
+
+```
+
+   Learn more about [Vertex Explainable AI](https://cloud.google.com/vertex-ai/docs/explainable-ai/overview).
+
+   Learn more about [Vertex AI Batch Prediction](https://cloud.google.com/vertex-ai/docs/tabular-data/classification-regression/get-batch-predictions).
+
 
 [Custom training tabular regression model for online prediction with explainabilty](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/explainable_ai/sdk_custom_tabular_regression_online_explain.ipynb)
 
@@ -84,11 +121,15 @@ The steps performed include:
 
 ```
 
+   Learn more about [Vertex Explainable AI](https://cloud.google.com/vertex-ai/docs/explainable-ai/overview).
+
+   Learn more about [Vertex AI Prediction](https://cloud.google.com/vertex-ai/docs/predictions/get-predictions).
+
 
 [Custom training tabular regression model for online prediction with explainabilty using get_metadata](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/explainable_ai/sdk_custom_tabular_regression_online_explain_get_metadata.ipynb)
 
 ```
-Learn how to create a custom model from a Python script in a Google prebuilt Docker container using the Vertex SDK, and then do a prediction with explanations on the deployed model by sending data.
+Learn how to create a custom model from a Python script in a Google prebuilt Docker container using the Vertex AI SDK, and then do a prediction with explanations on the deployed model by sending data.
 
 The steps performed include:
 
@@ -104,19 +145,7 @@ The steps performed include:
 
 ```
 
+   Learn more about [Vertex Explainable AI](https://cloud.google.com/vertex-ai/docs/explainable-ai/overview).
 
-[Custom training image classification model for batch prediction with explainabilty](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/explainable_ai/sdk_custom_image_classification_batch_explain.ipynb)
-
-```
-Learn to use `Vertex AI Training and Explainable AI` to create a custom image classification model with explanations, and then you learn to use `Vertex AI Batch Prediction` to make a batch prediction request with explanations.
-
-The steps performed include:
-
-- Create a `Vertex AI` custom job for training a TensorFlow model.
-- View the model evaluation for the trained model.
-- Set explanation parameters for when the model is deployed.
-- Upload the trained model artifacts and explanation parameters as a `Model` resource.
-- Make a batch prediction with explanations.
-
-```
+   Learn more about [Vertex AI Prediction](https://cloud.google.com/vertex-ai/docs/predictions/get-predictions).
 

notebooks/official/explainable_ai/sdk_automl_tabular_binary_classification_batch_explain.ipynb

@@ -62,7 +62,9 @@
         "## Overview\n",
         "\n",
         "\n",
-        "This tutorial demonstrates how to use the Vertex AI SDK to create tabular binary classification models and do batch prediction with explanation using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) model."
+        "This tutorial demonstrates how to use the Vertex AI SDK to create tabular binary classification models and do batch prediction with explanation using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) model.\n",
+        "\n",
+        "Learn more about [AutoML Tabular](https://cloud.google.com/vertex-ai/docs/tabular-data/overview) and [Vertex Explainable AI](https://cloud.google.com/vertex-ai/docs/explainable-ai/overview)."
       ]
     },
     {

notebooks/official/explainable_ai/sdk_automl_tabular_classification_online_explain.ipynb

@@ -62,7 +62,9 @@
         "## Overview\n",
         "\n",
         "\n",
-        "This tutorial demonstrates how to use the Vertex AI SDK to create tabular classification models and do online prediction with explanation using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) model."
+        "This tutorial demonstrates how to use the Vertex AI SDK to create tabular classification models and do online prediction with explanation using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) model.\n",
+        "\n",
+        "Learn more about [AutoML Tabular](https://cloud.google.com/vertex-ai/docs/tabular-data/overview) and [Vertex Explainable AI](https://cloud.google.com/vertex-ai/docs/explainable-ai/overview)."
       ]
     },
     {

notebooks/official/explainable_ai/sdk_custom_image_classification_batch_explain.ipynb

@@ -62,7 +62,9 @@
         "## Overview\n",
         "\n",
         "\n",
-        "This tutorial demonstrates how to use the Vertex AI SDK to train and deploy a custom image classification model for batch prediction with explanation."
+        "This tutorial demonstrates how to use the Vertex AI SDK to train and deploy a custom image classification model for batch prediction with explanation.\n",
+        "\n",
+        "Learn more about [Vertex Explainable AI](https://cloud.google.com/vertex-ai/docs/explainable-ai/overview) and [Vertex AI Batch Prediction](https://cloud.google.com/vertex-ai/docs/tabular-data/classification-regression/get-batch-predictions)."
       ]
     },
     {

notebooks/official/explainable_ai/sdk_custom_image_classification_online_explain.ipynb

@@ -62,7 +62,9 @@
         "## Overview\n",
         "\n",
         "\n",
-        "This tutorial demonstrates how to use the Vertex AI SDK to train and deploy a custom image classification model for online prediction with explanation."
+        "This tutorial demonstrates how to use the Vertex AI SDK to train and deploy a custom image classification model for online prediction with explanation.\n",
+        "\n",
+        "Learn more about [Vertex Explainable AI](https://cloud.google.com/vertex-ai/docs/explainable-ai/overview) and [Vertex AI Prediction](https://cloud.google.com/vertex-ai/docs/predictions/get-predictions)."
       ]
     },
     {

notebooks/official/explainable_ai/sdk_custom_tabular_regression_batch_explain.ipynb

@@ -62,7 +62,9 @@
         "## Overview\n",
         "\n",
         "\n",
-        "This tutorial demonstrates how to use the Vertex AI SDK to train and deploy a custom tabular regression model for batch prediction with explanation."
+        "This tutorial demonstrates how to use the Vertex AI SDK to train and deploy a custom tabular regression model for batch prediction with explanation.\n",
+        "\n",
+        "Learn more about [Vertex Explainable AI](https://cloud.google.com/vertex-ai/docs/explainable-ai/overview) and [Vertex AI Batch Prediction](https://cloud.google.com/vertex-ai/docs/tabular-data/classification-regression/get-batch-predictions)."
       ]
     },
     {

notebooks/official/explainable_ai/sdk_custom_tabular_regression_online_explain.ipynb

@@ -62,7 +62,9 @@
         "## Overview\n",
         "\n",
         "\n",
-        "This tutorial demonstrates how to use the Vertex AI SDK to train and deploy a custom tabular regression model for online prediction with explanation."
+        "This tutorial demonstrates how to use the Vertex AI SDK to train and deploy a custom tabular regression model for online prediction with explanation.\n",
+        "\n",
+        "Learn more about [Vertex Explainable AI](https://cloud.google.com/vertex-ai/docs/explainable-ai/overview) and [Vertex AI Prediction](https://cloud.google.com/vertex-ai/docs/predictions/get-predictions)."
       ]
     },
     {

notebooks/official/explainable_ai/sdk_custom_tabular_regression_online_explain_get_metadata.ipynb

@@ -62,7 +62,9 @@
         "## Overview\n",
         "\n",
         "\n",
-        "This tutorial demonstrates how to use the Vertex AI SDK to train and deploy a custom tabular regression model for online prediction with explanation."
+        "This tutorial demonstrates how to use the Vertex AI SDK to train and deploy a custom tabular regression model for online prediction with explanation.\n",
+        "\n",
+        "Learn more about [Vertex Explainable AI](https://cloud.google.com/vertex-ai/docs/explainable-ai/overview) and [Vertex AI Prediction](https://cloud.google.com/vertex-ai/docs/predictions/get-predictions)."
       ]
     },
     {

notebooks/official/feature_store/README.md

@@ -1,3 +1,20 @@
+
+[Streaming ingestion SDK](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/feature_store/feature_store_streaming_ingestion_sdk.ipynb)
+
+```
+Learn how to ingest features from a `Pandas DataFrame` into your Vertex AI Feature Store using `write_feature_values` method from the Vertex AI SDK.
+
+The steps performed include:
+
+- Create `Feature Store`
+- Create new `Entity Type` for your `Feature Store`
+- Ingest feature values from `Pandas DataFrame` into `Feature Store`'s `Entity Types`.
+
+```
+
+   Learn more about [Vertex AI Feature Store](https://cloud.google.com/vertex-ai/docs/featurestore).
+
+
 [Using Vertex AI Feature Store with Pandas Dataframe](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/feature_store/sdk-feature-store-pandas.ipynb)
 
 ```
@@ -15,6 +32,8 @@ The steps performed include:
 
 ```
 
+   Learn more about [Vertex AI Feature Store](https://cloud.google.com/vertex-ai/docs/featurestore).
+
 
 [Online and Batch predictions using Vertex AI Feature Store](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/feature_store/sdk-feature-store.ipynb)
 
@@ -27,6 +46,9 @@ The steps performed include:
 - Import feature data into `Vertex AI Feature Store` resource.
 - Serve online prediction requests using the imported features.
 - Access imported features in offline jobs, such as training jobs.
+- Use streaming ingestion to ingest small amount of data.
 
 ```
 
+   Learn more about [Vertex AI Feature Store](https://cloud.google.com/vertex-ai/docs/featurestore).
+

notebooks/official/feature_store/feature_store_streaming_ingestion_sdk.ipynb

@@ -72,7 +72,9 @@
       "source": [
         "## Overview\n",
         "\n",
-        "This notebook demonstrates how to use Vertex AI Feature Store's streaming ingestion at the SDK layer."
+        "This notebook demonstrates how to use Vertex AI Feature Store's streaming ingestion at the SDK layer.\n",
+        "\n",
+        "Learn more about [Vertex AI Feature Store](https://cloud.google.com/vertex-ai/docs/featurestore)."
       ]
     },
     {

notebooks/official/feature_store/sdk-feature-store-pandas.ipynb

@@ -62,7 +62,9 @@
       "source": [
         "## Overview\n",
         "\n",
-        "This notebook introduces Pandas support for Feature Store using Vertex AI SDK. For pre-requisites and introduction on Vertex AI SDK and Feature Store native support, please go through this [Colab notebook](https://colab.sandbox.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/feature_store/sdk-feature-store.ipynb). "
+        "This notebook introduces Pandas support for Feature Store using Vertex AI SDK. For pre-requisites and introduction on Vertex AI SDK and Feature Store native support, please go through this [Colab notebook](https://colab.sandbox.google.com/github/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/feature_store/sdk-feature-store.ipynb). \n",
+        "\n",
+        "Learn more about [Vertex AI Feature Store](https://cloud.google.com/vertex-ai/docs/featurestore)."
       ]
     },
     {

notebooks/official/feature_store/sdk-feature-store.ipynb

@@ -62,7 +62,9 @@
         "\n",
         "This notebook introduces Vertex AI Feature Store, a managed cloud service for machine learning engineers and data scientists to store, serve, manage and share machine learning features at a large scale.\n",
         "\n",
-        "This notebook assumes that you understand basic Google Cloud concepts such as [Project](https://cloud.google.com/storage/docs/projects), [Storage](https://cloud.google.com/storage) and [Vertex AI](https://cloud.google.com/vertex-ai/docs). Some machine learning knowledge is also helpful but not required.\n"
+        "This notebook assumes that you understand basic Google Cloud concepts such as [Project](https://cloud.google.com/storage/docs/projects), [Storage](https://cloud.google.com/storage) and [Vertex AI](https://cloud.google.com/vertex-ai/docs). Some machine learning knowledge is also helpful but not required.\n",
+        "\n",
+        "Learn more about [Vertex AI Feature Store](https://cloud.google.com/vertex-ai/docs/featurestore)."
       ]
     },
     {
@@ -84,7 +86,8 @@
         "- Create featurestore, entity type, and feature resources.\n",
         "- Import feature data into `Vertex AI Feature Store` resource.\n",
         "- Serve online prediction requests using the imported features.\n",
-        "- Access imported features in offline jobs, such as training jobs."
+        "- Access imported features in offline jobs, such as training jobs.\n",
+        "- Use streaming ingestion to ingest small amount of data."
       ]
     },
     {
@@ -185,7 +188,7 @@
       "source": [
         "## Installation\n",
         "\n",
-        "Install the packages required for executing this notebook."
+        "Install the packages required to execute this notebook."
       ]
     },
     {
@@ -220,7 +223,7 @@
       "source": [
         "### Restart the kernel\n",
         "\n",
-        "After you install the SDK, you need to restart the notebook kernel so it can find the packages. You can restart kernel from *Kernel -> Restart Kernel*, or running the following:"
+        "After you install the SDK, you need to restart the notebook kernel so it can find the packages. You can restart kernel from *Kernel -> Restart Kernel*, or by running the following:"
       ]
     },
     {
@@ -256,14 +259,14 @@
         "\n",
         "1. [Make sure that billing is enabled for your project](https://cloud.google.com/billing/docs/how-to/modify-project).\n",
         "\n",
-        "1. [Enable the Vertex AI API and Compute Engine API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component).\n",
+        "1. [Enable the Vertex AI API and the Compute Engine API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component).\n",
         "\n",
-        "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
+        "1. If you are running this notebook locally, you need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
         "\n",
-        "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
+        "1. Enter your project ID in the cell below, and then run the cell to make sure the\n",
         "Cloud SDK uses the right project for all the commands in this notebook.\n",
         "\n",
-        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
+        "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and interpolates Python variables prefixed with `$` into these commands."
       ]
     },
     {
@@ -274,7 +277,7 @@
       "source": [
         "#### Set your project ID\n",
         "\n",
-        "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
+        "**If you don't know your project ID**, you can get your project ID using `gcloud`."
       ]
     },
     {
@@ -329,7 +332,7 @@
         "#### Region\n",
         "\n",
         "You can also change the `REGION` variable, which is used for operations\n",
-        "throughout the rest of this notebook.  Below are regions supported for Vertex AI. We recommend that you choose the region closest to you.\n",
+        "throughout the rest of this notebook. The following regions are supported for Vertex AI. We recommend that you choose the region closest to you.\n",
         "\n",
         "- Americas: `us-central1`\n",
         "- Europe: `europe-west4`\n",
@@ -361,7 +364,7 @@
       "source": [
         "#### UUID\n",
         "\n",
-        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a uuid for each instance session, and append it onto the name of resources you create in this tutorial."
+        "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name conflicts between users on resources created, you create a UUID for each instance session, and append it onto the name of resources you create in this tutorial."
       ]
     },
     {
@@ -376,7 +379,7 @@
         "import string\n",
         "\n",
         "\n",
-        "# Generate a uuid of a specifed length(default=8)\n",
+        "# Generate a UUID of a specifed length(default=8)\n",
         "def generate_uuid(length: int = 8) -> str:\n",
         "    return \"\".join(random.choices(string.ascii_lowercase + string.digits, k=length))\n",
         "\n",
@@ -452,7 +455,7 @@
         "\n",
         "        google_auth.authenticate_user()\n",
         "\n",
-        "    # If you are running this notebook locally, replace the string below with the\n",
+        "    # If you are running this notebook locally, replace the following string with the\n",
         "    # path to your service account key and run this cell to authenticate your GCP\n",
         "    # account.\n",
         "    elif not os.getenv(\"IS_TESTING\"):\n",
@@ -489,19 +492,19 @@
         "id": "h_HmF24mBHv9"
       },
       "source": [
-        "## Terminology and Concept\n",
+        "## Terminology and concept\n",
         "\n",
-        "### Featurestore Data model\n",
+        "### Featurestore data model\n",
         "\n",
         "Vertex AI Feature Store organizes data with the following 3 important hierarchical concepts:\n",
         "```\n",
         "Featurestore -> Entity type -> Feature\n",
         "```\n",
-        "* **Featurestore**: the place to store your features\n",
-        "* **Entity type**: under a Featurestore, an Entity type describes an object to be modeled, real one or virtual one.\n",
-        "* **Feature**: under an Entity type, a Feature describes an attribute of the Entity type\n",
+        "* **Featurestore**: The place to store your features\n",
+        "* **Entity type**: Under a featurestore, an entity type describes an object to be modeled, real one or virtual one.\n",
+        "* **Feature**: Under an entity type, a feature describes an attribute of the entity type\n",
         "\n",
-        "In the movie prediction example, you will create a featurestore called `movie_prediction`. This store has 2 entity types: `users` and `movies`. The `users` entity type has the `age`, `gender`, and `liked_genres` features. The `movies` entity type has the `titles`, `genres`, and `average rating` features.\n"
+        "The movie prediction example lets you create a featurestore called `movie_prediction`. This store has 2 entity types. `users` and `movies`. The `users` entity type has the `age`, `gender`, and `liked_genres` features. The `movies` entity type has the `titles`, `genres`, and `average rating` features.\n"
       ]
     },
     {
@@ -510,7 +513,7 @@
         "id": "9UvxYyGUimKw"
       },
       "source": [
-        "## Create Featurestore and Define Schemas"
+        "## Create featurestore and define schemas"
       ]
     },
     {
@@ -519,11 +522,11 @@
         "id": "buQBIv3ZL3A0"
       },
       "source": [
-        "### Create Featurestore\n",
+        "### Create featurestore\n",
         "\n",
-        "The method to create a Featurestore returns a\n",
+        "The method to create a featurestore returns a\n",
         "[long-running operation](https://google.aip.dev/151) (LRO). An LRO starts an asynchronous job. LROs are returned for other API\n",
-        "methods too, such as updating or deleting a featurestore. Running the code cell will create a featurestore and print the process log."
+        "methods too, such as updating or deleting a featurestore. Running the code cell creates a featurestore and print the process log."
       ]
     },
     {
@@ -549,7 +552,7 @@
         "id": "ag8pCQ7rNjVf"
       },
       "source": [
-        "Use the function call below to retrieve a Featurestore and check that it has been created.\n"
+        "Use the following function call to retrieve a featurestore and check that it has been created.\n"
       ]
     },
     {
@@ -574,9 +577,9 @@
         "id": "EpmJq75zXjmT"
       },
       "source": [
-        "### Create Entity Type\n",
+        "### Create entity Type\n",
         "\n",
-        "Entity types can be created within the Featurestore class. Below, create the Users entity type and Movies entity type. A process log will be printed out."
+        "Entity types can be created within the `Featurestore` class. Below, create the `users` and `movies` entity types. A process log is printed out."
       ]
     },
     {
@@ -587,7 +590,7 @@
       },
       "outputs": [],
       "source": [
-        "# Create users entity type\n",
+        "# Create the `users` entity type\n",
         "users_entity_type = fs.create_entity_type(\n",
         "    entity_type_id=\"users\",\n",
         "    description=\"Users entity\",\n",
@@ -602,7 +605,7 @@
       },
       "outputs": [],
       "source": [
-        "# Create movies entity type\n",
+        "# Create the `movies` entity type\n",
         "movies_entity_type = fs.create_entity_type(\n",
         "    entity_type_id=\"movies\",\n",
         "    description=\"Movies entity\",\n",
@@ -649,8 +652,8 @@
         "id": "FJW4q-0jO2Xf"
       },
       "source": [
-        "### Create Feature\n",
-        "Features can be created within each entity type. Add defining features to the Users entity type and Movies entity type by using the `create_feature` method."
+        "### Create feature\n",
+        "You can create features within each entity type. Use the `create_feature` method to add features to the `users` and `movies` entity types."
       ]
     },
     {
@@ -661,7 +664,7 @@
       },
       "outputs": [],
       "source": [
-        "# to create features one at a time use\n",
+        "# To create one feature at a time, use:\n",
         "users_feature_age = users_entity_type.create_feature(\n",
         "    feature_id=\"age\",\n",
         "    value_type=\"INT64\",\n",
@@ -687,7 +690,7 @@
         "id": "RQ9-AyFYBvcX"
       },
       "source": [
-        "Use the [list_features](https://github.com/googleapis/python-aiplatform/blob/main/google/cloud/aiplatform/featurestore/entity_type.py#L349) method to list all the features of a given entity type."
+        "Use the [`list_features`](https://github.com/googleapis/python-aiplatform/blob/main/google/cloud/aiplatform/featurestore/entity_type.py#L349) method to list all the features of a given entity type."
       ]
     },
     {
@@ -746,12 +749,14 @@
       "source": [
         "## Search created features\n",
         "\n",
-        "While the `list_features` method allows you to easily view all features of a single\n",
-        "entity type, the [search](https://github.com/googleapis/python-aiplatform/blob/main/google/cloud/aiplatform/featurestore/feature.py#L352) method in the Feature class searches across all featurestores and entity types in a given location (such as `us-central1`), and returns a list of features. This can help you discover features that were created by someone else.\n",
+        "While the `list_features` method lets you view all features for the same entity type,\n",
+        "the [`search`](https://github.com/googleapis/python-aiplatform/blob/main/google/cloud/aiplatform/featurestore/feature.py#L352) method in the `Feature` class searches across all featurestores and entity types in a given location (such as `us-central1`) and returns a list of features. This lets you discover features created by someone else.\n",
         "\n",
-        "You can query based on feature properties including feature ID, entity type ID, and feature description. You can also limit results by filtering on a specific featurestore, feature value type, and/or labels. Some search examples are shown below. \n",
+        "You can query based on feature properties including feature ID, entity type ID, and feature description. You can also limit results by filtering based on a specific featurestore, feature value type, and/or label. Some search examples are shown below. \n",
         "\n",
-        "Search for all features within a featurestore with the code snippet below."
+        "**Example of using the `search` method**\n",
+        "\n",
+        "Use the following code snippet to search for all features within a feature store:\n"
       ]
     },
     {
@@ -820,9 +825,9 @@
         "id": "K3n5XdK8Xjmw"
       },
       "source": [
-        "## Import Feature Values\n",
+        "## Import feature values\n",
         "\n",
-        "You need to import feature values before you can use them for online/offline serving. In this step, you learn how to import feature values by ingesting the values from Cloud Storage. You can also import feature values from BigQuery or a Pandas dataframe.\n"
+        "You need to import feature values before you can use them for online or offline serving. In this step, you learn how to import feature values by ingesting the values from GCS (Google Cloud Storage). You can also import feature values from BigQuery or a pandas dataFrame.\n"
       ]
     },
     {
@@ -831,11 +836,11 @@
         "id": "BlqJ-QdTcs6W"
       },
       "source": [
-        "### Source Data Format and Layout\n",
+        "### Source data format and layout\n",
         "\n",
-        "BigQuery table/Avro/CSV are supported as input data types. No matter what format you are using, each imported entity *must* have an ID; also, each entity can *optionally* have a timestamp, specifying when the feature values are generated. This notebook uses Avro as an input, located at this public [bucket](https://console.cloud.google.com/storage/browser/cloud-samples-data-us-central1/vertex-ai/feature-store/datasets). The Avro schemas are as follows:\n",
+        "BigQuery table/Avro/CSV are supported as input data types. No matter what format you are using, each imported entity *must* have an ID. Each entity can *optionally* have a timestamp, specifying when the feature values are generated. This notebook uses Avro as an input, located at this public [bucket](https://console.cloud.google.com/storage/browser/cloud-samples-data-us-central1/vertex-ai/feature-store/datasets). The Avro schemas are as follows:\n",
         "\n",
-        "**For the Users entity**:\n",
+        "**For the `users` entity**:\n",
         "```\n",
         "schema = {\n",
         "  \"type\": \"record\",\n",
@@ -865,7 +870,7 @@
         " }\n",
         "```\n",
         "\n",
-        "**For the Movies entity**:\n",
+        "**For the `movies` entity**:\n",
         "```\n",
         "schema = {\n",
         " \"type\": \"record\",\n",
@@ -902,7 +907,7 @@
         "id": "m7DyDa6chbJx"
       },
       "source": [
-        "### Import feature values for Users entity type\n",
+        "### Import feature values for `users` entity type\n",
         "\n",
         "When importing, specify the following in your request:\n",
         "\n",
@@ -955,9 +960,9 @@
         "id": "laXdJPIqkLJO"
       },
       "source": [
-        "### Import feature values for Movies entity type\n",
+        "### Import feature values for `movies` entity type\n",
         "\n",
-        "Similarly, import feature values for the Movies entity type into the featurestore.\n"
+        "Similarly, import feature values for the `movies` entity type into the featurestore.\n"
       ]
     },
     {
@@ -1014,7 +1019,7 @@
       },
       "source": [
         "[Online serving](https://cloud.google.com/vertex-ai/docs/featurestore/serving-online)\n",
-        "lets you serve feature values for small batches of entities. It's designed for latency-sensitive service, such as online model prediction. For example, for a movie service, you might want to quickly show movies that the current user would most likely watch."
+        "lets you serve feature values for small batches of entities. It's designed for latency-sensitive services, such as online model prediction. For example, for a movie service, you might want to quickly show movies that the current user would most likely watch."
       ]
     },
     {
@@ -1025,9 +1030,9 @@
       "source": [
         "### Read one entity per request\n",
         "\n",
-        "With the Vertex AI SDK, it is easy to read feature values of one entity. By default, the SDK will return the  latest value of each feature, meaning the feature values with the most recent  timestamp.\n",
+        "With the Python SDK, it's easy to read feature values of one entity. By default, the SDK returns the latest value of each feature, that is, the feature values with the most recent timestamps.\n",
         "\n",
-        "To read feature values, specify the entity type ID and features to read. By default all the features of an entity type will be selected. The response will output and display the selected entity type ID and the selected feature values as a Pandas dataframe."
+        "To read feature values, specify the entity type ID and features to read. By default all the features of an entity type are selected. The output response displays the selected entity type ID and the selected feature values as a Pandas dataframe."
       ]
     },
     {
@@ -1060,7 +1065,7 @@
       "source": [
         "### Read multiple entities per request\n",
         "\n",
-        "To read feature values from multiple entities, specify the different entity type IDs. By default all the features of an entity type will be selected. Note that fetching only a small number of entities is recommended when using this SDK due to its latency-sensitive nature."
+        "To read feature values from multiple entities, specify the different entity type IDs. By default, all the features of an entity type are selected. Note that fetching only a small number of entities is recommended when using this SDK due to its latency-sensitive nature."
       ]
     },
     {
@@ -1115,16 +1120,16 @@
       "source": [
         "### Use case\n",
         "\n",
-        "**The task** is to prepare a training dataset to train a model, which predicts if a given user will watch a given movie. To achieve this, you need 2 sets of input:\n",
+        "**The task** is to prepare a training dataset to train a model, which predicts if a given user is going to watch a movie. To achieve this, you need 2 sets of input:\n",
         "\n",
         "*   Features: you already imported into the featurestore.\n",
         "*   Labels: the ground-truth data recorded that user X has watched movie Y.\n",
         "\n",
         "\n",
         "To be more specific, the ground-truth observation is described in Table 1 and the desired training dataset is described in Table 2. Each row in Table 2 is a result of joining the imported feature values from Vertex AI Feature Store according to the entity IDs and timestamps in Table 1. In this example,  the `age`, `gender` and `liked_genres` features from `users` and\n",
-        "the `titles`, `genres` and `average_rating` features from `movies` are chosen to train the model. Note that only positive examples are shown in these 2 tables, i.e., you can imagine there is a label column whose values are all `True`.\n",
+        "the `titles`, `genres` and `average_rating` features from `movies` are chosen to train the model. Note that only positive examples are shown in these 2 tables, that is, you can imagine there is a label column whose values are all `True`.\n",
         "\n",
-        "[batch_serve_to_bq](https://github.com/googleapis/python-aiplatform/blob/main/google/cloud/aiplatform/featurestore/featurestore.py#L770) takes Table 1 as\n",
+        "[`batch_serve_to_bq`](https://github.com/googleapis/python-aiplatform/blob/main/google/cloud/aiplatform/featurestore/featurestore.py#L770) takes Table 1 as\n",
         "input, joins all required feature values from the featurestore, and returns Table 2 for training.\n",
         "\n",
         "<h4 align=\"center\">Table 1. Ground-truth data</h4>\n",
@@ -1154,7 +1159,7 @@
       "source": [
         "#### Why timestamp?\n",
         "\n",
-        "Note that there is a `timestamp` column in Table 2. This indicates the time when the ground-truth was observed. This is to avoid data inconsistency.\n",
+        "Note that there is a `timestamp` column in Table 2 to indicate the time when the ground-truth was observed. This is to avoid data inconsistency.\n",
         "\n",
         "For example, the 2nd row of Table 2 indicates that user `alice` watched movie `Cinema Paradiso` on `2019-11-01T00:00:00Z`. The featurestore keeps feature values for all timestamps but fetches feature values *only* at the given timestamp during batch serving. On that day, Alice might have been 54 years old, but now Alice might be 56; featurestore returns `age=54` as Alice's age, instead of `age=56`, because that is the value of the feature at the observation time. Similarly, other features might be time-variant as well, such as `liked_genres`."
       ]
@@ -1167,7 +1172,7 @@
       "source": [
         "### Create BigQuery dataset for output\n",
         "\n",
-        "You need a BigQuery dataset to host the output data in `us-central1`. Input the name of the dataset you want to create and specify the name of the table you want to store the output created later. These will be used in the next section.\n",
+        "You need a BigQuery dataset to host the output data in `us-central1`. Input the name of the dataset you want to create and specify the name of the table you want to store the output created later. These are used in the next section.\n",
         "\n",
         "**Make sure that the table name does NOT already exist**.\n"
       ]
@@ -1232,9 +1237,9 @@
         "id": "W8dLJ9nuDFgI"
       },
       "source": [
-        "### Batch Read Feature Values\n",
+        "### Batch read feature values\n",
         "\n",
-        "Assemble the request which specify the following info:\n",
+        "Assemble the request which specifies the following info:\n",
         "\n",
         "*   Where is the label data, i.e., Table 1.\n",
         "*   Which features are read, i.e., the column names in Table 2.\n",
@@ -1281,6 +1286,96 @@
         "After the LRO finishes, you should be able to see the result in the [BigQuery console](https://console.cloud.google.com/bigquery), as a new table under the BigQuery dataset created earlier."
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "7190f3c8b625"
+      },
+      "source": [
+        "## Streaming ingestion\n",
+        "\n",
+        "Streaming ingestion is currently public preview. \n",
+        "\n",
+        "Streaming ingestion lets you make real-time updates to feature values. While batch import is suitable for importing a large volume of data with high latency, streaming ingestion is suitable for ingesting small amount of data with low latency. The written data becomes available to read using batch export and online serving."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "560e835c93db"
+      },
+      "outputs": [],
+      "source": [
+        "# Since streaming ingestion is public preview, the feature is available in aiplatform_v1beta1.\n",
+        "from google.cloud.aiplatform_v1beta1 import (\n",
+        "    FeaturestoreOnlineServingServiceClient, FeaturestoreServiceClient)\n",
+        "from google.cloud.aiplatform_v1beta1.types import \\\n",
+        "    featurestore_online_service as featurestore_online_service_pb2\n",
+        "from google.cloud.aiplatform_v1beta1.types import types as types_pb2\n",
+        "\n",
+        "API_ENDPOINT = \"{}-aiplatform.googleapis.com\".format(REGION)\n",
+        "# Create client connection\n",
+        "admin_client = FeaturestoreServiceClient(client_options={\"api_endpoint\": API_ENDPOINT})\n",
+        "data_client = FeaturestoreOnlineServingServiceClient(\n",
+        "    client_options={\"api_endpoint\": API_ENDPOINT}\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "f53a06c9ab5c"
+      },
+      "outputs": [],
+      "source": [
+        "# Call `write_feature_values` to ingest data to `users` entity type.\n",
+        "data_client.write_feature_values(\n",
+        "    entity_type=admin_client.entity_type_path(\n",
+        "        PROJECT_ID, REGION, FEATURESTORE_ID, \"users\"\n",
+        "    ),\n",
+        "    payloads=[\n",
+        "        featurestore_online_service_pb2.WriteFeatureValuesPayload(\n",
+        "            entity_id=\"1305\",\n",
+        "            feature_values={\n",
+        "                \"age\": featurestore_online_service_pb2.FeatureValue(int64_value=34),\n",
+        "                \"gender\": featurestore_online_service_pb2.FeatureValue(\n",
+        "                    string_value=\"female\"\n",
+        "                ),\n",
+        "                \"liked_genres\": featurestore_online_service_pb2.FeatureValue(\n",
+        "                    string_array_value=types_pb2.StringArray(values=[\"drama\", \"action\"])\n",
+        "                ),\n",
+        "            },\n",
+        "        ),\n",
+        "        featurestore_online_service_pb2.WriteFeatureValuesPayload(\n",
+        "            entity_id=\"1306\",\n",
+        "            feature_values={\n",
+        "                \"age\": featurestore_online_service_pb2.FeatureValue(int64_value=50),\n",
+        "                \"gender\": featurestore_online_service_pb2.FeatureValue(\n",
+        "                    string_value=\"male\"\n",
+        "                ),\n",
+        "                \"liked_genres\": featurestore_online_service_pb2.FeatureValue(\n",
+        "                    string_array_value=types_pb2.StringArray(\n",
+        "                        values=[\"suspense\", \"comedy\"]\n",
+        "                    )\n",
+        "                ),\n",
+        "            },\n",
+        "        ),\n",
+        "    ],\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "700a9f1ebd19"
+      },
+      "source": [
+        "Upon successful completion, the `write_feature_values` API returns an empty response.\n",
+        "Similarly, ingest data to the `movies` entity type"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -1292,7 +1387,7 @@
         "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
         "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
         "\n",
-        "You can also keep the project but delete the featurestore and the BigQuery dataset by running the code below:"
+        "You can also keep the project, but delete the featurestore and the BigQuery dataset by running the following code:"
       ]
     },
     {

notebooks/official/matching_engine/README.md

@@ -1,18 +1,3 @@
-[Create Vertex AI Matching Engine index](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/matching_engine/sdk_matching_engine_for_indexing.ipynb)
-
-```
-Learn how to create Approximate Nearest Neighbor (ANN) Index, query against indexes, and validate the performance of the index.
-
-The steps performed include:
-
-* Create ANN Index and Brute Force Index
-* Create an IndexEndpoint with VPC Network
-* Deploy ANN Index and Brute Force Index
-* Perform online query
-* Compute recall
-
-```
-
 
 [Introduction to builtin Swivel embedding algorithm](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/matching_engine/intro-swivel.ipynb)
 
@@ -30,11 +15,31 @@ The steps performed include:
 
 ```
 
+   Learn more about [Vertex AI Matching Engine](https://cloud.google.com/vertex-ai/docs/matching-engine/overview).
 
-[Introduction to builtin Two-towers embedding algorithm](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/matching_engine/two-tower-model-introduction.ipynb)
+
+[Create Vertex AI Matching Engine index](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/matching_engine/sdk_matching_engine_for_indexing.ipynb)
 
 ```
-Learn how to run the two-tower model.
+Learn how to create Approximate Nearest Neighbor (ANN) Index, query against indexes, and validate the performance of the index.
+
+The steps performed include:
+
+* Create ANN Index and Brute Force Index
+* Create an IndexEndpoint with VPC Network
+* Deploy ANN Index and Brute Force Index
+* Perform online query
+* Compute recall
+
+```
+
+   Learn more about [Vertex AI Matching Engine](https://cloud.google.com/vertex-ai/docs/matching-engine/overview).
+
+
+[Introduction to builtin Two-Towers embedding algorithm](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/matching_engine/two-tower-model-introduction.ipynb)
+
+```
+Learn how to run the Two-Tower model.
 
 The steps performed include:
 1. **Setup**: Importing the required libraries and setting your global variables.
@@ -47,3 +52,5 @@ The steps performed include:
 
 ```
 
+   Learn more about [Vertex AI Matching Engine](https://cloud.google.com/vertex-ai/docs/matching-engine/overview).
+

notebooks/official/matching_engine/intro-swivel.ipynb

@@ -60,7 +60,7 @@
       "source": [
         "## Overview\n",
         "\n",
-        "This notebook demonstrate how to train an embedding with Submatrix-wise Vector Embedding Learner ([Swivel](https://arxiv.org/abs/1602.02215)) using Vertex Pipelines. The purpose of the embedding learner is to compute cooccurrences between tokens in a given dataset and to use the cooccurrences to generate embeddings.\n",
+        "This notebook demonstrate how to train an embedding with Submatrix-wise Vector Embedding Learner ([Swivel](https://arxiv.org/abs/1602.02215)) using Vertex AI Pipelines. The purpose of the embedding learner is to compute cooccurrences between tokens in a given dataset and to use the cooccurrences to generate embeddings.\n",
         "\n",
         "Vertex AI provides a pipeline template\n",
         "for training with Swivel, so you don't need to design your own pipeline or write\n",
@@ -68,7 +68,9 @@
         "\n",
         "It will require you provide a bucket where the dataset will be stored.\n",
         "\n",
-        "Note: you may incur charges for training, storage or usage of other GCP products (Dataflow) in connection with testing this SDK.\n"
+        "Note: you may incur charges for training, storage or usage of other GCP products (Dataflow) in connection with testing this SDK.\n",
+        "\n",
+        "Learn more about [Vertex AI Matching Engine](https://cloud.google.com/vertex-ai/docs/matching-engine/overview)."
       ]
     },
     {

notebooks/official/matching_engine/sdk_matching_engine_for_indexing.ipynb

@@ -60,7 +60,9 @@
       "source": [
         "## Overview\n",
         "\n",
-        "This example demonstrates how to use the Vertex AI ANN Service. It is a high scale, low latency solution, to find similar vectors (or more specifically \"embeddings\") for a large corpus. Moreover, it is a fully managed offering, further reducing operational overhead. It is built upon [Approximate Nearest Neighbor (ANN) technology](https://ai.googleblog.com/2020/07/announcing-scann-efficient-vector.html) developed by Google Research."
+        "This example demonstrates how to use the Vertex AI ANN Service. It is a high scale, low latency solution, to find similar vectors (or more specifically \"embeddings\") for a large corpus. Moreover, it is a fully managed offering, further reducing operational overhead. It is built upon [Approximate Nearest Neighbor (ANN) technology](https://ai.googleblog.com/2020/07/announcing-scann-efficient-vector.html) developed by Google Research.\n",
+        "\n",
+        "Learn more about [Vertex AI Matching Engine](https://cloud.google.com/vertex-ai/docs/matching-engine/overview)."
       ]
     },
     {

notebooks/official/matching_engine/two-tower-model-introduction.ipynb

@@ -62,7 +62,9 @@
         "\n",
         "This tutorial demonstrates how to use the Two-Tower built-in algorithm on the Vertex AI platform.\n",
         "\n",
-        "Two-tower models learn to represent two items of various types (such as user profiles, search queries, web documents, answer passages, or images) in the same vector space, so that similar or related items are close to each other. These two items are referred to as the query and candidate object, since when paired with a nearest neighbor search service such as Vertex Matching Engine, the two-tower model can retrieve candidate objects related to an input query object. These objects are encoded by a query and candidate encoder (the two \"towers\") respectively, which are trained on pairs of relevant items. This built-in algorithm exports trained query and candidate encoders as model artifacts, which can be deployed in Vertex Prediction for usage in a recommendation system.\n"
+        "Two-tower models learn to represent two items of various types (such as user profiles, search queries, web documents, answer passages, or images) in the same vector space, so that similar or related items are close to each other. These two items are referred to as the query and candidate object, since when paired with a nearest neighbor search service such as Vertex AI Matching Engine, the two-tower model can retrieve candidate objects related to an input query object. These objects are encoded by a query and candidate encoder (the two \"towers\") respectively, which are trained on pairs of relevant items. This built-in algorithm exports trained query and candidate encoders as model artifacts, which can be deployed in Vertex Prediction for usage in a recommendation system.\n",
+        "\n",
+        "Learn more about [Vertex AI Matching Engine](https://cloud.google.com/vertex-ai/docs/matching-engine/overview)."
       ]
     },
     {

notebooks/official/migration/README.md

@@ -1,3 +1,117 @@
+
+[AutoML Image Classification](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ1 Vertex SDK AutoML Image Classification.ipynb)
+
+```
+Learn to use `AutoML` to train an image model and use `Vertex AI Prediction` and `Vertex AI Batch Prediction` to do online and batch predictions.
+
+The steps performed include:
+
+- Train an AutoML image classification model.
+- Make a batch prediction.
+- Deploy model to a endpoint
+- Make a online prediction
+
+```
+
+   Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai).
+
+   Learn more about [AutoML Image](https://cloud.google.com/vertex-ai/docs/tutorials/image-recognition-automl/training).
+
+
+[Custom Scikit-Learn model with pre-built training container](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ10 Vertex SDK Custom Scikit-Learn with pre-built training container.ipynb)
+
+```
+Learn to use `Vertex AI Training` to create a custom trained model and use `Vertex AI Batch Prediction` to do a batch prediction on the trained model.
+
+The steps performed include:
+
+- Create a `Vertex AI` custom job for training a scikit-learn model.
+- Upload the trained model artifacts as a `Model` resource.
+- Make a batch prediction.
+- Deploy model to a endpoint
+- Make a online prediction
+
+```
+
+   Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai).
+
+   Learn more about [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training).
+
+
+[Hyperparameter Tuning](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ11 Vertex SDK Hyperparameter Tuning.ipynb)
+
+```
+Learn to use `Vertex AI Hyperparameter` to create and tune a custom trained model.
+
+The steps performed include:
+
+- Create a `Vertex AI` hyperparameter tuning job for training a TensorFlow model.
+
+```
+
+   Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/training/hyperparameter-tuning-overview).
+
+   Learn more about [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training).
+
+
+[AutoML Video Classificaton](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ14 Vertex SDK AutoML Video Classification.ipynb)
+
+```
+Learn to use `AutoML` to train a video model and use `Vertex AI Batch Prediction` to do batch predictions.
+
+The steps performed include:
+
+- Train an AutoML video classification model.
+- Make a batch prediction.
+
+```
+
+   Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai).
+
+   Learn more about [AutoML Video](https://cloud.google.com/vertex-ai/docs/tutorials/video-classification-automl/training).
+
+
+[AutoML Video Object Tracking](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ15 Vertex SDK AutoML Object Tracking.ipynb)
+
+```
+Learn to use `AutoML` to train a video model and use `Vertex AI Batch Prediction` to do batch predictions.
+
+The steps performed include:
+
+- Train an AutoML video object tracking model.
+- Make a batch prediction.
+
+```
+
+   Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai).
+
+   Learn more about [AutoML Video](https://cloud.google.com/video-intelligence/automl/object-tracking/docs/index-object-tracking).
+
+
+[Custom Image Classification w/pre-built training container](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ2,12 Vertex SDK Custom Image Classification with pre-built training container.ipynb)
+
+```
+Learn how to train a tensorflow image classification model using a prebuilt container and Vertex AI training.
+
+The steps performed include:
+
+- *Package the training code into a python application.*
+- *Containerize the training application using Cloud Build and Artifact Registry.*
+- *Create a custom container training job in Vertex AI and run it.*
+- *Evaluate the model generated from the training job.*
+- *Create a model resource for the trained model in Vertex AI Model Registry.*
+- *Run a Vertex AI batch prediction job.*
+- *Deploy the model resource to a Vertex AI Endpoint.*
+- *Run a online prediction job on the model resource.*
+- *Clean up the resources created.*
+
+```
+
+   Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai).
+
+   Learn more about [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training).
+
+
 [Custom Image Classification w/custom training container](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ3 Vertex SDK Custom Image Classification with custom training container.ipynb)
 
 ```
@@ -17,6 +131,10 @@ The steps performed include:
 
 ```
 
+   Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai).
+
+   Learn more about [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training).
+
 
 [AutoML Tabular Binary Classification](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ4 Vertex SDK AutoML Tabular Binary Classification.ipynb)
 
@@ -34,39 +152,28 @@ The steps performed include:
 
 ```
 
+   Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai).
 
-[Custom Scikit-Learn model with pre-built training container](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ10 Vertex SDK Custom Scikit-Learn with pre-built training container.ipynb)
+   Learn more about [AutoML Tabular](https://cloud.google.com/vertex-ai/docs/start/automl-users#tables).
+
+
+[AutoML Image Object Detection](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ5 Vertex SDK AutoML Image Object Detection.ipynb)
 
 ```
-Learn to use `Vertex AI Training` to create a custom trained model and use `Vertex AI Batch Prediction` to do a batch prediction on the trained model.
+Learn to use `AutoML` to train an image model and use `Vertex AI Prediction` and `Vertex AI Batch Prediction` to do online and batch predictions.
 
 The steps performed include:
 
-- Create a `Vertex AI` custom job for training a scikit-learn model.
-- Upload the trained model artifacts as a `Model` resource.
+- Train an AutoML object detection model.
 - Make a batch prediction.
 - Deploy model to a endpoint
 - Make a online prediction
 
 ```
 
+   Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai).
 
-[AutoML Text Entity Extraction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ7 Vertex SDK AutoML Text Entity Extraction.ipynb)
-
-```
-The objective of this notebook is to build a AutoML Text Entity Extrasction Model.
-
-The steps performed include the following:
-
-* Set your task name, and GCS prefix
-* Copy AutoML video demo train data for creating managed dataset
-* Create a dataset on Vertex AI.
-* Configure a training job
-* Launch a training job and create a model on Vertex AI
-* Copy AutoML Video Demo Prediction Data for creating batch prediction job
-* Perform batch prediction job on the model
-
-```
+   Learn more about [AutoML Image](https://cloud.google.com/vertex-ai/docs/tutorials/image-recognition-automl/training).
 
 
 [AutoML Text Classification](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ6 Vertex SDK AutoML Text Classification.ipynb)
@@ -86,3 +193,70 @@ The steps performed include the following:
 
 ```
 
+   Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai).
+
+   Learn more about [AutoML Text](https://cloud.google.com/vertex-ai/docs/text-data/classification/prepare-data).
+
+
+[AutoML Text Entity Extraction](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ7 Vertex SDK AutoML Text Entity Extraction.ipynb)
+
+```
+The objective of this notebook is to build a AutoML Text Entity Extraction Model.
+
+The steps performed include the following:
+
+* Set your task name, and GCS prefix
+* Copy AutoML video demo train data for creating managed dataset
+* Create a dataset on Vertex AI.
+* Configure a training job
+* Launch a training job and create a model on Vertex AI
+* Copy AutoML Video Demo Prediction Data for creating batch prediction job
+* Perform batch prediction job on the model
+
+```
+
+   Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai).
+
+   Learn more about [AutoML Text](https://cloud.google.com/vertex-ai/docs/text-data/entity-extraction/prepare-data).
+
+
+[AutoML Text Sentiment Analysis](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ8 Vertex SDK AutoML Text Sentiment Analysis.ipynb)
+
+```
+The objective of this notebook is to build a AutoML Text Sentiment Analysis model.
+
+The steps performed include the following:
+
+* Copy AutoML video demo train data for creating managed dataset
+* Create a dataset on Vertex AI.
+* Configure a training job
+* Launch a training job and create a model on Vertex AI
+* Copy AutoML Video Demo Prediction Data for creating batch prediction job
+* Perform batch prediction job on the model
+
+```
+
+   Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai).
+
+   Learn more about [AutoML Text](https://cloud.google.com/vertex-ai/docs/text-data/sentiment-analysis/prepare-data).
+
+
+[Custom XGBoost model with pre-built training container](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/migration/UJ9 Vertex SDK Custom XGBoost with pre-built training container.ipynb)
+
+```
+Learn to use `Vertex AI Training` to create a custom trained model and use `Vertex AI Batch Prediction` to do a batch prediction on the trained model.
+
+The steps performed include:
+
+- Create a `Vertex AI` custom job for training a scikit-learn model.
+- Upload the trained model artifacts as a `Model` resource.
+- Make a batch prediction.
+- Deploy model to a endpoint
+- Make a online prediction
+
+```
+
+   Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai).
+
+   Learn more about [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training).
+

notebooks/official/migration/UJ1 Vertex SDK AutoML Image Classification.ipynb

@@ -54,6 +54,45 @@
         "<br/><br/><br/>"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "7a8a13b86a8b"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "This tutorial demonstrates how to use the Vertex AI SDK for Python to train and deploy an AutoML image classification model.\n",
+        "\n",
+        "Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai) and [AutoML Image](https://cloud.google.com/vertex-ai/docs/tutorials/image-recognition-automl/training)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "618cfedf829a"
+      },
+      "source": [
+        "### Objective\n",
+        "\n",
+        "In this tutorial, you learn to use `AutoML` to train an image model and use `Vertex AI Prediction` and `Vertex AI Batch Prediction` to do online and batch predictions.\n",
+        "\n",
+        "\n",
+        "This tutorial uses the following Google Cloud ML services:\n",
+        "\n",
+        "- `AutoML`\n",
+        "- `Vertex AI Batch Prediction`\n",
+        "- `Vertex AI Model` resource\n",
+        "- `Vertex AI Endpoint` resource\n",
+        "\n",
+        "The steps performed include:\n",
+        "\n",
+        "- Train an AutoML image classification model.\n",
+        "- Make a batch prediction.\n",
+        "- Deploy model to a endpoint\n",
+        "- Make a online prediction"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {

notebooks/official/migration/UJ10 Vertex SDK Custom Scikit-Learn with pre-built training container.ipynb

@@ -62,7 +62,9 @@
         "## Overview\n",
         "\n",
         "\n",
-        "This tutorial demonstrates how to use the Vertex AI SDK for Python to train and deploy a custom tabular classification scikit-learn model for batch prediction."
+        "This tutorial demonstrates how to use the Vertex AI SDK for Python to train and deploy a custom tabular classification scikit-learn model for batch prediction.\n",
+        "\n",
+        "Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai) and [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training)."
       ]
     },
     {

notebooks/official/migration/UJ11 Vertex SDK Hyperparameter Tuning.ipynb

@@ -54,6 +54,42 @@
         "<br/><br/><br/>"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "7a8a13b86a8b"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "\n",
+        "This tutorial demonstrates how to use the Vertex AI SDK for Python to hyperparamer tune a custom tabular classification TemsorFlow model.\n",
+        "\n",
+        "Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/training/hyperparameter-tuning-overview) and [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "618cfedf829a"
+      },
+      "source": [
+        "### Objective\n",
+        "\n",
+        "In this tutorial, you learn to use `Vertex AI Hyperparameter` to create and tune a custom trained model.\n",
+        "\n",
+        "You learn how to create and tune a custom-trained model from a Python script in a Docker container using the Vertex AI SDK for Python.\n",
+        "\n",
+        "This tutorial uses the following Google Cloud ML services:\n",
+        "\n",
+        "- `Vertex AI Training`\n",
+        "- `Vertex AI Hyperparameter Tuning`\n",
+        "\n",
+        "The steps performed include:\n",
+        "\n",
+        "- Create a `Vertex AI` hyperparameter tuning job for training a TensorFlow model."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {

notebooks/official/migration/UJ14 Vertex SDK AutoML Video Classification.ipynb

@@ -54,6 +54,43 @@
         "<br/><br/><br/>"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "7a8a13b86a8b"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "This tutorial demonstrates how to use the Vertex AI SDK for Python to train a AutoML video classification model and do a batch prediction.\n",
+        "\n",
+        "Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai) and [AutoML Video](https://cloud.google.com/vertex-ai/docs/tutorials/video-classification-automl/training)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "618cfedf829a"
+      },
+      "source": [
+        "### Objective\n",
+        "\n",
+        "In this tutorial, you learn to use `AutoML` to train a video model and use `Vertex AI Batch Prediction` to do batch predictions.\n",
+        "\n",
+        "\n",
+        "This tutorial uses the following Google Cloud ML services:\n",
+        "\n",
+        "- `AutoML`\n",
+        "- `Vertex AI Batch Prediction`\n",
+        "- `Vertex AI Model` resource\n",
+        "- `Vertex AI Endpoint` resource\n",
+        "\n",
+        "The steps performed include:\n",
+        "\n",
+        "- Train an AutoML video classification model.\n",
+        "- Make a batch prediction."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {

notebooks/official/migration/UJ15 Vertex SDK AutoML Object Tracking.ipynb

@@ -54,6 +54,43 @@
         "<br/><br/><br/>"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "7a8a13b86a8b"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "This tutorial demonstrates how to use the Vertex AI SDK for Python to train a AutoML video object tracking model and do a batch prediction.\n",
+        "\n",
+        "Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai) and [AutoML Video](https://cloud.google.com/video-intelligence/automl/object-tracking/docs/index-object-tracking)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "618cfedf829a"
+      },
+      "source": [
+        "### Objective\n",
+        "\n",
+        "In this tutorial, you learn to use `AutoML` to train a video model and use `Vertex AI Batch Prediction` to do batch predictions.\n",
+        "\n",
+        "\n",
+        "This tutorial uses the following Google Cloud ML services:\n",
+        "\n",
+        "- `AutoML`\n",
+        "- `Vertex AI Batch Prediction`\n",
+        "- `Vertex AI Model` resource\n",
+        "- `Vertex AI Endpoint` resource\n",
+        "\n",
+        "The steps performed include:\n",
+        "\n",
+        "- Train an AutoML video object tracking model.\n",
+        "- Make a batch prediction."
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {

notebooks/official/migration/UJ2,12 Vertex SDK Custom Image Classification with pre-built training container.ipynb

@@ -54,6 +54,51 @@
         "<br/><br/><br/>"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "7a8a13b86a8b"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "This tutorial demonstrates how to use the Vertex AI SDK for Python to train using a pre-built container and deploy a custom image classification model for online and batch prediction.\n",
+        "\n",
+        "Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai) and [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "f1ae7d54ad29"
+      },
+      "source": [
+        "### Objective\n",
+        "\n",
+        "In this tutorial, you learn how to train a tensorflow image classification model using a prebuilt container and Vertex AI training. After training, you also deploy the model to Vertex AI using a pre-built container and generate both batch and online predictions on it. \n",
+        "\n",
+        "This tutorial uses the following Google Cloud ML services and resources:\n",
+        "\n",
+        "- Vertex AI Training\n",
+        "- Vertex AI Model Registry\n",
+        "- Vertex AI Predictions\n",
+        "- Vertex AI Batch Predictions\n",
+        "- Vertex AI Endpoints\n",
+        "\n",
+        "\n",
+        "The steps performed include:\n",
+        "\n",
+        "- *Package the training code into a python application.*\n",
+        "- *Containerize the training application using Cloud Build and Artifact Registry.*\n",
+        "- *Create a custom container training job in Vertex AI and run it.*\n",
+        "- *Evaluate the model generated from the training job.*\n",
+        "- *Create a model resource for the trained model in Vertex AI Model Registry.*\n",
+        "- *Run a Vertex AI batch prediction job.*\n",
+        "- *Deploy the model resource to a Vertex AI Endpoint.*\n",
+        "- *Run a online prediction job on the model resource.*\n",
+        "- *Clean up the resources created.*"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {

notebooks/official/migration/UJ3 Vertex SDK Custom Image Classification with custom training container.ipynb

@@ -61,7 +61,9 @@
       "source": [
         "## Overview\n",
         "\n",
-        "This notebook demonstrates training a custom image classification model using Tensorflow and Vertex AI SDK by creating a custom training container. Additionally, the notebooks also deploys the trained model to Vertex AI and predictions are generated from it."
+        "This notebook demonstrates training a custom image classification model using Tensorflow and Vertex AI SDK by creating a custom training container. Additionally, the notebooks also deploys the trained model to Vertex AI and predictions are generated from it.\n",
+        "\n",
+        "Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai) and [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training)."
       ]
     },
     {

notebooks/official/migration/UJ4 Vertex SDK AutoML Tabular Binary Classification.ipynb

@@ -62,7 +62,9 @@
         "## Overview\n",
         "\n",
         "\n",
-        "This tutorial demonstrates how to use the Vertex AI SDK to create tabular binary classification models and do online prediction using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) model."
+        "This tutorial demonstrates how to use the Vertex AI SDK to create tabular binary classification models and do online prediction using a Google Cloud [AutoML](https://cloud.google.com/vertex-ai/docs/start/automl-users) model.\n",
+        "\n",
+        "Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai) and [AutoML Tabular](https://cloud.google.com/vertex-ai/docs/start/automl-users#tables)."
       ]
     },
     {

notebooks/official/migration/UJ5 Vertex SDK AutoML Image Object Detection.ipynb

@@ -55,6 +55,45 @@
         "<br/><br/><br/>"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "7a8a13b86a8b"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "This tutorial demonstrates how to use the Vertex AI SDK for Python to train and deploy an AutoML object detection model.\n",
+        "\n",
+        "Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai) and [AutoML Image](https://cloud.google.com/vertex-ai/docs/tutorials/image-recognition-automl/training)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "618cfedf829a"
+      },
+      "source": [
+        "### Objective\n",
+        "\n",
+        "In this tutorial, you learn to use `AutoML` to train an image model and use `Vertex AI Prediction` and `Vertex AI Batch Prediction` to do online and batch predictions.\n",
+        "\n",
+        "\n",
+        "This tutorial uses the following Google Cloud ML services:\n",
+        "\n",
+        "- `AutoML`\n",
+        "- `Vertex AI Batch Prediction`\n",
+        "- `Vertex AI Model` resource\n",
+        "- `Vertex AI Endpoint` resource\n",
+        "\n",
+        "The steps performed include:\n",
+        "\n",
+        "- Train an AutoML object detection model.\n",
+        "- Make a batch prediction.\n",
+        "- Deploy model to a endpoint\n",
+        "- Make a online prediction"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {

notebooks/official/migration/UJ6 Vertex SDK AutoML Text Classification.ipynb

@@ -61,11 +61,11 @@
       "source": [
         "## Overview\n",
         "\n",
-        "<a name=\"section-1\"></a>\n",
-        "\n",
         "This notebook demonstrates how to create an AutoML Video Classification Model, with a Vertex AI video dataset, and how to serve the model for batch prediction. It requires you provide a bucket where the dataset will be stored.\n",
         "\n",
-        "Note: you may incur charges for training, prediction, storage or usage of other GCP products in connection with testing this SDK.\n"
+        "Note: you may incur charges for training, prediction, storage or usage of other GCP products in connection with testing this SDK.\n",
+        "\n",
+        "Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai) and [AutoML Text](https://cloud.google.com/vertex-ai/docs/text-data/classification/prepare-data)."
       ]
     },
     {

notebooks/official/migration/UJ7 Vertex SDK AutoML Text Entity Extraction.ipynb

@@ -61,11 +61,11 @@
       "source": [
         "## Overview\n",
         "\n",
-        "<a name=\"section-1\"></a>\n",
+        "This notebook demonstrates how to create an AutoML Text Entity Extraction model, with a Vertex AI ncbi disease research dataset, and how to serve the model for batch prediction. It requires you provide a bucket where the dataset will be stored.\n",
         "\n",
-        "This notebook demonstrates how to create an AutoML Text Entity Extrasction Model, with a Vertex AI ncbi disease research dataset, and how to serve the model for batch prediction. It requires you provide a bucket where the dataset will be stored.\n",
+        "Note: you may incur charges for training, prediction, storage or usage of other Google Cloud products in connection with testing this SDK.\n",
         "\n",
-        "Note: you may incur charges for training, prediction, storage or usage of other GCP products in connection with testing this SDK."
+        "Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai) and [AutoML Text](https://cloud.google.com/vertex-ai/docs/text-data/entity-extraction/prepare-data)."
       ]
     },
     {
@@ -76,7 +76,7 @@
       "source": [
         "### Objective\n",
         "\n",
-        "The objective of this notebook is to build a AutoML Text Entity Extrasction Model. The following steps have been followed:\n",
+        "The objective of this notebook is to build a AutoML Text Entity Extraction Model. The following steps have been followed:\n",
         "This tutorial uses the following Google Cloud ML services :\n",
         "\n",
         "* Vertex AI Dataset resource\n",

notebooks/official/migration/UJ8 Vertex SDK AutoML Text Sentiment Analysis.ipynb

@@ -3,7 +3,6 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "id": "bdccc50b",
       "metadata": {
         "id": "copyright"
       },
@@ -26,7 +25,6 @@
     },
     {
       "cell_type": "markdown",
-      "id": "c6c22009",
       "metadata": {
         "id": "title:migration,new"
       },
@@ -58,7 +56,47 @@
     },
     {
       "cell_type": "markdown",
-      "id": "b3558cd7",
+      "metadata": {
+        "id": "2277f661a148"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "This notebook demonstrates how to create an AutoML Text Sentiment Analysis model, with a Vertex AI ncbi disease research dataset, and how to serve the model for batch prediction. It requires you provide a bucket where the dataset will be stored.\n",
+        "\n",
+        "Note: you may incur charges for training, prediction, storage or usage of other Google Cloud products in connection with testing this SDK.\n",
+        "\n",
+        "Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai) and [AutoML Text](https://cloud.google.com/vertex-ai/docs/text-data/sentiment-analysis/prepare-data)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "f926ec7acab3"
+      },
+      "source": [
+        "### Objective\n",
+        "\n",
+        "The objective of this notebook is to build a AutoML Text Sentiment Analysis model. The following steps have been followed:\n",
+        "This tutorial uses the following Google Cloud ML services :\n",
+        "\n",
+        "* Vertex AI Dataset resource\n",
+        "* AutoML Training\n",
+        "* Vertex AI Model resource\n",
+        "* Vertex AI Batch Prediction\n",
+        "\n",
+        "The steps performed include the following:\n",
+        "\n",
+        "* Copy AutoML video demo train data for creating managed dataset\n",
+        "* Create a dataset on Vertex AI.\n",
+        "* Configure a training job\n",
+        "* Launch a training job and create a model on Vertex AI\n",
+        "* Copy AutoML Video Demo Prediction Data for creating batch prediction job\n",
+        "* Perform batch prediction job on the model"
+      ]
+    },
+    {
+      "cell_type": "markdown",
       "metadata": {
         "id": "dataset:claritin,tst"
       },
@@ -70,7 +108,6 @@
     },
     {
       "cell_type": "markdown",
-      "id": "9b9362da",
       "metadata": {
         "id": "costs"
       },
@@ -91,7 +128,6 @@
     },
     {
       "cell_type": "markdown",
-      "id": "05425dbe",
       "metadata": {
         "id": "setup_local"
       },
@@ -125,7 +161,6 @@
     },
     {
       "cell_type": "markdown",
-      "id": "070c64e0",
       "metadata": {
         "id": "install_aip:mbsdk"
       },
@@ -138,7 +173,6 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "id": "f6b14b99",
       "metadata": {
         "id": "install_aip:mbsdk"
       },
@@ -157,7 +191,6 @@
     },
     {
       "cell_type": "markdown",
-      "id": "81f60b84",
       "metadata": {
         "id": "restart"
       },
@@ -170,7 +203,6 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "id": "a95627f0",
       "metadata": {
         "id": "restart"
       },
@@ -188,7 +220,6 @@
     },
     {
       "cell_type": "markdown",
-      "id": "b7f6b038",
       "metadata": {
         "id": "before_you_begin:nogpu"
       },
@@ -220,7 +251,6 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "id": "f55cca7c",
       "metadata": {
         "id": "set_project_id"
       },
@@ -232,7 +262,6 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "id": "6917314c",
       "metadata": {
         "id": "autoset_project_id"
       },
@@ -248,7 +277,6 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "id": "53236aa8",
       "metadata": {
         "id": "set_gcloud_project_id"
       },
@@ -259,7 +287,6 @@
     },
     {
       "cell_type": "markdown",
-      "id": "c009cc18",
       "metadata": {
         "id": "region"
       },
@@ -281,7 +308,6 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "id": "071a11c0",
       "metadata": {
         "id": "region"
       },
@@ -295,7 +321,6 @@
     },
     {
       "cell_type": "markdown",
-      "id": "ae48374d",
       "metadata": {
         "id": "timestamp"
       },
@@ -308,7 +333,6 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "id": "41ba0990",
       "metadata": {
         "id": "timestamp"
       },
@@ -328,7 +352,6 @@
     },
     {
       "cell_type": "markdown",
-      "id": "2128e871",
       "metadata": {
         "id": "gcp_authenticate"
       },
@@ -357,7 +380,6 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "id": "433e860c",
       "metadata": {
         "id": "gcp_authenticate"
       },
@@ -390,7 +412,6 @@
     },
     {
       "cell_type": "markdown",
-      "id": "b57cb5f6",
       "metadata": {
         "id": "bucket:mbsdk"
       },
@@ -407,7 +428,6 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "id": "61b082b1",
       "metadata": {
         "id": "bucket"
       },
@@ -420,7 +440,6 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "id": "ff81b3cc",
       "metadata": {
         "id": "autoset_bucket"
       },
@@ -433,7 +452,6 @@
     },
     {
       "cell_type": "markdown",
-      "id": "f8c009cd",
       "metadata": {
         "id": "create_bucket"
       },
@@ -444,7 +462,6 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "id": "2f881cb5",
       "metadata": {
         "id": "create_bucket"
       },
@@ -455,7 +472,6 @@
     },
     {
       "cell_type": "markdown",
-      "id": "d746d0f0",
       "metadata": {
         "id": "validate_bucket"
       },
@@ -466,7 +482,6 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "id": "8c435668",
       "metadata": {
         "id": "validate_bucket"
       },
@@ -477,7 +492,6 @@
     },
     {
       "cell_type": "markdown",
-      "id": "f578b01b",
       "metadata": {
         "id": "setup_vars"
       },
@@ -491,7 +505,6 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "id": "f41ecf1e",
       "metadata": {
         "id": "import_aip:mbsdk"
       },
@@ -502,7 +515,6 @@
     },
     {
       "cell_type": "markdown",
-      "id": "292245fd",
       "metadata": {
         "id": "init_aip:mbsdk"
       },
@@ -515,7 +527,6 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "id": "56dc88d8",
       "metadata": {
         "id": "init_aip:mbsdk"
       },
@@ -526,7 +537,6 @@
     },
     {
       "cell_type": "markdown",
-      "id": "87e20f86",
       "metadata": {
         "id": "import_file:u_dataset,csv"
       },
@@ -539,7 +549,6 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "id": "fffdec5d",
       "metadata": {
         "id": "import_file:claritin,csv,tst"
       },
@@ -551,7 +560,6 @@
     },
     {
       "cell_type": "markdown",
-      "id": "9c8d950b",
       "metadata": {
         "id": "quick_peek:csv"
       },
@@ -566,7 +574,6 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "id": "da6d0980",
       "metadata": {
         "id": "quick_peek:csv"
       },
@@ -586,7 +593,6 @@
     },
     {
       "cell_type": "markdown",
-      "id": "00e9f81e",
       "metadata": {
         "id": "create_a_dataset:migration"
       },
@@ -596,7 +602,6 @@
     },
     {
       "cell_type": "markdown",
-      "id": "14b72768",
       "metadata": {
         "id": "datasets_create:migration,new,mbsdk"
       },
@@ -606,7 +611,6 @@
     },
     {
       "cell_type": "markdown",
-      "id": "00d777bd",
       "metadata": {
         "id": "create_dataset:text,tst"
       },
@@ -625,7 +629,6 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "id": "ca8a6f66",
       "metadata": {
         "id": "create_dataset:text,tst"
       },
@@ -642,7 +645,6 @@
     },
     {
       "cell_type": "markdown",
-      "id": "068df169",
       "metadata": {
         "id": "create_dataset:text,tst"
       },
@@ -662,7 +664,6 @@
     },
     {
       "cell_type": "markdown",
-      "id": "fb50a4ce",
       "metadata": {
         "id": "train_a_model:migration"
       },
@@ -672,7 +673,6 @@
     },
     {
       "cell_type": "markdown",
-      "id": "293160ba",
       "metadata": {
         "id": "trainingpipelines_create:migration,new,mbsdk"
       },
@@ -682,7 +682,6 @@
     },
     {
       "cell_type": "markdown",
-      "id": "84801634",
       "metadata": {
         "id": "create_automl_pipeline:text,tst"
       },
@@ -709,7 +708,6 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "id": "69eaae0e",
       "metadata": {
         "id": "create_automl_pipeline:text,tst"
       },
@@ -726,7 +724,6 @@
     },
     {
       "cell_type": "markdown",
-      "id": "da9ecb4e",
       "metadata": {
         "id": "create_automl_pipeline:text,tst"
       },
@@ -738,7 +735,6 @@
     },
     {
       "cell_type": "markdown",
-      "id": "55f19997",
       "metadata": {
         "id": "run_automl_pipeline:text"
       },
@@ -761,7 +757,6 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "id": "6149074c",
       "metadata": {
         "id": "run_automl_pipeline:text"
       },
@@ -778,7 +773,6 @@
     },
     {
       "cell_type": "markdown",
-      "id": "6e8fe148",
       "metadata": {
         "id": "run_automl_pipeline:text"
       },
@@ -804,7 +798,6 @@
     },
     {
       "cell_type": "markdown",
-      "id": "c25dee28",
       "metadata": {
         "id": "evaluate_the_model:migration"
       },
@@ -814,7 +807,6 @@
     },
     {
       "cell_type": "markdown",
-      "id": "903e8226",
       "metadata": {
         "id": "models_evaluations_list:migration,new"
       },
@@ -824,7 +816,6 @@
     },
     {
       "cell_type": "markdown",
-      "id": "cb2d95f3",
       "metadata": {
         "id": "evaluate_the_model:mbsdk"
       },
@@ -838,7 +829,6 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "id": "9b1ec312",
       "metadata": {
         "id": "evaluate_the_model:mbsdk"
       },
@@ -860,7 +850,6 @@
     },
     {
       "cell_type": "markdown",
-      "id": "9eab460e",
       "metadata": {
         "id": "evaluate_the_model:mbsdk"
       },
@@ -901,7 +890,6 @@
     },
     {
       "cell_type": "markdown",
-      "id": "d4111c50",
       "metadata": {
         "id": "make_batch_predictions:migration"
       },
@@ -911,7 +899,6 @@
     },
     {
       "cell_type": "markdown",
-      "id": "f73fad68",
       "metadata": {
         "id": "batchpredictionjobs_create:migration,new,mbsdk"
       },
@@ -921,7 +908,6 @@
     },
     {
       "cell_type": "markdown",
-      "id": "ba77f1c7",
       "metadata": {
         "id": "get_test_items:batch_prediction"
       },
@@ -934,7 +920,6 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "id": "1c9fc91f",
       "metadata": {
         "id": "get_test_items:automl,tst,csv"
       },
@@ -956,7 +941,6 @@
     },
     {
       "cell_type": "markdown",
-      "id": "2a18c8e2",
       "metadata": {
         "id": "make_batch_file:automl,text"
       },
@@ -976,7 +960,6 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "id": "70461c41",
       "metadata": {
         "id": "make_batch_file:automl,text"
       },
@@ -1006,7 +989,6 @@
     },
     {
       "cell_type": "markdown",
-      "id": "254cbdbb",
       "metadata": {
         "id": "batch_request:mbsdk"
       },
@@ -1024,7 +1006,6 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "id": "8f3cf8b6",
       "metadata": {
         "id": "batch_request:mbsdk"
       },
@@ -1042,7 +1023,6 @@
     },
     {
       "cell_type": "markdown",
-      "id": "530dbf5b",
       "metadata": {
         "id": "batch_request:mbsdk"
       },
@@ -1062,7 +1042,6 @@
     },
     {
       "cell_type": "markdown",
-      "id": "89414481",
       "metadata": {
         "id": "batch_request_wait:mbsdk"
       },
@@ -1075,7 +1054,6 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "id": "a579bd4a",
       "metadata": {
         "id": "batch_request_wait:mbsdk"
       },
@@ -1086,7 +1064,6 @@
     },
     {
       "cell_type": "markdown",
-      "id": "2cba4cc6",
       "metadata": {
         "id": "batch_request_wait:mbsdk"
       },
@@ -1121,7 +1098,6 @@
     },
     {
       "cell_type": "markdown",
-      "id": "c46e3e76",
       "metadata": {
         "id": "get_batch_prediction:mbsdk,tst"
       },
@@ -1140,7 +1116,6 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "id": "d2af5ea8",
       "metadata": {
         "id": "get_batch_prediction:mbsdk,tst"
       },
@@ -1169,7 +1144,6 @@
     },
     {
       "cell_type": "markdown",
-      "id": "9fc83253",
       "metadata": {
         "id": "get_batch_prediction:mbsdk,tst"
       },
@@ -1181,7 +1155,6 @@
     },
     {
       "cell_type": "markdown",
-      "id": "19466786",
       "metadata": {
         "id": "make_online_predictions:migration"
       },
@@ -1191,7 +1164,6 @@
     },
     {
       "cell_type": "markdown",
-      "id": "e97f1e55",
       "metadata": {
         "id": "deploy_model:migration,new,mbsdk"
       },
@@ -1201,7 +1173,6 @@
     },
     {
       "cell_type": "markdown",
-      "id": "d2745f77",
       "metadata": {
         "id": "deploy_model:mbsdk,automatic"
       },
@@ -1214,7 +1185,6 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "id": "6d30aa15",
       "metadata": {
         "id": "deploy_model:mbsdk,automatic"
       },
@@ -1225,7 +1195,6 @@
     },
     {
       "cell_type": "markdown",
-      "id": "c2c876d0",
       "metadata": {
         "id": "deploy_model:mbsdk,automatic"
       },
@@ -1244,7 +1213,6 @@
     },
     {
       "cell_type": "markdown",
-      "id": "9bb982a8",
       "metadata": {
         "id": "endpoints_predict:migration,new,mbsdk"
       },
@@ -1254,7 +1222,6 @@
     },
     {
       "cell_type": "markdown",
-      "id": "246945bb",
       "metadata": {
         "id": "get_test_item"
       },
@@ -1267,7 +1234,6 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "id": "e21c3f76",
       "metadata": {
         "id": "get_test_item:automl,tst,csv"
       },
@@ -1284,7 +1250,6 @@
     },
     {
       "cell_type": "markdown",
-      "id": "95ffe1ea",
       "metadata": {
         "id": "predict_request:mbsdk,tst"
       },
@@ -1313,7 +1278,6 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "id": "16b7ab95",
       "metadata": {
         "id": "predict_request:mbsdk,tst"
       },
@@ -1327,7 +1291,6 @@
     },
     {
       "cell_type": "markdown",
-      "id": "f4c79c7f",
       "metadata": {
         "id": "predict_request:mbsdk,tst"
       },
@@ -1339,7 +1302,6 @@
     },
     {
       "cell_type": "markdown",
-      "id": "52717fb9",
       "metadata": {
         "id": "undeploy_model:mbsdk"
       },
@@ -1352,7 +1314,6 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "id": "7c164d13",
       "metadata": {
         "id": "undeploy_model:mbsdk"
       },
@@ -1363,7 +1324,6 @@
     },
     {
       "cell_type": "markdown",
-      "id": "4b844c87",
       "metadata": {
         "id": "cleanup:mbsdk"
       },
@@ -1389,7 +1349,6 @@
     {
       "cell_type": "code",
       "execution_count": null,
-      "id": "2ea906d0",
       "metadata": {
         "id": "cleanup:mbsdk"
       },

notebooks/official/migration/UJ9 Vertex SDK Custom XGBoost with pre-built training container.ipynb

@@ -53,6 +53,49 @@
         "<br/><br/><br/>"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "7a8a13b86a8b"
+      },
+      "source": [
+        "## Overview\n",
+        "\n",
+        "\n",
+        "This tutorial demonstrates how to use the Vertex AI SDK for Python to train and deploy a custom tabular classification XGBoost model for batch prediction.\n",
+        "\n",
+        "Learn more about [Migrate to Vertex AI](https://cloud.google.com/vertex-ai/docs/start/migrating-to-vertex-ai) and [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training)."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "618cfedf829a"
+      },
+      "source": [
+        "### Objective\n",
+        "\n",
+        "In this tutorial, you learn to use `Vertex AI Training` to create a custom trained model and use `Vertex AI Batch Prediction` to do a batch prediction on the trained model.\n",
+        "\n",
+        "\n",
+        "You learn how to create a custom-trained model from a Python script in a Docker container using the Vertex AI SDK for Python, and then do a prediction on the deployed model by sending data.\n",
+        "\n",
+        "This tutorial uses the following Google Cloud ML services:\n",
+        "\n",
+        "- `Vertex AI Training`\n",
+        "- `Vertex AI Batch Prediction`\n",
+        "- `Vertex AI Model` resource\n",
+        "- `Vertex AI Endpoint` resource\n",
+        "\n",
+        "The steps performed include:\n",
+        "\n",
+        "- Create a `Vertex AI` custom job for training a scikit-learn model.\n",
+        "- Upload the trained model artifacts as a `Model` resource.\n",
+        "- Make a batch prediction.\n",
+        "- Deploy model to a endpoint\n",
+        "- Make a online prediction"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {

notebooks/official/ml_metadata/README.md

@@ -1,3 +1,20 @@
+
+[Track parameters and metrics for custom training jobs](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-custom-jobs.ipynb)
+
+```
+Learn how to use Vertex AI SDK for Python to:
+
+The steps performed include:
+- Track training parameters and prediction metrics for a custom training job.
+- Extract and perform analysis for all parameters and metrics within an Experiment.
+
+```
+
+   Learn more about [Vertex ML Metadata](https://cloud.google.com/vertex-ai/docs/ml-metadata).
+
+   Learn more about [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training).
+
+
 [Track parameters and metrics for locally trained models](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-locally-trained-models.ipynb)
 
 ```
@@ -10,17 +27,7 @@ The steps performed include:
 
 ```
 
-
-[Track parameters and metrics for custom training jobs](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-custom-jobs.ipynb)
-
-```
-Learn how to use Vertex AI SDK for Python to:
-
-The steps performed include:
-- Track training parameters and prediction metrics for a custom training job.
-- Extract and perform analysis for all parameters and metrics within an Experiment.
-
-```
+   Learn more about [Vertex ML Metadata](https://cloud.google.com/vertex-ai/docs/ml-metadata).
 
 
 [Track artifacts and metrics across Vertex AI Pipelines runs using Vertex ML Metadata](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/ml_metadata/vertex-pipelines-ml-metadata.ipynb)
@@ -39,3 +46,7 @@ The steps performed include:
 
 ```
 
+   Learn more about [Vertex ML Metadata](https://cloud.google.com/vertex-ai/docs/ml-metadata).
+
+   Learn more about [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction).
+

notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-custom-jobs.ipynb

@@ -61,7 +61,9 @@
       "source": [
         "## Overview\n",
         "\n",
-        "This notebook demonstrates how to track metrics and parameters for Vertex AI custom training jobs, and how to perform detailed analysis using this data."
+        "This notebook demonstrates how to track metrics and parameters for Vertex AI custom training jobs, and how to perform detailed analysis using this data.\n",
+        "\n",
+        "Learn more about [Vertex ML Metadata](https://cloud.google.com/vertex-ai/docs/ml-metadata) and [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training)."
       ]
     },
     {

notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-locally-trained-models.ipynb

@@ -61,7 +61,9 @@
       "source": [
         "## Overview\n",
         "\n",
-        "This notebook demonstrates how to track metrics and parameters for ML training jobs and analyze this metadata using Vertex AI SDK for Python."
+        "This notebook demonstrates how to track metrics and parameters for ML training jobs and analyze this metadata using Vertex AI SDK for Python.\n",
+        "\n",
+        "Learn more about [Vertex ML Metadata](https://cloud.google.com/vertex-ai/docs/ml-metadata)"
       ]
     },
     {

notebooks/official/ml_metadata/vertex-pipelines-ml-metadata.ipynb

@@ -61,7 +61,9 @@
       "source": [
         "## Overview\n",
         "\n",
-        "This notebook demonstrates how to track metrics and artifacts across Vertex AI Pipelines runs, and analyze this metadata using the Vertex AI SDK. If you'd prefer to follow a step-by-step tutorial, check out the [codelab version](https://codelabs.developers.google.com/vertex-mlmd-pipelines#0) of this notebook."
+        "This notebook demonstrates how to track metrics and artifacts across Vertex AI Pipelines runs, and analyze this metadata using the Vertex AI SDK. If you'd prefer to follow a step-by-step tutorial, check out the [codelab version](https://codelabs.developers.google.com/vertex-mlmd-pipelines#0) of this notebook.\n",
+        "\n",
+        "Learn more about [Vertex ML Metadata](https://cloud.google.com/vertex-ai/docs/ml-metadata) and [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction)."
       ]
     },
     {

notebooks/official/model_evaluation/README.md

@@ -1,3 +1,4 @@
+
 [Evaluating batch prediction results from an AutoML Tabular classification model](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model_evaluation/automl_tabular_classification_model_evaluation.ipynb)
 
 ```
@@ -14,22 +15,9 @@ The steps performed include:
 
 ```
 
+   Learn more about [Vertex AI Model Evaluation](https://cloud.google.com/vertex-ai/docs/evaluation/introduction).
 
-[Evaluating batch prediction results from AutoML Video classification model](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model_evaluation/automl_video_classification_model_evaluation.ipynb)
-
-```
-Learn how to train a Vertex AI AutoML Video classification model and learn how to evaluate it through a Vertex AI pipeline job using `google_cloud_pipeline_components`:
-
-The steps performed include:
-
-- Create a `Vertex AI Dataset`.
-- Train a Automl Video Classification model on the `Vertex AI Dataset` resource.
-- Import the trained `AutoML Vertex AI Model resource` into the pipeline.
-- Run a batch prediction job inside the pipeline.
-- Evaulate the AutoML model using the classification evaluation component.
-- Import the classification metrics to the AutoML Vertex AI Model resource.
-
-```
+   Learn more about [AutoML Tabular](https://cloud.google.com/vertex-ai/docs/start/automl-users#tables).
 
 
 [Evaluating batch prediction results from AutoML Tabular regression model](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model_evaluation/automl_tabular_regression_model_evaluation.ipynb)
@@ -49,25 +37,9 @@ The steps performed include:
 
 ```
 
+   Learn more about [Vertex AI Model Evaluation](https://cloud.google.com/vertex-ai/docs/evaluation/introduction).
 
-[Evaluating batch prediction results from custom tabular regression model](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model_evaluation/custom_tabular_regression_model_evaluation.ipynb)
-
-```
-Learn how to evaluate a Vertex AI model resource through a Vertex AI pipeline job using `google_cloud_pipeline_components`:
-
-The steps performed include:
-
-- Create a Vertex AI `CustomTrainingJob` for training a model.
-- Run the `CustomTrainingJob` 
-- Retrieve and load the model artifacts.
-- View the model evaluation.
-- Upload the model as a Vertex AI Model resource.
-- Import a pre-trained `Vertex AI model resource` into the pipeline.
-- Run a `batch prediction` job in the pipeline.
-- Evaulate the model using the `regression evaluation component`.
-- Import the Regression Metrics to the Vertex AI model resource.
-
-```
+   Learn more about [AutoML Tabular](https://cloud.google.com/vertex-ai/docs/start/automl-users#tables).
 
 
 [AutoML text classification pipelines using google-cloud-pipeline-components](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model_evaluation/automl_text_classification_model_evaluation.ipynb)
@@ -86,6 +58,31 @@ The steps performed include:
 
 ```
 
+   Learn more about [Vertex AI Model Evaluation](https://cloud.google.com/vertex-ai/docs/evaluation/introduction).
+
+   Learn more about [AutoML Text](https://cloud.google.com/vertex-ai/docs/text-data/classification/prepare-data).
+
+
+[Evaluating batch prediction results from AutoML Video classification model](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model_evaluation/automl_video_classification_model_evaluation.ipynb)
+
+```
+Learn how to train a Vertex AI AutoML Video classification model and learn how to evaluate it through a Vertex AI pipeline job using `google_cloud_pipeline_components`:
+
+The steps performed include:
+
+- Create a `Vertex AI Dataset`.
+- Train a Automl Video Classification model on the `Vertex AI Dataset` resource.
+- Import the trained `AutoML Vertex AI Model resource` into the pipeline.
+- Run a batch prediction job inside the pipeline.
+- Evaulate the AutoML model using the classification evaluation component.
+- Import the classification metrics to the AutoML Vertex AI Model resource.
+
+```
+
+   Learn more about [Vertex AI Model Evaluation](https://cloud.google.com/vertex-ai/docs/evaluation/introduction).
+
+   Learn more about [AutoML Video](https://cloud.google.com/vertex-ai/docs/video-data/classification/prepare-data).
+
 
 [Evaluating BatchPrediction results from a Custom Tabular classification model](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model_evaluation/custom_tabular_classification_model_evaluation.ipynb)
 
@@ -109,3 +106,31 @@ The steps performed include:
 
 ```
 
+   Learn more about [Vertex AI Model Evaluation](https://cloud.google.com/vertex-ai/docs/evaluation/introduction).
+
+   Learn more about [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training).
+
+
+[Evaluating batch prediction results from custom tabular regression model](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model_evaluation/custom_tabular_regression_model_evaluation.ipynb)
+
+```
+Learn how to evaluate a Vertex AI model resource through a Vertex AI pipeline job using `google_cloud_pipeline_components`:
+
+The steps performed include:
+
+- Create a Vertex AI `CustomTrainingJob` for training a model.
+- Run the `CustomTrainingJob` 
+- Retrieve and load the model artifacts.
+- View the model evaluation.
+- Upload the model as a Vertex AI Model resource.
+- Import a pre-trained `Vertex AI model resource` into the pipeline.
+- Run a `batch prediction` job in the pipeline.
+- Evaulate the model using the `regression evaluation component`.
+- Import the Regression Metrics to the Vertex AI model resource.
+
+```
+
+   Learn more about [Vertex AI Model Evaluation](https://cloud.google.com/vertex-ai/docs/evaluation/introduction).
+
+   Learn more about [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training).
+

notebooks/official/model_evaluation/automl_tabular_classification_model_evaluation.ipynb

@@ -61,7 +61,9 @@
       "source": [
         "## Overview\n",
         "\n",
-        "This notebook demonstrates how to use the Vertex AI classification model evaluation component to evaluate an AutoML Tabular classification model. Model evaluation helps determine your model's performance based on the evaluation metrics and improve the model whenever necessary. "
+        "This notebook demonstrates how to use the Vertex AI classification model evaluation component to evaluate an AutoML Tabular classification model. Model evaluation helps determine your model's performance based on the evaluation metrics and improve the model whenever necessary. \n",
+        "\n",
+        "Learn more about [Vertex AI Model Evaluation](https://cloud.google.com/vertex-ai/docs/evaluation/introduction) and [AutoML Tabular](https://cloud.google.com/vertex-ai/docs/start/automl-users#tables)."
       ]
     },
     {

notebooks/official/model_evaluation/automl_tabular_regression_model_evaluation.ipynb

@@ -61,7 +61,9 @@
       "source": [
         "## Overview\n",
         "\n",
-        "This notebook demonstrates how to use Vertex AI regression model evaluation component to evaluate an AutoML Tabular regression model. Model evaluation helps you determine your model performance based on the evaluation metrics and improve the model if necessary. "
+        "This notebook demonstrates how to use Vertex AI regression model evaluation component to evaluate an AutoML Tabular regression model. Model evaluation helps you determine your model performance based on the evaluation metrics and improve the model if necessary. \n",
+        "\n",
+        "Learn more about [Vertex AI Model Evaluation](https://cloud.google.com/vertex-ai/docs/evaluation/introduction) and [AutoML Tabular](https://cloud.google.com/vertex-ai/docs/start/automl-users#tables)."
       ]
     },
     {

notebooks/official/model_evaluation/automl_text_classification_model_evaluation.ipynb

@@ -61,7 +61,9 @@
       "source": [
         "## Overview\n",
         "\n",
-        "This notebook demonstrates how to use the Vertex AI classification model evaluation component to evaluate an AutoML text classification model. Model evaluation helps you determine your model performance based on the evaluation metrics and improve the model if necessary. "
+        "This notebook demonstrates how to use the Vertex AI classification model evaluation component to evaluate an AutoML text classification model. Model evaluation helps you determine your model performance based on the evaluation metrics and improve the model if necessary. \n",
+        "\n",
+        "Learn more about [Vertex AI Model Evaluation](https://cloud.google.com/vertex-ai/docs/evaluation/introduction) and [AutoML Text](https://cloud.google.com/vertex-ai/docs/text-data/classification/prepare-data)."
       ]
     },
     {

notebooks/official/model_evaluation/automl_video_classification_model_evaluation.ipynb

@@ -61,7 +61,9 @@
       "source": [
         "## Overview\n",
         "\n",
-        "This notebook demonstrates how to use the Vertex AI classification model evaluation component to evaluate an AutoML video classification model. Model evaluation helps you determine your model performance based on the evaluation metrics and improve the model if necessary. "
+        "This notebook demonstrates how to use the Vertex AI classification model evaluation component to evaluate an AutoML video classification model. Model evaluation helps you determine your model performance based on the evaluation metrics and improve the model if necessary. \n",
+        "\n",
+        "Learn more about [Vertex AI Model Evaluation](https://cloud.google.com/vertex-ai/docs/evaluation/introduction) and [AutoML Video](https://cloud.google.com/vertex-ai/docs/video-data/classification/prepare-data)."
       ]
     },
     {
@@ -202,7 +204,7 @@
         "\n",
         "! pip3 install --upgrade google-cloud-aiplatform \\\n",
         "                            kfp \\\n",
-        "                            google-cloud-pipeline-components \\\n",
+        "                            google-cloud-pipeline-components==1.0.26 \\\n",
         "                            google-cloud-storage {USER_FLAG} -q"
       ]
     },
@@ -752,9 +754,9 @@
         "\n",
         "- `display_name`: The human readable name for the `TrainingJob` resource.\n",
         "- `prediction_type`: The type task to train the model for.\n",
-        "- `classification`: A video classification model.\n",
-        "- `object_tracking`: A video object tracking model.\n",
-        "- `action_recognition`: A video action recognition model.\n"
+        "  - `classification`: A video classification model.\n",
+        "  - `object_tracking`: A video object tracking model.\n",
+        "  - `action_recognition`: A video action recognition model.\n"
       ]
     },
     {
@@ -1013,12 +1015,13 @@
         "\n",
         "- `GetVertexModelOp`: Gets a Vertex AI Model Artifact. \n",
         "- `EvaluationDataSamplerOp`: Randomly downsamples an input dataset to a specified size for generating predictions from AutoML and custom models. Creates a Dataflow job with Apache Beam to downsample the dataset. \n",
+        "- `TargetFieldDataRemoverOp`: Removes the target field from the input dataset.\n",
         "- `ModelBatchPredictOp`: Creates a Google Cloud Vertex BatchPredictionJob and waits for it to complete. \n",
         "- `ModelEvaluationClassificationOp`: Compute evaluation metrics on a trained model’s batch prediction results. Creates a Dataflow job with Apache Beam and TFMA to compute evaluation metrics. Supports mutliclass classification evaluation for image, video, and text data. \n",
         "\n",
         "- `ModelImportEvaluationOp`: Imports a model evaluation artifact to an existing Vertex model with ModelService.ImportModelEvaluation. \n",
         "\n",
-        "Learn more about [Google Cloud Pipeline Model Evaluation components](https://google-cloud-pipeline-components.readthedocs.io/en/google-cloud-pipeline-components-1.0.20/google_cloud_pipeline_components.experimental.evaluation.html)."
+        "Learn more about [Google Cloud Pipeline Model Evaluation components](https://google-cloud-pipeline-components.readthedocs.io/en/google-cloud-pipeline-components-1.0.26/google_cloud_pipeline_components.experimental.evaluation.html)."
       ]
     },
     {
@@ -1036,9 +1039,9 @@
         "    root_dir: str,\n",
         "    prediction_type: str,\n",
         "    model_name: str,\n",
-        "    target_column_name: str,\n",
+        "    target_field_name: str,\n",
         "    ground_truth_gcs_uri: list,\n",
-        "    class_labels: list = \"{}\",\n",
+        "    class_labels: list,\n",
         "    batch_predict_instances_format: str = \"jsonl\",\n",
         "    batch_predict_predictions_format: str = \"jsonl\",\n",
         "    batch_predict_machine_type: str = \"n1-standard-16\",\n",
@@ -1070,7 +1073,7 @@
         "        root_dir=root_dir,\n",
         "        gcs_source_uris=data_sampler_task.outputs[\"gcs_output_directory\"],\n",
         "        instances_format=batch_predict_instances_format,\n",
-        "        target_field_name=target_column_name,\n",
+        "        target_field_name=target_field_name,\n",
         "    )\n",
         "\n",
         "    # Run Batch Prediction.\n",
@@ -1095,10 +1098,10 @@
         "        location=location,\n",
         "        root_dir=root_dir,\n",
         "        ground_truth_gcs_source=data_sampler_task.outputs[\"gcs_output_directory\"],\n",
-        "        target_field_name=target_column_name,\n",
+        "        target_field_name=target_field_name,\n",
         "        prediction_score_column=\"prediction.confidence\",\n",
         "        prediction_label_column=\"prediction.displayName\",\n",
-        "        class_labels=[\"brush_hair\", \"cartwheel\"],\n",
+        "        class_labels=class_labels,\n",
         "        ground_truth_format=batch_predict_instances_format,\n",
         "        predictions_format=batch_predict_predictions_format,\n",
         "        predictions_gcs_source=batch_predict_task.outputs[\"gcs_output_directory\"],\n",
@@ -1152,7 +1155,7 @@
         "- `location`: Region where the pipeline is run.\n",
         "- `root_dir`: The GCS directory for keeping staging files and artifacts. A random subdirectory is created under the directory to keep job info for resuming the job in case of failure.\n",
         "- `model_name`: Resource name of the trained AutoML Video Classification model.\n",
-        "- `target_column_name`: Name of the column to be used as the target for classification.\n",
+        "- `target_field_name`: Name of the column to be used as the target for classification.\n",
         "- `batch_predict_instances_format`: Format of the input instances for batch prediction. Can be '**jsonl**' or '**bigquery**' or '**csv**'.\n",
         "- `batch_predict_sample_size`: Size of the samples to be considered for batch prediction and evaluation."
       ]
@@ -1168,14 +1171,16 @@
         "LABEL_COLUMN = \"outputLabel\"\n",
         "PIPELINE_ROOT = f\"{BUCKET_URI}/pipeline_root/pen{UUID}\"\n",
         "SAMPLE_SIZE = 2\n",
+        "CLASS_LABELS = [\"brush_hair\", \"cartwheel\"]\n",
         "parameters = {\n",
         "    \"project\": PROJECT_ID,\n",
         "    \"location\": REGION,\n",
         "    \"root_dir\": PIPELINE_ROOT,\n",
         "    \"prediction_type\": \"segment-classification\",\n",
         "    \"model_name\": MODEL_RSC_NAME,\n",
-        "    \"target_column_name\": LABEL_COLUMN,\n",
+        "    \"target_field_name\": LABEL_COLUMN,\n",
         "    \"ground_truth_gcs_uri\": [gcs_ground_truth_uri],\n",
+        "    \"class_labels\": CLASS_LABELS,\n",
         "    \"batch_predict_instances_format\": \"jsonl\",\n",
         "    \"batch_predict_sample_size\": SAMPLE_SIZE,\n",
         "}"

notebooks/official/model_evaluation/custom_tabular_classification_model_evaluation.ipynb

@@ -61,7 +61,9 @@
       "source": [
         "## Overview\n",
         "\n",
-        "This notebook demonstrates how to use the Vertex AI classification model evaluation component to evaluate a custom-trained tabular classification model saved in Vertex AI Model Registry. Model evaluation helps you determine your model performance based on the evaluation metrics and improve the model if necessary. "
+        "This notebook demonstrates how to use the Vertex AI classification model evaluation component to evaluate a custom-trained tabular classification model saved in Vertex AI Model Registry. Model evaluation helps you determine your model performance based on the evaluation metrics and improve the model if necessary. \n",
+        "\n",
+        "Learn more about [Vertex AI Model Evaluation](https://cloud.google.com/vertex-ai/docs/evaluation/introduction) and [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training)."
       ]
     },
     {
@@ -150,7 +152,7 @@
       "source": [
         "# Install the latest versions of the following packages\n",
         "! pip3 install --upgrade google-cloud-aiplatform \\\n",
-        "                            google-cloud-pipeline-components \\\n",
+        "                            google-cloud-pipeline-components==1.0.26 \\\n",
         "                            matplotlib \\\n",
         "                            pyarrow -q\n",
         "# Install the specified versions of the following packages\n",
@@ -666,7 +668,7 @@
       "source": [
         "# Create a bigquery dataset\n",
         "bq_dataset = bigquery.Dataset(f\"{PROJECT_ID}.{PREDICTION_INPUT_DATASET_ID}\")\n",
-        "bq_dataset = bq_client.create_dataset(bq_dataset)\n",
+        "bq_dataset = bq_client.create_dataset(bq_dataset, exists_ok=True)\n",
         "print(f\"Created dataset {bq_client.project}.{bq_dataset.dataset_id}\")"
       ]
     },
@@ -1163,7 +1165,7 @@
         "    location: str,\n",
         "    root_dir: str,\n",
         "    model_name: str,\n",
-        "    target_column_name: str,\n",
+        "    target_field_name: str,\n",
         "    bigquery_source_input_uri: str,\n",
         "    bigquery_destination_output_uri: str,\n",
         "    batch_predict_instances_format: str,\n",
@@ -1203,7 +1205,7 @@
         "        root_dir=root_dir,\n",
         "        bigquery_source_uri=data_sampler_task.outputs[\"bigquery_output_table\"],\n",
         "        instances_format=batch_predict_instances_format,\n",
-        "        target_field_name=target_column_name,\n",
+        "        target_field_name=target_field_name,\n",
         "    )\n",
         "\n",
         "    # Run the batch prediction task\n",
@@ -1229,7 +1231,7 @@
         "        class_labels=evaluation_class_names,\n",
         "        prediction_label_column=evaluation_prediction_label_column,\n",
         "        prediction_score_column=evaluation_prediction_score_column,\n",
-        "        target_field_name=target_column_name,\n",
+        "        target_field_name=target_field_name,\n",
         "        ground_truth_format=batch_predict_instances_format,\n",
         "        ground_truth_bigquery_source=data_sampler_task.outputs[\"bigquery_output_table\"],\n",
         "        predictions_format=batch_predict_predictions_format,\n",
@@ -1283,7 +1285,7 @@
         "- `location`: Region where the pipeline needs to be run. If not set, the pipeline defaults to the region that Vertex AI SDK is configured with.\n",
         "- `root_dir`: The Cloud Storage directory for keeping the staged files and artifacts. A random subdirectory is created under the directory to keep the job information for resuming the job in case of a failure.\n",
         "- `model_name`: Resource name of the trained custom tabular classification model.\n",
-        "- `target_column_name`: Name of the column to be used as the ground truth for evaluation.\n",
+        "- `target_field_name`: Name of the column to be used as the ground truth for evaluation.\n",
         "- `bigquery_source_input_uri`: BigQuery table URI where the test input is stored.\n",
         "- `bigquery_destination_output_uri`: BigQuery dataset URI for exporting predictions on the test set.\n",
         "- `batch_predict_instances_format`: Format of the input for batch prediction and evaluation.\n",
@@ -1305,7 +1307,7 @@
         "    \"location\": REGION,\n",
         "    \"root_dir\": PIPELINE_ROOT,\n",
         "    \"model_name\": aip_model.resource_name,\n",
-        "    \"target_column_name\": TARGET,\n",
+        "    \"target_field_name\": TARGET,\n",
         "    \"bigquery_source_input_uri\": f\"bq://{PROJECT_ID}.{table_ref.dataset_id}.{table_ref.table_id}\",\n",
         "    \"bigquery_destination_output_uri\": f\"bq://{PROJECT_ID}.{table_ref.dataset_id}\",\n",
         "    \"batch_predict_instances_format\": \"bigquery\",\n",

notebooks/official/model_evaluation/custom_tabular_regression_model_evaluation.ipynb

@@ -61,7 +61,9 @@
       "source": [
         "## Overview\n",
         "\n",
-        "This notebook demonstrates how to use the Vertex AI regression model evaluation component to evaluate a custom regression model. Model evaluation helps you determine your model performance based on the evaluation metrics and improve the model if necessary. "
+        "This notebook demonstrates how to use the Vertex AI regression model evaluation component to evaluate a custom regression model. Model evaluation helps you determine your model performance based on the evaluation metrics and improve the model if necessary. \n",
+        "\n",
+        "Learn more about [Vertex AI Model Evaluation](https://cloud.google.com/vertex-ai/docs/evaluation/introduction) and [Vertex AI Training](https://cloud.google.com/vertex-ai/docs/training/custom-training)."
       ]
     },
     {
@@ -1518,7 +1520,7 @@
         "\n",
         "- `serving_input`: The name of the input layer of the underlying model.\n",
         "- `content`: The feature values of the test item as a list.\n",
-        "- `ground_truth_column`: Give any name to this key. Use the same name in target_column_name in the below pipeline parameters.\n",
+        "- `ground_truth_column`: Give any name to this key. Use the same name in target_field_name in the below pipeline parameters.\n",
         "- `value`: Ground truth value of this instance.\n",
         "\n",
         " "
@@ -1600,14 +1602,14 @@
         "\n",
         "- `GetVertexModelOp`: Gets a Vertex AI Model resource Artifact. \n",
         "- `EvaluationDataSamplerOp`: Randomly downsamples an input dataset to a specified size for computing Vertex Explainable AI feature attributions for AutoML Tabular and custom models. Creates a Dataflow job with Apache Beam to downsample the dataset. \n",
-        "- `TargetFieldDataRemoverOp`: Removes the target field from the input dataset for supporting unstructured AutoML models and custom models for Vertex Batch Prediction. Creates a Dataflow job with Apache Beam to remove the target field.. \n",
+        "- `TargetFieldDataRemoverOp`: Removes the Ground Truth columns from the input dataset for supporting unstructured AutoML models and custom models in Batch Prediction. Creates a Dataflow job with Apache Beam to remove the ground truth columns. \n",
         "- `ModelBatchPredictOp`: Creates a Google Cloud Vertex BatchPredictionJob and waits for it to complete. \n",
         "- `ModelEvaluationRegressionOp`: Compute evaluation metrics on a trained model’s batch prediction results. Creates a Dataflow job with Apache Beam and TFMA to compute evaluation metrics. Supports regression for tabular data.\n",
         "- `ModelEvaluationFeatureAttributionOp`: Compute feature attribution on a trained model’s batch explanation results. Creates a Dataflow job with Apache Beam and TFMA to compute feature attributions. \n",
         "- `ModelImportEvaluationOp`: Imports a model evaluation artifact to an existing Vertex AI Model resource with ModelService.ImportModelEvaluation. \n",
         "\n",
         "\n",
-        "Learn more about [Google Cloud Pipeline Model Evaluation components](https://google-cloud-pipeline-components.readthedocs.io/en/google-cloud-pipeline-components-1.0.20/google_cloud_pipeline_components.experimental.evaluation.html).\n",
+        "Learn more about [Google Cloud Pipeline Model Evaluation components](https://google-cloud-pipeline-components.readthedocs.io/en/google-cloud-pipeline-components-1.0.26/google_cloud_pipeline_components.experimental.evaluation.html).\n",
         "\n",
         "##### Example workflow\n",
         "\n",
@@ -1648,7 +1650,7 @@
         "    location: str,\n",
         "    root_dir: str,\n",
         "    model_name: str,\n",
-        "    target_column_name: str,\n",
+        "    target_field_name: str,\n",
         "    batch_predict_gcs_source_uris: list,\n",
         "    batch_predict_instances_format: str,\n",
         "    batch_predict_sample_size: int,\n",
@@ -1682,7 +1684,7 @@
         "        root_dir=root_dir,\n",
         "        gcs_source_uris=data_sampler_task.outputs[\"gcs_output_directory\"],\n",
         "        instances_format=batch_predict_instances_format,\n",
-        "        target_field_name=target_column_name,\n",
+        "        target_field_name=target_field_name,\n",
         "    )\n",
         "\n",
         "    # Run Batch Explanations\n",
@@ -1711,7 +1713,7 @@
         "        ground_truth_gcs_source=data_sampler_task.outputs[\"gcs_output_directory\"],\n",
         "        predictions_format=batch_predict_predictions_format,\n",
         "        prediction_score_column=\"prediction\",\n",
-        "        target_field_name=target_column_name,\n",
+        "        target_field_name=target_field_name,\n",
         "    )\n",
         "\n",
         "    # Get Feature Attributions\n",
@@ -1773,7 +1775,7 @@
         "- `location`: Region where the pipeline is run.\n",
         "- `root_dir`: The Cloud Storage directory for keeping staging files and artifacts. A random subdirectory will be created under the directory to keep job info for resuming the job in case of failure.\n",
         "- `model_name`: Resource name of the trained Custom Tabular Regression model.\n",
-        "- `target_column_name`: Name of the column to be used as the target for regression.\n",
+        "- `target_field_name`: Name of the column to be used as the target for regression.\n",
         "- `batch_predict_gcs_source_uris`: List of the Cloud Storage bucket uris of input instances for batch prediction.\n",
         "- `batch_predict_instances_format`:  Format of the input instances for batch prediction. Can be \"jsonl\", \"csv\" or \"bigquery\".\n",
         "- `batch_predict_explanation_data_sample_size`: Size of the samples to be considered for batch prediction and evaluation.\n"
@@ -1794,7 +1796,7 @@
         "    \"location\": REGION,\n",
         "    \"root_dir\": PIPELINE_ROOT,\n",
         "    \"model_name\": model.resource_name,\n",
-        "    \"target_column_name\": \"MEDV\",\n",
+        "    \"target_field_name\": \"MEDV\",\n",
         "    \"batch_predict_gcs_source_uris\": [\n",
         "        BUCKET_URI + \"/\" + \"test_file_with_ground_truth.jsonl\"\n",
         "    ],\n",

notebooks/official/model_monitoring/README.md

@@ -1,7 +1,4 @@
 
-### model_monitoring 
-
-
 [Vertex AI Batch Prediction with Model Monitoring](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model_monitoring/batch_prediction_model_monitoring.ipynb)
 
 ```
@@ -15,6 +12,8 @@ The steps performed include:
 
 ```
 
+   Learn more about [Vertex AI Model Monitoring for batch predictions](https://cloud.google.com/vertex-ai/docs/model-monitoring/model-monitoring-batch-predictions).
+
 
 [Vertex AI Model Monitoring with Explainable AI Feature Attributions](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model_monitoring/model_monitoring.ipynb)
 
@@ -33,3 +32,5 @@ The steps performed include:
 
 ```
 
+   Learn more about [Vertex AI Model Monitoring](https://cloud.google.com/vertex-ai/docs/model-monitoring).
+

notebooks/official/model_monitoring/batch_prediction_model_monitoring.ipynb

@@ -61,7 +61,9 @@
       "source": [
         "## Overview\n",
         "\n",
-        "In this notebook, you will learn how to use Model Monitoring with batch prediction requests on a deployed Vertex AI Model resource. In a companion notebook, <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model_monitoring/model_monitoring.ipynb\" target=\"_blank\">Vertex AI Model Monitoring with Explainable AI Feature Attributions</a>, you can learn about how to apply model monitoring to streaming, real-time predictions."
+        "In this notebook, you will learn how to use Model Monitoring with batch prediction requests on a deployed Vertex AI Model resource. In a companion notebook, <a href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model_monitoring/model_monitoring.ipynb\" target=\"_blank\">Vertex AI Model Monitoring with Explainable AI Feature Attributions</a>, you can learn about how to apply model monitoring to streaming, real-time predictions.\n",
+        "\n",
+        "Learn more about [Vertex AI Model Monitoring for batch predictions](https://cloud.google.com/vertex-ai/docs/model-monitoring/model-monitoring-batch-predictions)."
       ]
     },
     {

notebooks/official/model_monitoring/model_monitoring.ipynb

@@ -80,7 +80,9 @@
         "\n",
         "[Vertex Explainable AI](https://cloud.google.com/vertex-ai/docs/explainable-ai/overview) adds another facet to model monitoring, which we call feature attribution monitoring. Explainable AI enables you to understand the relative contribution of each feature to a resulting prediction. In essence, it assesses the magnitude of each feature's influence.\n",
         "\n",
-        "If production traffic differs from  training data, or varies substantially over time, **either in terms of model predictions or feature attributions**, that's likely to impact the quality of the answers your model produces. When that happens, you'd like to be alerted automatically and responsively, so that **you can anticipate problems before they affect your customer experiences or your revenue streams**."
+        "If production traffic differs from  training data, or varies substantially over time, **either in terms of model predictions or feature attributions**, that's likely to impact the quality of the answers your model produces. When that happens, you'd like to be alerted automatically and responsively, so that **you can anticipate problems before they affect your customer experiences or your revenue streams**.\n",
+        "\n",
+        "Learn more about [Vertex AI Model Monitoring](https://cloud.google.com/vertex-ai/docs/model-monitoring)."
       ]
     },
     {

notebooks/official/model_registry/README.md

@@ -1,7 +1,4 @@
 
-### model_registry 
-
-
 [Deploy BiqQuery ML Model on Vertex AI Model Registry and make predictions](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/model_registry/bqml_vertexai_model_registry.ipynb)
 
 ```
@@ -18,3 +15,7 @@ The steps performed include:
 
 ```
 
+   Learn more about [Vertex AI Model Registry](https://cloud.google.com/vertex-ai/docs/model-registry/introduction).
+
+   Learn more about [BigQuery ML](https://cloud.google.com/bigquery-ml/docs/introduction).
+

notebooks/official/model_registry/bqml_vertexai_model_registry.ipynb

@@ -61,7 +61,9 @@
       "source": [
         "## Overview\n",
         "\n",
-        "This tutorial demonstrates how to train a model with BigQuery ML and upload it on Vertex AI Model Registry, then make batch predictions.\n"
+        "This tutorial demonstrates how to train a model with BigQuery ML and upload it on Vertex AI Model Registry, then make batch predictions.\n",
+        "\n",
+        "Learn more about [Vertex AI Model Registry](https://cloud.google.com/vertex-ai/docs/model-registry/introduction) and [BigQuery ML](https://cloud.google.com/bigquery-ml/docs/introduction)."
       ]
     },
     {

notebooks/official/pipelines/README.md

@@ -1,76 +1,4 @@
 
-### pipelines 
-
-
-[AutoML image classification pipelines using google-cloud-pipeline-components](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/google_cloud_pipeline_components_automl_images.ipynb)
-
-```
-Learn how to use `Vertex AI Pipelines` and `Google Cloud Pipeline Components` to build an `AutoML` image classification model.
-
-The steps performed include:
-
-- Create a KFP pipeline:
-    - Create a `Dataset` resource.
-    - Train an AutoML image classification `Model` resource.
-    - Create an `Endpoint` resource.
-    - Deploys the `Model` resource to the `Endpoint` resource.
-- Compile the KFP pipeline.
-- Execute the KFP pipeline using `Vertex AI Pipelines`
-
-```
-
-
-[Metrics visualization and run comparison using the KFP SDK](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/metrics_viz_run_compare_kfp.ipynb)
-
-```
-Learn how to use the KFP SDK to build pipelines that generate evaluation metrics.
-
-The steps performed include:
-
-- Create KFP components:
-    - Generate ROC curve and confusion matrix visualizations for classification results
-    - Write metrics
-- Create KFP pipelines.
-- Execute KFP pipelines
-- Compare metrics across pipeline runs
-
-```
-
-
-[Lightweight Python function-based components, and component I/O](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/lightweight_functions_component_io_kfp.ipynb)
-
-```
-Learn to use the KFP SDK to build lightweight Python function-based components, and then you learn to use `Vertex AI Pipelines` to execute the pipeline.
-
-The steps performed include:
-
-- Build Python function-based KFP components.
-- Construct a KFP pipeline.
-- Pass *Artifacts* and *parameters* between components, both by path reference and by value.
-- Use the `kfp.dsl.importer` method.
-- Compile the KFP pipeline.
-- Execute the KFP pipeline using `Vertex AI Pipelines`
-
-```
-
-
-[Custom training with pre-built Google Cloud Pipeline Components](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/custom_model_training_and_batch_prediction.ipynb)
-
-```
-Learn to use `Vertex AI Pipelines` and `Google Cloud Pipeline Components` to build a custom model.
-
-The steps performed include:
-
-- Create a KFP pipeline:
-    - Train a custom model.
-    - Upload the trained model as a `Model` resource.
-    - Create an `Endpoint` resource.
-    - Deploy the `Model` resource to the `Endpoint` resource.
-    - Make a batch prediction request.
-
-```
-
-
 [AutoML Tabular pipelines using google-cloud-pipeline-components](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/automl_tabular_classification_beans.ipynb)
 
 ```
@@ -88,68 +16,9 @@ The steps performed include:
 
 ```
 
+   Learn more about [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction).
 
-[Training an acquisition-prediction model using Swivel, BigQuery ML and Vertex AI Pipelines](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/google_cloud_pipeline_components_bqml_text.ipynb)
-
-```
-Learn how to build a simple BigQuery ML pipeline using Vertex AI pipelines in order to calculate text embeddings of content from articles and classify them
-into the *corporate acquisitions* category.
-
-The steps performed include:
-
-- Creating a component for Dataflow job that ingests data to BigQuery.
-- Creating a component for preprocessing steps to run on the data in BigQuery.
-- Creating a component for training a logistic regression model using BigQuery ML.
-- Building and configuring a Kubeflow DSL pipeline with all the created components.
-- Compiling and running the pipeline in Vertex AI Pipelines.
-
-```
-
-
-[Pipelines introduction for KFP](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/pipelines_intro_kfp.ipynb)
-
-```
-Learn how to use the KFP SDK to build pipelines that generate evaluation metrics.
-
-The steps performed include:
-
-- Define and compile a `Vertex AI` pipeline.
-- Specify which service account to use for a pipeline run.
-
-```
-
-
-[AutoML tabular regression pipelines using google-cloud-pipeline-components](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/google_cloud_pipeline_components_automl_tabular.ipynb)
-
-```
-Learn to use `Vertex AI Pipelines` and `Google Cloud Pipeline Components` to build an `AutoML` tabular regression model.
-
-The steps performed include:
-
-- Create a KFP pipeline:
-    - Create a `Dataset` resource.
-    - Train an AutoML tabular regression `Model` resource.
-    - Create an `Endpoint` resource.
-    - Deploys the `Model` resource to the `Endpoint` resource.
-- Compile the KFP pipeline.
-- Execute the KFP pipeline using `Vertex AI Pipelines`
-
-```
-
-
-[Model upload, predict, and evaluate using google-cloud-pipeline-components](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/google_cloud_pipeline_components_model_upload_predict_evaluate.ipynb)
-
-```
-Learn how to evaluate a custom model using a pipeline with components from `google_cloud_pipeline_components` and a custom pipeline component you build.
-
-The steps performed include:
-
-- Upload a pre-trained model as a `Model` resource.
-- Run a `BatchPredictionJob` on the `Model` resource with ground truth data.
-- Generate evaluation `Metrics` artifact about the `Model` resource.
-- Compare the evaluation metrics to a threshold.
-
-```
+   Learn more about [AutoML components](https://cloud.google.com/vertex-ai/docs/pipelines/vertex-automl-component).
 
 
 [Pipeline control structures using the KFP SDK](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/control_flow_kfp.ipynb)
@@ -166,58 +35,29 @@ The steps performed include:
 
 ```
 
+   Learn more about [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction).
 
-[Loan eligibility prediction using `google-cloud-pipeline-components` and Spark ML](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/google_cloud_pipeline_components_dataproc_tabular.ipynb)
+
+[Custom training with pre-built Google Cloud Pipeline Components](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/custom_model_training_and_batch_prediction.ipynb)
 
 ```
-Learn how to build a Vertex AI pipeline and train a Random-forest model using Spark ML for loan-eligibility classification problem.
-
-The steps performed include:
-
-*   Use the `DataprocPySparkBatchOp` to preprocess data.
-*   Create a Vertex AI dataset resource on the training data.
-*   Train a random forest model using Pyspark.
-*   Build a Vertex AI pipeline and run the training job.
-*   Use the Spark serving image in order to deploy a Spark model on Vertex AI Endpoint.
-
-```
-
-
-[Model train, upload, and deploy using Google Cloud Pipeline Components](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/google_cloud_pipeline_components_model_train_upload_deploy.ipynb)
-
-```
-Learn how to use `Vertex AI Pipelines` and `Google Cloud Pipeline Components` to build and deploy a custom model.
+Learn to use `Vertex AI Pipelines` and `Google Cloud Pipeline Components` to build a custom model.
 
 The steps performed include:
 
 - Create a KFP pipeline:
     - Train a custom model.
-    - Uploads the trained model as a `Model` resource.
-    - Creates an `Endpoint` resource.
-    - Deploys the `Model` resource to the `Endpoint` resource.
-- Compile the KFP pipeline.
-- Execute the KFP pipeline using `Vertex AI Pipelines`
-
-```
-
-
-[AutoML text classification pipelines using google-cloud-pipeline-components](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/google_cloud_pipeline_components_automl_text.ipynb)
-
-```
-Learn to use `Vertex AI Pipelines` and `Google Cloud Pipeline Components` to build an `AutoML` text classification model.
-
-The steps performed include:
-
-- Create a KFP pipeline:
-    - Create a `Dataset` resource.
-    - Train an AutoML text classification `Model` resource.
+    - Upload the trained model as a `Model` resource.
     - Create an `Endpoint` resource.
-    - Deploys the `Model` resource to the `Endpoint` resource.
-- Compile the KFP pipeline.
-- Execute the KFP pipeline using `Vertex AI Pipelines`
+    - Deploy the `Model` resource to the `Endpoint` resource.
+    - Make a batch prediction request.
 
 ```
 
+   Learn more about [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction).
+
+   Learn more about [Vertex AI Training components](https://cloud.google.com/vertex-ai/docs/training/create-training-pipeline).
+
 
 [Training and batch prediction with BigQuery source and destinantion for a custom tabular classification model](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/custom_tabular_train_batch_pred_bq_pipeline.ipynb)
 
@@ -240,3 +80,208 @@ The steps performed include:
 
 ```
 
+   Learn more about [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction).
+
+   Learn more about [Vertex AI Batch Prediction components](https://cloud.google.com/vertex-ai/docs/pipelines/batchprediction-component).
+
+
+[AutoML image classification pipelines using google-cloud-pipeline-components](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/google_cloud_pipeline_components_automl_images.ipynb)
+
+```
+Learn how to use `Vertex AI Pipelines` and `Google Cloud Pipeline Components` to build an `AutoML` image classification model.
+
+The steps performed include:
+
+- Create a KFP pipeline:
+    - Create a `Dataset` resource.
+    - Train an AutoML image classification `Model` resource.
+    - Create an `Endpoint` resource.
+    - Deploys the `Model` resource to the `Endpoint` resource.
+- Compile the KFP pipeline.
+- Execute the KFP pipeline using `Vertex AI Pipelines`
+
+```
+
+   Learn more about [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction).
+
+   Learn more about [AutoML components](https://cloud.google.com/vertex-ai/docs/pipelines/vertex-automl-component).
+
+
+[AutoML tabular regression pipelines using google-cloud-pipeline-components](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/google_cloud_pipeline_components_automl_tabular.ipynb)
+
+```
+Learn to use `Vertex AI Pipelines` and `Google Cloud Pipeline Components` to build an `AutoML` tabular regression model.
+
+The steps performed include:
+
+- Create a KFP pipeline:
+    - Create a `Dataset` resource.
+    - Train an AutoML tabular regression `Model` resource.
+    - Create an `Endpoint` resource.
+    - Deploys the `Model` resource to the `Endpoint` resource.
+- Compile the KFP pipeline.
+- Execute the KFP pipeline using `Vertex AI Pipelines`
+
+```
+
+   Learn more about [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction).
+
+   Learn more about [AutoML components](https://cloud.google.com/vertex-ai/docs/pipelines/vertex-automl-component).
+
+
+[AutoML text classification pipelines using google-cloud-pipeline-components](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/google_cloud_pipeline_components_automl_text.ipynb)
+
+```
+Learn to use `Vertex AI Pipelines` and `Google Cloud Pipeline Components` to build an `AutoML` text classification model.
+
+The steps performed include:
+
+- Create a KFP pipeline:
+    - Create a `Dataset` resource.
+    - Train an AutoML text classification `Model` resource.
+    - Create an `Endpoint` resource.
+    - Deploys the `Model` resource to the `Endpoint` resource.
+- Compile the KFP pipeline.
+- Execute the KFP pipeline using `Vertex AI Pipelines`
+
+```
+
+   Learn more about [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction).
+
+   Learn more about [AutoML components](https://cloud.google.com/vertex-ai/docs/pipelines/vertex-automl-component).
+
+
+[Training an acquisition-prediction model using Swivel, BigQuery ML and Vertex AI Pipelines](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/google_cloud_pipeline_components_bqml_text.ipynb)
+
+```
+Learn how to build a simple BigQuery ML pipeline using Vertex AI pipelines in order to calculate text embeddings of content from articles and classify them
+into the *corporate acquisitions* category.
+
+The steps performed include:
+
+- Creating a component for Dataflow job that ingests data to BigQuery.
+- Creating a component for preprocessing steps to run on the data in BigQuery.
+- Creating a component for training a logistic regression model using BigQuery ML.
+- Building and configuring a Kubeflow DSL pipeline with all the created components.
+- Compiling and running the pipeline in Vertex AI Pipelines.
+
+```
+
+   Learn more about [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction).
+
+   Learn more about [BigQuery ML components](https://cloud.google.com/vertex-ai/docs/pipelines/bigqueryml-component).
+
+
+[Loan eligibility prediction using `google-cloud-pipeline-components` and Spark ML](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/google_cloud_pipeline_components_dataproc_tabular.ipynb)
+
+```
+Learn how to build a Vertex AI pipeline and train a random-forest model using Spark ML for loan-eligibility classification problem.
+
+The steps performed include:
+
+*   Use the `DataprocPySparkBatchOp` to preprocess data.
+*   Create a Vertex AI dataset resource on the training data.
+*   Train a random forest model using PySpark.
+*   Build a Vertex AI pipeline and run the training job.
+*   Use the Spark serving image in order to deploy a Spark model on Vertex AI Endpoint.
+
+```
+
+   Learn more about [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction).
+
+   Learn more about [Dataproc components](https://cloud.google.com/vertex-ai/docs/pipelines/dataproc-component).
+
+
+[Model train, upload, and deploy using Google Cloud Pipeline Components](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/google_cloud_pipeline_components_model_train_upload_deploy.ipynb)
+
+```
+Learn how to use `Vertex AI Pipelines` and `Google Cloud Pipeline Components` to build and deploy a custom model.
+
+The steps performed include:
+
+- Create a KFP pipeline:
+    - Train a custom model.
+    - Uploads the trained model as a `Model` resource.
+    - Creates an `Endpoint` resource.
+    - Deploys the `Model` resource to the `Endpoint` resource.
+- Compile the KFP pipeline.
+- Execute the KFP pipeline using `Vertex AI Pipelines`
+
+```
+
+   Learn more about [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction).
+
+   Learn more about [Vertex AI Training components](https://cloud.google.com/vertex-ai/docs/pipelines/customjob-component).
+
+
+[Model upload, predict, and evaluate using google-cloud-pipeline-components](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/google_cloud_pipeline_components_model_upload_predict_evaluate.ipynb)
+
+```
+Learn how to evaluate a custom model using a pipeline with components from `google_cloud_pipeline_components` and a custom pipeline component you build.
+
+The steps performed include:
+
+- Upload a pre-trained model as a `Model` resource.
+- Run a `BatchPredictionJob` on the `Model` resource with ground truth data.
+- Generate evaluation `Metrics` artifact about the `Model` resource.
+- Compare the evaluation metrics to a threshold.
+
+```
+
+   Learn more about [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction).
+
+   Learn more about [Vertex AI Model components](https://cloud.google.com/vertex-ai/docs/pipelines/model-endpoint-component).
+
+
+[Lightweight Python function-based components, and component I/O](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/lightweight_functions_component_io_kfp.ipynb)
+
+```
+Learn to use the KFP SDK to build lightweight Python function-based components, and then you learn to use `Vertex AI Pipelines` to execute the pipeline.
+
+The steps performed include:
+
+- Build Python function-based KFP components.
+- Construct a KFP pipeline.
+- Pass *Artifacts* and *parameters* between components, both by path reference and by value.
+- Use the `kfp.dsl.importer` method.
+- Compile the KFP pipeline.
+- Execute the KFP pipeline using `Vertex AI Pipelines`
+
+```
+
+   Learn more about [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction).
+
+
+[Metrics visualization and run comparison using the KFP SDK](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/metrics_viz_run_compare_kfp.ipynb)
+
+```
+Learn how to use the KFP SDK to build pipelines that generate evaluation metrics.
+
+The steps performed include:
+
+- Create KFP components:
+    - Generate ROC curve and confusion matrix visualizations for classification results
+    - Write metrics
+- Create KFP pipelines.
+- Execute KFP pipelines
+- Compare metrics across pipeline runs
+
+```
+
+   Learn more about [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction).
+
+
+[Pipelines introduction for KFP](https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/official/pipelines/pipelines_intro_kfp.ipynb)
+
+```
+Learn how to use the KFP SDK to build pipelines that generate evaluation metrics.
+
+The steps performed include:
+
+- Define and compile a `Vertex AI` pipeline.
+- Specify which service account to use for a pipeline run.
+
+```
+
+   Learn more about [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction).
+

notebooks/official/pipelines/automl_tabular_classification_beans.ipynb

@@ -66,7 +66,9 @@
         "\n",
         "You build a pipeline in this notebook that looks like this:\n",
         "\n",
-        "<a href=\"https://storage.googleapis.com/amy-jo/images/mp/beans.png\" target=\"_blank\"><img src=\"https://storage.googleapis.com/amy-jo/images/mp/beans.png\" width=\"95%\"/></a>"
+        "<a href=\"https://storage.googleapis.com/amy-jo/images/mp/beans.png\" target=\"_blank\"><img src=\"https://storage.googleapis.com/amy-jo/images/mp/beans.png\" width=\"95%\"/></a>\n",
+        "\n",
+        "Learn more about [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction) and [AutoML components](https://cloud.google.com/vertex-ai/docs/pipelines/vertex-automl-component)."
       ]
     },
     {

notebooks/official/pipelines/control_flow_kfp.ipynb

@@ -61,7 +61,9 @@
       "source": [
         "## Overview\n",
         "\n",
-        "This notebooks shows how to use [the Kubeflow Pipelines (KFP) SDK](https://www.kubeflow.org/docs/components/pipelines/) to build [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines) that use control structures."
+        "This notebooks shows how to use [the Kubeflow Pipelines (KFP) SDK](https://www.kubeflow.org/docs/components/pipelines/) to build [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines) that use control structures.\n",
+        "\n",
+        "Learn more about [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction)."
       ]
     },
     {

notebooks/official/pipelines/custom_model_training_and_batch_prediction.ipynb

@@ -63,7 +63,9 @@
         "## Overview\n",
         "\n",
         "\n",
-        "This tutorial demonstrates how to use Vertex AI Pipelines with pre-built Google Cloud Pipeline Components for custom training."
+        "This tutorial demonstrates how to use Vertex AI Pipelines with pre-built Google Cloud Pipeline Components for custom training.\n",
+        "\n",
+        "Learn more about [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction) and [Vertex AI Training components](https://cloud.google.com/vertex-ai/docs/training/create-training-pipeline)."
       ]
     },
     {

notebooks/official/pipelines/custom_tabular_train_batch_pred_bq_pipeline.ipynb

@@ -61,7 +61,9 @@
       "source": [
         "## Overview\n",
         "\n",
-        "This notebook demonstrates performing training and batch prediction for a custom tabular classification model inside a Vertex AI pipeline. The batch prediction job takes data from a BigQuery source and writes the results to a BigQuery destination."
+        "This notebook demonstrates performing training and batch prediction for a custom tabular classification model inside a Vertex AI pipeline. The batch prediction job takes data from a BigQuery source and writes the results to a BigQuery destination.\n",
+        "\n",
+        "Learn more about [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction) and [Vertex AI Batch Prediction components](https://cloud.google.com/vertex-ai/docs/pipelines/batchprediction-component)."
       ]
     },
     {

notebooks/official/pipelines/google_cloud_pipeline_components_automl_images.ipynb

@@ -61,7 +61,9 @@
       "source": [
         "## Overview\n",
         "\n",
-        "This notebook shows how to use the components defined in [`google_cloud_pipeline_components`](https://github.com/kubeflow/pipelines/tree/master/components/google-cloud) to build an `AutoML` image classification workflow on [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines)."
+        "This notebook shows how to use the components defined in [`google_cloud_pipeline_components`](https://github.com/kubeflow/pipelines/tree/master/components/google-cloud) to build an `AutoML` image classification workflow on [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines).\n",
+        "\n",
+        "Learn more about [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction) and [AutoML components](https://cloud.google.com/vertex-ai/docs/pipelines/vertex-automl-component)."
       ]
     },
     {

notebooks/official/pipelines/google_cloud_pipeline_components_automl_tabular.ipynb

@@ -61,7 +61,9 @@
       "source": [
         "## Overview\n",
         "\n",
-        "This notebook shows how to use the components defined in [`google_cloud_pipeline_components`](https://github.com/kubeflow/pipelines/tree/master/components/google-cloud) to build an AutoML tabular regression workflow on [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines)."
+        "This notebook shows how to use the components defined in [`google_cloud_pipeline_components`](https://github.com/kubeflow/pipelines/tree/master/components/google-cloud) to build an AutoML tabular regression workflow on Vertex AI Pipelines.\n",
+        "\n",
+        "Learn more about [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction) and [AutoML components](https://cloud.google.com/vertex-ai/docs/pipelines/vertex-automl-component)."
       ]
     },
     {

notebooks/official/pipelines/google_cloud_pipeline_components_automl_text.ipynb

@@ -61,7 +61,9 @@
       "source": [
         "## Overview\n",
         "\n",
-        "This notebook shows how to use the components defined in [`google_cloud_pipeline_components`](https://github.com/kubeflow/pipelines/tree/master/components/google-cloud) to build an AutoML text classification workflow on [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines)."
+        "This notebook shows how to use the components defined in [`google_cloud_pipeline_components`](https://github.com/kubeflow/pipelines/tree/master/components/google-cloud) to build an AutoML text classification workflow on [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines).\n",
+        "\n",
+        "Learn more about [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction) and [AutoML components](https://cloud.google.com/vertex-ai/docs/pipelines/vertex-automl-component)."
       ]
     },
     {

notebooks/official/pipelines/google_cloud_pipeline_components_bqml_text.ipynb

@@ -70,7 +70,9 @@
         "3. Apply the Swivel model to generate embeddings of your document’s content.\n",
         "4. Train a Logistic regression model to classify if an article is about corporate acquisitions (`acq` category). \n",
         "5. Evaluate the model.\n",
-        "6. Apply the model to a dataset in order to generate predictions."
+        "6. Apply the model to a dataset in order to generate predictions.\n",
+        "\n",
+        "Learn more about [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction) and [BigQuery ML components](https://cloud.google.com/vertex-ai/docs/pipelines/bigqueryml-component)."
       ]
     },
     {

notebooks/official/pipelines/google_cloud_pipeline_components_dataproc_tabular.ipynb

@@ -61,7 +61,9 @@
       "source": [
         "## Overview\n",
         "\n",
-        "This notebook shows how to build a Spark ML pipeline using Spark MLlib and DataprocPySparkBatchOp component to determine the customer eligibility for a loan from a banking company. In particular, the pipeline covers a Spark MLib pipeline, from data preprocessing to hyperparameter tuning of a random forest classifier which predicts the probability of a customer being eligible for a loan. "
+        "This notebook shows how to build a Spark ML pipeline using Spark MLlib and DataprocPySparkBatchOp component to determine the customer eligibility for a loan from a banking company. In particular, the pipeline covers a Spark MLib pipeline, from data preprocessing to hyperparameter tuning of a random forest classifier which predicts the probability of a customer being eligible for a loan. \n",
+        "\n",
+        "Learn more about [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction) and [Dataproc components](https://cloud.google.com/vertex-ai/docs/pipelines/dataproc-component)."
       ]
     },
     {
@@ -72,7 +74,7 @@
       "source": [
         "### Objective\n",
         "\n",
-        "In this notebook, you learn how to build a Vertex AI pipeline and train a Random-forest model using Spark ML for loan-eligibility classification problem.  \n",
+        "In this notebook, you learn how to build a Vertex AI pipeline and train a random-forest model using Spark ML for loan-eligibility classification problem.  \n",
         "\n",
         "This tutorial uses the following Google Cloud ML services and resources:\n",
         "\n",
@@ -85,7 +87,7 @@
         "\n",
         "*   Use the `DataprocPySparkBatchOp` to preprocess data.\n",
         "*   Create a Vertex AI dataset resource on the training data.\n",
-        "*   Train a random forest model using Pyspark.\n",
+        "*   Train a random forest model using PySpark.\n",
         "*   Build a Vertex AI pipeline and run the training job.\n",
         "*   Use the Spark serving image in order to deploy a Spark model on Vertex AI Endpoint."
       ]
@@ -98,7 +100,7 @@
       "source": [
         "### Dataset\n",
         "\n",
-        "The dataset is a preprocessed version of the [loan eligiability dataset](https://datasetsearch.research.google.com/search?src=2&query=Loan%20Eligible%20Dataset&docid=L2cvMTFsajJrM3EzcA%3D%3D)."
+        "The dataset is a preprocessed version of the [loan eligibility dataset](https://datasetsearch.research.google.com/search?src=2&query=Loan%20Eligible%20Dataset&docid=L2cvMTFsajJrM3EzcA%3D%3D)."
       ]
     },
     {
@@ -671,7 +673,7 @@
       "source": [
         "### Load preprocessing data\n",
         "\n",
-        "The notebook uses a preprocessed set of data you would read from the Vertex AI Feature Store. "
+        "The notebook uses a preprocessed set of data you read from the Vertex AI Feature Store. "
       ]
     },
     {
@@ -726,7 +728,7 @@
       "source": [
         "### Create the Docker repository\n",
         "\n",
-        "You create a Docker repository in the Artefact Registry for the custom dataproc image that you are going to create."
+        "You create a Docker repository in the Artifact Registry for the custom dataproc image that you are going to create."
       ]
     },
     {
@@ -737,7 +739,7 @@
       },
       "outputs": [],
       "source": [
-        "REPO_NAME = \"loan-eligiability-spark-demo\"\n",
+        "REPO_NAME = \"loan-eligibility-spark-demo\"\n",
         "\n",
         "!gcloud artifacts repositories create $REPO_NAME \\\n",
         "    --repository-format=docker \\\n",
@@ -769,8 +771,8 @@
         "\n",
         "from google.cloud import aiplatform as vertex_ai\n",
         "from kfp.v2 import compiler, dsl\n",
-        "from kfp.v2.dsl import (Artifact, ClassificationMetrics, Condition, Input,\n",
-        "                        Metrics, Output, component)"
+        "from kfp.v2.dsl import (ClassificationMetrics, Condition, Metrics, Output,\n",
+        "                        component)"
       ]
     },
     {
@@ -791,7 +793,7 @@
         "IMAGE_TAG = \"1.0.0\"\n",
         "\n",
         "# Pipeline\n",
-        "PIPELINE_NAME = \"pyspark-loan-eligiability-pipeline\"\n",
+        "PIPELINE_NAME = \"pyspark-loan-eligibility-pipeline\"\n",
         "PIPELINE_ROOT = f\"{BUCKET_URI}/pipelines\"\n",
         "PIPELINE_PACKAGE_PATH = str(BUILD_PATH / f\"pipeline_{UUID}.json\")\n",
         "RUNTIME_CONTAINER_IMAGE = f\"gcr.io/{PROJECT_ID}/{RUNTIME_IMAGE}:{IMAGE_TAG}\"\n",
@@ -813,10 +815,6 @@
         "    PROCESSED_DATA_URI,\n",
         "]\n",
         "\n",
-        "# Dataset\n",
-        "DATASET_NAME = f\"preprocessed-dataset-{UUID}\"\n",
-        "GCS_PREPROCESSED_URI = f\"{PROCESSED_DATA_URI}/*/?.csv\"\n",
-        "\n",
         "# Training\n",
         "TRAINING_PYTHON_FILE_URI = f\"{BUCKET_URI}/src/model_training.py\"\n",
         "MODEL_URI = f\"{BUCKET_URI}/deliverables/model/rfor/{UUID}/train_model\"\n",
@@ -861,6 +859,9 @@
         "    \"spark.jars.packages\": \"ml.combust.mleap:mleap-spark-base_2.12:0.20.0,ml.combust.mleap:mleap-spark_2.12:0.20.0\"\n",
         "}\n",
         "\n",
+        "# Experiment\n",
+        "EXPERIMENT_NAME = \"loan-eligibility\"\n",
+        "\n",
         "# Deploy\n",
         "SERVING_IMAGE_URI = f\"{REGION}-docker.pkg.dev/{PROJECT_ID}/{REPO_NAME}/spark-ml-serving\""
       ]
@@ -1120,11 +1121,11 @@
       "source": [
         "#### Create the source code for model-training\n",
         "\n",
-        "Create the `model_training.py` file for training a Random-forest classifier model on the training data. The training is performed using Spark ML inside a Spark session. The code fetches the training data from the Cloud storage bucket, processes it and trains the Random-forest model. The trained model and the metrics obtained from the trained model (like AUC-ROC, accuracy, precision etc.) are then saved to the provided output Cloud Storage path. This code accepts the following arguments:\n",
+        "Create the `model_training.py` file for training a random-forest classifier model on the training data. The training is performed using Spark ML inside a Spark session. The code fetches the training data from the Cloud storage bucket, processes it and trains the random-forest model. The trained model and the metrics obtained from the trained model (like AUC-ROC, accuracy, precision etc.) are then saved to the provided output Cloud Storage path. This code accepts the following arguments:\n",
         "\n",
-        "- `--train-path`: The GCS path of the training sample.\n",
-        "- `--model-path`: The GCS path to store the trained model.\n",
-        "- `--metrics-path`: The GCS path to store the metrics of model."
+        "- `--train-path`: The Cloud Storage path of the training sample.\n",
+        "- `--model-path`: The Cloud Storage path to store the trained model.\n",
+        "- `--metrics-path`: The Cloud Storage path to store the metrics of model."
       ]
     },
     {
@@ -1480,7 +1481,7 @@
       "source": [
         "#### Create the source code for hyperparameter-tuning\n",
         "\n",
-        "Create the `hp_tuning.py` file for tuning the hyperparameters of the Random-forest classifier model using crossvalidation. This code accepts the following arguments:\n",
+        "Create the `hp_tuning.py` file for tuning the hyperparameters of the random-forest classifier model using crossvalidation. This code accepts the following arguments:\n",
         "\n",
         "- `--train-path`: The GCS path of the training sample.\n",
         "- `--model-path`: The GCS path to store the trained model.\n",
@@ -1889,9 +1890,9 @@
       "source": [
         "### Build a custom Dataproc Serverless container image\n",
         "\n",
-        "Dataproc Serverless provides [default runtime images](https://cloud.google.com/dataproc-serverless/docs/concepts/versions/spark-runtime-versions). You can also use custom container images for your Dataproc Serverless workloads. \n",
+        "Dataproc Serverless provides default runtime images. Learn more about the [Dataproc Serverless Spark runtime releases](https://cloud.google.com/dataproc-serverless/docs/concepts/versions/spark-runtime-versions).\n",
         "\n",
-        "The steps in this section builds a custom container image that includes additional dependencies. The custom container image can be specified when using the `DataprocPySparkBatchOp` component to launch the workload within a pipeline."
+        "You can also use custom container images for your Dataproc Serverless workloads. The steps in this section builds a custom container image that includes additional dependencies. The custom container image can be specified when using the `DataprocPySparkBatchOp` component to launch the workload within a pipeline."
       ]
     },
     {
@@ -2008,7 +2009,7 @@
         "id": "ZXzI2xInqb3V"
       },
       "source": [
-        "#### Build the Dataproc Serverless custom runtime using Google Cloud Build\n",
+        "#### Build the Dataproc Serverless custom runtime using Cloud Build\n",
         "\n",
         "**Note:** this step may take approximately 5 to 10 minutes to complete."
       ]
@@ -2032,15 +2033,8 @@
       "source": [
         "### Build custom components for pipeline arguments\n",
         "\n",
-        "In order to pass job arguments, you create some custom components for each step of the pipeline. Next, you create a `register_model` component in order to register the PySpark model in Vertex AI Metadata. "
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "sSzUGhjq1SrR"
-      },
-      "source": [
+        "In order to pass job arguments, you create some custom components for each step of the pipeline.\n",
+        "\n",
         "#### Create component for passing args to preprocessing component\n",
         "\n",
         "The following component passes the args `--train-data-path` and `--out-process-path` in the required format for the preprocessing function defined earlier."
@@ -2084,9 +2078,7 @@
       "outputs": [],
       "source": [
         "@component(base_image=\"python:3.8-slim\")\n",
-        "def build_training_args(\n",
-        "    dataset_uri: Input[Artifact], train_path: str, model_path: str, metrics_path: str\n",
-        ") -> list:\n",
+        "def build_training_args(train_path: str, model_path: str, metrics_path: str) -> list:\n",
         "    return [\n",
         "        \"--train-path\",\n",
         "        train_path,\n",
@@ -2103,7 +2095,7 @@
         "id": "5VesQlnEm4qR"
       },
       "source": [
-        "#### Model Evaluation custom component\n",
+        "#### Create model evaluation custom component\n",
         "\n",
         "Define the component for processing the metrics for model evaluation. The `metrics_uri`, `metrics` and `plots` obtained as outputs from the model training component are further evaluated through this component."
       ]
@@ -2134,7 +2126,7 @@
         "\n",
         "    # Variables --------------------------------------------------------------------------------------------------------------------------\n",
         "    metrics_path = metrics_uri.replace(\"gs://\", \"/gcs/\")\n",
-        "    labels = [\"not eligiable\", \"eligiable\"]\n",
+        "    labels = [\"not eligible\", \"eligible\"]\n",
         "\n",
         "    # Helpers --------------------------------------------------------------------------------------------------------------------------\n",
         "    def calculate_roc(metrics, true, score):\n",
@@ -2205,7 +2197,6 @@
       "source": [
         "@component(base_image=\"python:3.8-slim\")\n",
         "def build_hpt_args(\n",
-        "    dataset_uri: Input[Artifact],\n",
         "    train_path: str,\n",
         "    model_path: str,\n",
         "    metrics_path: str,\n",
@@ -2256,7 +2247,7 @@
       "source": [
         "### Build the model serving container image\n",
         "\n",
-        "A *serving container image* is required to import your model into the Model Registry. The serving container image provides the model serving implementation for the model. The following replicates the instructions from [Serving Spark ML model using Vertex AI](https://cloud.google.com/architecture/spark-ml-model-with-vertexai) to build the serving container image.\n",
+        "A *serving container image* is required to import your model into the Model Registry. The serving container image provides the model serving implementation for the model. Learn more about [serving Spark ML models using Vertex AI](https://cloud.google.com/architecture/spark-ml-model-with-vertexai).\n",
         "\n",
         "**Note:** this step may take approximately 5 to 10 minutes to complete."
       ]
@@ -2331,7 +2322,9 @@
         "id": "0d83d9e80923"
       },
       "source": [
-        "The serving container requires the model schema in JSON format, which is read during container startup. See [Provide the model schema](https://cloud.google.com/architecture/spark-ml-model-with-vertexai#provide_the_model_schema) for more information.\n",
+        "### Define the schema for model serving\n",
+        "\n",
+        "The serving container requires the model schema in JSON format, which is read during container startup. Learn more about [providing the model schema](https://cloud.google.com/architecture/spark-ml-model-with-vertexai#provide_the_model_schema).\n",
         "\n",
         "Write the model schema file:"
       ]
@@ -2415,7 +2408,9 @@
         "id": "d0b88e26570a"
       },
       "source": [
-        "Copy the model schema configuration file to GCS. The serving container reads the model schema file location from the `AIP_STORAGE_URI` environment at startup. See [Import the model into Vertex AI](https://cloud.google.com/architecture/spark-ml-model-with-vertexai#import-the-model-into-vertex-ai) for more information."
+        "### Copy the model schema configuration file to GCS.\n",
+        "\n",
+        "The serving container reads the model schema file location from the `AIP_STORAGE_URI` environment at startup. See [Import the model into Vertex AI](https://cloud.google.com/architecture/spark-ml-model-with-vertexai#import-the-model-into-vertex-ai) for more information."
       ]
     },
     {
@@ -2435,9 +2430,9 @@
         "id": "wQd9M1_9bif7"
       },
       "source": [
-        "### Define your workflow using Kubeflow Pipelines\n",
+        "### Define your workflow as a Vertex AI Pipeline\n",
         "\n",
-        "Below, you use the Kubelflow Pipelines' DSL package to build your pipeline using the defined components and containers."
+        "Use the Kubeflow Pipelines SDK to define your workflow as a machine learning pipeline. The pipeline uses the custom components defined earlier, in addition to components from the `google-cloud-pipeline-components` package."
       ]
     },
     {
@@ -2453,8 +2448,6 @@
         "    preprocessing_main_python_file_uri: str = PREPROCESSING_PYTHON_FILE_URI,\n",
         "    train_data_path: str = FEATURES_TRAIN_URI,\n",
         "    preprocessed_data_path: str = PROCESSED_DATA_URI,\n",
-        "    dataset_name: str = DATASET_NAME,\n",
-        "    dataset_uri: str = GCS_PREPROCESSED_URI,\n",
         "    training_main_python_file_uri: str = TRAINING_PYTHON_FILE_URI,\n",
         "    train_path: str = PROCESSED_DATA_URI,\n",
         "    model_path: str = MODEL_URI,\n",
@@ -2475,8 +2468,6 @@
         "):\n",
         "    from google_cloud_pipeline_components.v1.dataproc import \\\n",
         "        DataprocPySparkBatchOp\n",
-        "    from google_cloud_pipeline_components.v1.dataset import \\\n",
-        "        TabularDatasetCreateOp\n",
         "    from google_cloud_pipeline_components.v1.endpoint import (EndpointCreateOp,\n",
         "                                                              ModelDeployOp)\n",
         "    from google_cloud_pipeline_components.v1.model import ModelUploadOp\n",
@@ -2496,21 +2487,12 @@
         "        subnetwork_uri=subnetwork_uri,\n",
         "    ).after(build_preprocessing_args_op)\n",
         "\n",
-        "    # create dataset\n",
-        "    create_dataset_op = TabularDatasetCreateOp(\n",
-        "        display_name=dataset_name,\n",
-        "        gcs_source=dataset_uri,\n",
-        "        project=project_id,\n",
-        "        location=location,\n",
-        "    ).after(data_preprocessing_op)\n",
-        "\n",
         "    # build training data args\n",
         "    build_training_args_op = build_training_args(\n",
-        "        dataset_uri=create_dataset_op.output,\n",
         "        train_path=train_path,\n",
         "        model_path=model_path,\n",
         "        metrics_path=metrics_path,\n",
-        "    ).after(create_dataset_op)\n",
+        "    ).after(data_preprocessing_op)\n",
         "\n",
         "    # training model\n",
         "    model_training_op = DataprocPySparkBatchOp(\n",
@@ -2533,7 +2515,6 @@
         "    ):\n",
         "\n",
         "        build_hpt_args_op = build_hpt_args(\n",
-        "            dataset_uri=create_dataset_op.output,\n",
         "            train_path=train_path,\n",
         "            model_path=hpt_model_path,\n",
         "            metrics_path=hpt_metrics_path,\n",
@@ -2618,7 +2599,9 @@
       "source": [
         "### Submit your pipeline run\n",
         "\n",
-        "Next, you use the Vertex AI Python SDK to submit and run your pipeline through Vertex AI Pipelines."
+        "Next, you use the Vertex AI Python SDK to submit and run your pipeline through Vertex AI Pipelines.\n",
+        "\n",
+        "The parameters, artifacts, and metrics produced from the pipeline run are automatically captured into Vertex AI Experiments as an experiment run."
       ]
     },
     {
@@ -2636,7 +2619,7 @@
         "    enable_caching=False,\n",
         ")\n",
         "\n",
-        "pipeline.submit(service_account=SERVICE_ACCOUNT)"
+        "pipeline.submit(service_account=SERVICE_ACCOUNT, experiment=EXPERIMENT_NAME)"
       ]
     },
     {
@@ -2661,6 +2644,33 @@
         "pipeline.wait()"
       ]
     },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "0574e2941aaf"
+      },
+      "source": [
+        "### (Optional) View experiment runs\n",
+        "\n",
+        "You can retrieve the parameters, artifacts, and metrics for all experiment runs as a pandas DataFrame. See [Compare and analyze runs](https://cloud.google.com/vertex-ai/docs/experiments/compare-analyze-runs) for more information on the topic."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "4f5ab528a98e"
+      },
+      "outputs": [],
+      "source": [
+        "experiment_df = vertex_ai.get_experiment_df(experiment=EXPERIMENT_NAME)\n",
+        "\n",
+        "# Show successfully completed experiment runs, sorted by F1 score\n",
+        "experiment_df.query('state == \"COMPLETE\"').sort_values(\n",
+        "    \"metric.Test_f1-score\", ascending=False\n",
+        ")"
+      ]
+    },
     {
       "cell_type": "markdown",
       "metadata": {
@@ -2671,15 +2681,10 @@
         "\n",
         "You can request online predictions if the model was deployed to a Vertex AI endpoint. Use the `google-cloud-aiplatform` client library to request predictions, or you can use `curl`.\n",
         "\n",
-        "For this model, the prediction response contains the predicted label (`0 == not eligible`, `1 == eligible`) for each prediction instance that is sent to the endpoint."
-      ]
-    },
-    {
-      "cell_type": "markdown",
-      "metadata": {
-        "id": "4c311f9fc363"
-      },
-      "source": [
+        "For this model, the prediction response contains the predicted label (`0 == not eligible`, `1 == eligible`) for each prediction instance that is sent to the endpoint.\n",
+        "\n",
+        "#### Use `google-cloud-aiplatform` to request online predictions\n",
+        "\n",
         "The following cell demonstrates how to use the `google-cloud-aiplatform` client library to request predictions from one or more instances."
       ]
     },
@@ -2706,6 +2711,8 @@
         "id": "1a2104c45e21"
       },
       "source": [
+        "#### Use `curl` to request online predictions\n",
+        "\n",
         "To use `curl`, first write the prediction instances to a file:"
       ]
     },
@@ -2784,12 +2791,7 @@
         "# Delete model\n",
         "model_list = vertex_ai.Model.list(filter=f'display_name=\"{MODEL_NAME}\"')\n",
         "for model in model_list:\n",
-        "    model.delete()\n",
-        "\n",
-        "# Delete dataset\n",
-        "dataset_list = vertex_ai.TabularDataset.list(filter=f'display_name=\"{DATASET_NAME}\"')\n",
-        "for dataset in dataset_list:\n",
-        "    dataset.delete()"
+        "    model.delete()"
       ]
     },
     {

notebooks/official/pipelines/google_cloud_pipeline_components_model_train_upload_deploy.ipynb

@@ -61,7 +61,9 @@
       "source": [
         "## Overview\n",
         "\n",
-        "This notebook shows how to use the components defined in [`google_cloud_pipeline_components`](https://github.com/kubeflow/pipelines/tree/master/components/google-cloud) to build a [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines) workflow that trains a [custom model](https://cloud.google.com/vertex-ai/docs/training/containers-overview), uploads the model as a `Model` resource, creates an `Endpoint` resource, and deploys the `Model` resource to the `Endpoint` resource."
+        "This notebook shows how to use the components defined in [`google_cloud_pipeline_components`](https://github.com/kubeflow/pipelines/tree/master/components/google-cloud) to build a [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines) workflow that trains a [custom model](https://cloud.google.com/vertex-ai/docs/training/containers-overview), uploads the model as a `Model` resource, creates an `Endpoint` resource, and deploys the `Model` resource to the `Endpoint` resource.\n",
+        "\n",
+        "Learn more about [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction) and [Vertex AI Training components](https://cloud.google.com/vertex-ai/docs/pipelines/customjob-component)."
       ]
     },
     {

notebooks/official/pipelines/google_cloud_pipeline_components_model_upload_predict_evaluate.ipynb

@@ -61,7 +61,9 @@
       "source": [
         "## Overview\n",
         "\n",
-        "This notebook shows how to use the components defined in [`google_cloud_pipeline_components`](https://github.com/kubeflow/pipelines/tree/master/components/google-cloud) in conjunction with an experimental `evaluation` method, to build a [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines) workflow that uploads a tabular custom model as a `Model` resource, creates a `BatchPredictionJob` resource, and evaluates the `Model` resource with the `BatchPredictionJob` results to create an evaluation `system.Metrics` artifact."
+        "This notebook shows how to use the components defined in [`google_cloud_pipeline_components`](https://github.com/kubeflow/pipelines/tree/master/components/google-cloud) in conjunction with an experimental `evaluation` method, to build a [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines) workflow that uploads a tabular custom model as a `Model` resource, creates a `BatchPredictionJob` resource, and evaluates the `Model` resource with the `BatchPredictionJob` results to create an evaluation `system.Metrics` artifact.\n",
+        "\n",
+        "Learn more about [Vertex AI Pipelines](https://cloud.google.com/vertex-ai/docs/pipelines/introduction) and [Vertex AI Model components](https://cloud.google.com/vertex-ai/docs/pipelines/model-endpoint-component)."
       ]
     },
     {