xiaohei/model_garden
1
0
Fork 0
mirror of https://github.com/GoogleCloudPlatform/vertex-ai-samples.git synced 2026-09-26 14:42:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

No public description

Browse Source 
MG_DOCKER_CODES_PIPER_ORIGIN_REV_ID: 878215121
This commit is contained in:
Rayan Dasoriya
2026-03-03 18:26:58 -08:00
committed by Copybara-Service
parent 062835174e
commit 1f9e93993c
2 changed files with 18 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files
notebooks/community/model_garden/docker_source_codes/model_oss/notebook_util/dataset_validation_util.py
+3 -23
View File
@@ -8,7 +8,6 @@ import json
import multiprocessing
import os
import subprocess
import sys
from typing import Any, Union
from absl import logging
import accelerate
@@ -552,10 +551,9 @@ def drop_long_sequences(
input_column: str,
max_sequence_length: int,
tokenizer: transformers.PreTrainedTokenizer,
dataset_dropped_threshold: float,
is_train: bool,
) -> tuple[Any, Any, int]:
"""Returns the dataset by removing examples that are longer than max_seq_length.
"""Drops examples longer than max_seq_length from the dataset.
Args:
dataset: The dataset to filter.
@@ -563,39 +561,21 @@ def drop_long_sequences(
input_column: The input column in the dataset to be used.
max_sequence_length: The maximum sequence length.
tokenizer: The tokenizer.
dataset_dropped_threshold: The threshold for the number of samples dropped
from the dataset.
is_train: Whether the dataset is for training.
Returns:
A tuple of (filtered_dataset, filtered_dataset_with_template,
dropped_samples).
"""
context_name = f"the {'train' if is_train else 'eval'} dataset"
indices_to_keep, original_length, dropped_samples = (
_get_indices_for_valid_length(
indices_to_keep, _, dropped_samples = _get_indices_for_valid_length(
dataset_with_template,
input_column,
max_sequence_length,
tokenizer,
context_name,
)
)
if (
original_length > 0
and dropped_samples / original_length * 100 > dataset_dropped_threshold
):
logging.error(
"More than %f%% of the samples were dropped from {%s} after"
" filtering for max_sequence_length=%d. Please check your dataset.",
dataset_dropped_threshold,
context_name,
max_sequence_length,
)
# handling library when available.
sys.exit(1)
filtered_dataset = dataset.select(indices_to_keep)
filtered_dataset_with_template = dataset_with_template.select(indices_to_keep)
notebooks/community/model_garden/docker_source_codes/notebook_util/dataset_validation_util.py
+3 -23
View File
@@ -8,7 +8,6 @@ import json
import multiprocessing
import os
import subprocess
import sys
from typing import Any, Union
from absl import logging
import accelerate
@@ -552,10 +551,9 @@ def drop_long_sequences(
input_column: str,
max_sequence_length: int,
tokenizer: transformers.PreTrainedTokenizer,
dataset_dropped_threshold: float,
is_train: bool,
) -> tuple[Any, Any, int]:
"""Returns the dataset by removing examples that are longer than max_seq_length.
"""Drops examples longer than max_seq_length from the dataset.
Args:
dataset: The dataset to filter.
@@ -563,39 +561,21 @@ def drop_long_sequences(
input_column: The input column in the dataset to be used.
max_sequence_length: The maximum sequence length.
tokenizer: The tokenizer.
dataset_dropped_threshold: The threshold for the number of samples dropped
from the dataset.
is_train: Whether the dataset is for training.
Returns:
A tuple of (filtered_dataset, filtered_dataset_with_template,
dropped_samples).
"""
context_name = f"the {'train' if is_train else 'eval'} dataset"
indices_to_keep, original_length, dropped_samples = (
_get_indices_for_valid_length(
indices_to_keep, _, dropped_samples = _get_indices_for_valid_length(
dataset_with_template,
input_column,
max_sequence_length,
tokenizer,
context_name,
)
)
if (
original_length > 0
and dropped_samples / original_length * 100 > dataset_dropped_threshold
):
logging.error(
"More than %f%% of the samples were dropped from {%s} after"
" filtering for max_sequence_length=%d. Please check your dataset.",
dataset_dropped_threshold,
context_name,
max_sequence_length,
)
# handling library when available.
sys.exit(1)
filtered_dataset = dataset.select(indices_to_keep)
filtered_dataset_with_template = dataset_with_template.select(indices_to_keep)