From 6219bef8245d76d1b5e306fd55ef2c972a446248 Mon Sep 17 00:00:00 2001 From: Andrew Chen <48723787+chuenchen309@users.noreply.github.com> Date: Thu, 16 Jul 2026 21:39:39 +0800 Subject: [PATCH 1/4] fix(dataprep): don't emit a degenerate chunk for empty text MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit smart_chunk_text feeds empty / whitespace-only text (which tokenizes to zero tokens) into the single-chunk branch, which unconditionally returns one chunk. That yields a lone-EOS "document" (input_ids=[eos]) or, when the tokenizer has no eos_token_id, a zero-length input_ids=[] — an invalid sample that breaks a downstream collator/trainer. load_from_file already guards against this with a ValueError, but chunk_text, smart_chunk_text and load_from_files do not, so batch-loading a directory that contains an empty file silently injects garbage rows. Return no chunks when the tokenized text is empty, so empty inputs contribute nothing instead of a degenerate sample. load_from_file keeps its explicit ValueError (its guard runs first). Co-Authored-By: Claude Opus 4.8 (1M context) --- tests/test_raw_text.py | 38 ++++++++++++++++++++++++++++++++++++ unsloth/dataprep/raw_text.py | 7 +++++++ 2 files changed, 45 insertions(+) diff --git a/tests/test_raw_text.py b/tests/test_raw_text.py index 18549adfe87..7be9d1200c0 100644 --- a/tests/test_raw_text.py +++ b/tests/test_raw_text.py @@ -312,8 +312,46 @@ def test_load_from_file_skips_non_object_json_lines(): return True +def test_smart_chunk_text_empty_input_returns_no_chunks(): + """Empty / whitespace-only text must yield no chunks, not a degenerate + sample. `load_from_file` already rejects such input with a ValueError, but + `chunk_text` / `smart_chunk_text` / `load_from_files` fed the single-chunk + branch an empty token list, emitting a lone-EOS 'document' (or a zero-length + input_ids when there is no eos_token_id) that would break a trainer.""" + + class MockTokenizer: + def __init__(self, eos_token_id): + self.eos_token = "" if eos_token_id is not None else None + self.eos_token_id = eos_token_id + + def __call__(self, text, return_tensors = None, add_special_tokens = False,): + token_ids = list(range(len(text.split()))) + if return_tensors == "pt": + return {"input_ids": [token_ids]} + return {"input_ids": token_ids} + + def decode(self, token_ids, skip_special_tokens = False,): + return " ".join(f"word_{i}" for i in token_ids) + + for eos_token_id in (2, None): + loader = RawTextDataLoader( + MockTokenizer(eos_token_id), chunk_size = 2048, stride = 512 + ) + for text in ("", " \n\t "): + assert loader.smart_chunk_text( + text, chunk_size = 2048, stride = 512, return_tokenized = True + ) == [], f"empty input should yield no chunks (eos={eos_token_id}, text={text!r})" + assert loader.chunk_text(text) == [], ( + f"chunk_text should yield no chunks for empty input " + f"(eos={eos_token_id}, text={text!r})" + ) + print("✅ test_smart_chunk_text_empty_input_returns_no_chunks passed!") + return True + + if __name__ == "__main__": success = test_raw_text_loader() success = test_smart_chunk_text_single_chunk_no_eos_returns_plain_list() and success success = test_load_from_file_skips_non_object_json_lines() and success + success = test_smart_chunk_text_empty_input_returns_no_chunks() and success sys.exit(0 if success else 1) diff --git a/unsloth/dataprep/raw_text.py b/unsloth/dataprep/raw_text.py index 8623285a25a..8d1b19d2ea4 100644 --- a/unsloth/dataprep/raw_text.py +++ b/unsloth/dataprep/raw_text.py @@ -151,6 +151,13 @@ def smart_chunk_text( # Tokenizer returned a count; build a range tokens = list(range(tokens)) + if len(tokens) == 0: + # Empty / whitespace-only text tokenizes to nothing. Emit no chunks + # rather than a degenerate sample (a lone-EOS "document", or a + # zero-length ``input_ids`` when the tokenizer has no eos_token_id, + # which would break a downstream collator/trainer). + return [] + if len(tokens) <= chunk_size: # Fits in a single chunk if return_tokenized: From d1c7dfb3b66230fa07a034790e20bea2dcafcb95 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 16 Jul 2026 13:41:14 +0000 Subject: [PATCH 2/4] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_raw_text.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/tests/test_raw_text.py b/tests/test_raw_text.py index 7be9d1200c0..6c9b5e1a3cc 100644 --- a/tests/test_raw_text.py +++ b/tests/test_raw_text.py @@ -324,23 +324,31 @@ def __init__(self, eos_token_id): self.eos_token = "" if eos_token_id is not None else None self.eos_token_id = eos_token_id - def __call__(self, text, return_tensors = None, add_special_tokens = False,): + def __call__( + self, + text, + return_tensors = None, + add_special_tokens = False, + ): token_ids = list(range(len(text.split()))) if return_tensors == "pt": return {"input_ids": [token_ids]} return {"input_ids": token_ids} - def decode(self, token_ids, skip_special_tokens = False,): + def decode( + self, + token_ids, + skip_special_tokens = False, + ): return " ".join(f"word_{i}" for i in token_ids) for eos_token_id in (2, None): - loader = RawTextDataLoader( - MockTokenizer(eos_token_id), chunk_size = 2048, stride = 512 - ) + loader = RawTextDataLoader(MockTokenizer(eos_token_id), chunk_size = 2048, stride = 512) for text in ("", " \n\t "): - assert loader.smart_chunk_text( - text, chunk_size = 2048, stride = 512, return_tokenized = True - ) == [], f"empty input should yield no chunks (eos={eos_token_id}, text={text!r})" + assert ( + loader.smart_chunk_text(text, chunk_size = 2048, stride = 512, return_tokenized = True) + == [] + ), f"empty input should yield no chunks (eos={eos_token_id}, text={text!r})" assert loader.chunk_text(text) == [], ( f"chunk_text should yield no chunks for empty input " f"(eos={eos_token_id}, text={text!r})" From 2924cffe281e7d064878587221c11472c3830448 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Wed, 22 Jul 2026 13:45:58 +0000 Subject: [PATCH 3/4] Guard empty/whitespace text before tokenizing in raw_text Real BPE/SentencePiece tokenizers emit tokens for spaces and newlines, so the len(tokens)==0 check let whitespace-only documents through as a degenerate lone-EOS sample. Guard on text.strip() before tokenizing (mirroring load_from_file), and raise in load_from_files when every file is empty so return_tokenized mode never falls back to a text-column dataset. Test now uses a whitespace-preserving tokenizer and covers both return_tokenized modes. --- tests/test_raw_text.py | 85 ++++++++++++++++++++++++------------ unsloth/dataprep/raw_text.py | 17 +++++--- 2 files changed, 66 insertions(+), 36 deletions(-) diff --git a/tests/test_raw_text.py b/tests/test_raw_text.py index 6c9b5e1a3cc..09c858f0dea 100644 --- a/tests/test_raw_text.py +++ b/tests/test_raw_text.py @@ -313,47 +313,73 @@ def test_load_from_file_skips_non_object_json_lines(): def test_smart_chunk_text_empty_input_returns_no_chunks(): - """Empty / whitespace-only text must yield no chunks, not a degenerate - sample. `load_from_file` already rejects such input with a ValueError, but - `chunk_text` / `smart_chunk_text` / `load_from_files` fed the single-chunk - branch an empty token list, emitting a lone-EOS 'document' (or a zero-length - input_ids when there is no eos_token_id) that would break a trainer.""" + """Empty/whitespace text must yield no chunks. This tokenizer keeps one token + per char (like BPE/SentencePiece keeping spaces), so a len(tokens)==0 check + would miss whitespace; the fix guards on text.strip() before tokenizing.""" - class MockTokenizer: + class WhitespacePreservingTokenizer: def __init__(self, eos_token_id): self.eos_token = "" if eos_token_id is not None else None self.eos_token_id = eos_token_id - def __call__( - self, - text, - return_tensors = None, - add_special_tokens = False, - ): - token_ids = list(range(len(text.split()))) + def __call__(self, text, return_tensors = None, add_special_tokens = False): + token_ids = [ord(c) % 100 for c in text] # whitespace -> real tokens if return_tensors == "pt": return {"input_ids": [token_ids]} return {"input_ids": token_ids} - def decode( - self, - token_ids, - skip_special_tokens = False, - ): - return " ".join(f"word_{i}" for i in token_ids) + def decode(self, token_ids, skip_special_tokens = False): + return "".join(chr(32 + (t % 90)) for t in token_ids) for eos_token_id in (2, None): - loader = RawTextDataLoader(MockTokenizer(eos_token_id), chunk_size = 2048, stride = 512) + loader = RawTextDataLoader(WhitespacePreservingTokenizer(eos_token_id), chunk_size = 2048, stride = 512) + # Whitespace tokenizes to >0 tokens, so [] proves the pre-tokenize guard. + assert len(loader.tokenizer(" \n\t ")["input_ids"]) > 0 for text in ("", " \n\t "): - assert ( - loader.smart_chunk_text(text, chunk_size = 2048, stride = 512, return_tokenized = True) - == [] - ), f"empty input should yield no chunks (eos={eos_token_id}, text={text!r})" - assert loader.chunk_text(text) == [], ( - f"chunk_text should yield no chunks for empty input " - f"(eos={eos_token_id}, text={text!r})" - ) - print("✅ test_smart_chunk_text_empty_input_returns_no_chunks passed!") + for return_tokenized in (True, False): + assert loader.smart_chunk_text( + text, chunk_size = 2048, stride = 512, return_tokenized = return_tokenized + ) == [], f"no chunks for empty input (eos={eos_token_id}, text={text!r}, tokenized={return_tokenized})" + assert loader.chunk_text(text, return_tokenized = return_tokenized) == [], ( + f"chunk_text: no chunks for empty input " + f"(eos={eos_token_id}, text={text!r}, tokenized={return_tokenized})" + ) + print("test_smart_chunk_text_empty_input_returns_no_chunks passed") + return True + + +def test_load_from_files_all_empty_raises(): + """All-empty file list must raise (like load_from_file) instead of returning + a 0-row text-column dataset in return_tokenized mode.""" + + class WhitespacePreservingTokenizer: + eos_token = "" + eos_token_id = 2 + + def __call__(self, text, return_tensors = None, add_special_tokens = False): + token_ids = [ord(c) % 100 for c in text] + if return_tensors == "pt": + return {"input_ids": [token_ids]} + return {"input_ids": token_ids} + + loader = RawTextDataLoader(WhitespacePreservingTokenizer(), chunk_size = 2048, stride = 512) + paths = [] + try: + for content in ("", " \n\t "): + with tempfile.NamedTemporaryFile("w", suffix = ".txt", delete = False) as f: + f.write(content) + paths.append(f.name) + raised = False + try: + loader.load_from_files(paths, return_tokenized = True) + except ValueError as e: + raised = True + assert "empty" in str(e).lower() or "whitespace" in str(e).lower(), str(e) + assert raised, "load_from_files must raise when all files are empty/whitespace" + finally: + for p in paths: + os.unlink(p) + print("test_load_from_files_all_empty_raises passed") return True @@ -362,4 +388,5 @@ def decode( success = test_smart_chunk_text_single_chunk_no_eos_returns_plain_list() and success success = test_load_from_file_skips_non_object_json_lines() and success success = test_smart_chunk_text_empty_input_returns_no_chunks() and success + success = test_load_from_files_all_empty_raises() and success sys.exit(0 if success else 1) diff --git a/unsloth/dataprep/raw_text.py b/unsloth/dataprep/raw_text.py index 8d1b19d2ea4..fdaba181f11 100644 --- a/unsloth/dataprep/raw_text.py +++ b/unsloth/dataprep/raw_text.py @@ -87,6 +87,10 @@ def load_from_files( text_content, self.chunk_size, self.stride, return_tokenized ) all_chunks.extend(chunks) + if not all_chunks: + # All files empty/whitespace: raise like load_from_file instead of + # create_causal_dataset([]) returning a 0-row text-column dataset. + raise ValueError("All files are empty or contain only whitespace") return self.create_causal_dataset(all_chunks) def chunk_text( @@ -139,6 +143,12 @@ def smart_chunk_text( f"stride ({stride}) must be smaller than chunk_size ({chunk_size}) to progress the chunking loop" ) + # Skip empty/whitespace text before tokenizing: BPE/SentencePiece emit + # real tokens for spaces/newlines, so a len(tokens)==0 check misses it + # and would yield a degenerate lone-EOS sample. Mirrors load_from_file. + if not text or not text.strip(): + return [] + # Tokenize the whole text once for accurate token counts tokenized = self.tokenizer(text, return_tensors = "pt", add_special_tokens = False) tokens = tokenized["input_ids"] @@ -151,13 +161,6 @@ def smart_chunk_text( # Tokenizer returned a count; build a range tokens = list(range(tokens)) - if len(tokens) == 0: - # Empty / whitespace-only text tokenizes to nothing. Emit no chunks - # rather than a degenerate sample (a lone-EOS "document", or a - # zero-length ``input_ids`` when the tokenizer has no eos_token_id, - # which would break a downstream collator/trainer). - return [] - if len(tokens) <= chunk_size: # Fits in a single chunk if return_tokenized: From 50356e6394a6690bf2fb80c68c00f7406b354a4e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 22 Jul 2026 13:46:19 +0000 Subject: [PATCH 4/4] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_raw_text.py | 33 ++++++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/tests/test_raw_text.py b/tests/test_raw_text.py index 09c858f0dea..bc75c2e44dd 100644 --- a/tests/test_raw_text.py +++ b/tests/test_raw_text.py @@ -322,24 +322,38 @@ def __init__(self, eos_token_id): self.eos_token = "" if eos_token_id is not None else None self.eos_token_id = eos_token_id - def __call__(self, text, return_tensors = None, add_special_tokens = False): + def __call__( + self, + text, + return_tensors = None, + add_special_tokens = False, + ): token_ids = [ord(c) % 100 for c in text] # whitespace -> real tokens if return_tensors == "pt": return {"input_ids": [token_ids]} return {"input_ids": token_ids} - def decode(self, token_ids, skip_special_tokens = False): + def decode( + self, + token_ids, + skip_special_tokens = False, + ): return "".join(chr(32 + (t % 90)) for t in token_ids) for eos_token_id in (2, None): - loader = RawTextDataLoader(WhitespacePreservingTokenizer(eos_token_id), chunk_size = 2048, stride = 512) + loader = RawTextDataLoader( + WhitespacePreservingTokenizer(eos_token_id), chunk_size = 2048, stride = 512 + ) # Whitespace tokenizes to >0 tokens, so [] proves the pre-tokenize guard. assert len(loader.tokenizer(" \n\t ")["input_ids"]) > 0 for text in ("", " \n\t "): for return_tokenized in (True, False): - assert loader.smart_chunk_text( - text, chunk_size = 2048, stride = 512, return_tokenized = return_tokenized - ) == [], f"no chunks for empty input (eos={eos_token_id}, text={text!r}, tokenized={return_tokenized})" + assert ( + loader.smart_chunk_text( + text, chunk_size = 2048, stride = 512, return_tokenized = return_tokenized + ) + == [] + ), f"no chunks for empty input (eos={eos_token_id}, text={text!r}, tokenized={return_tokenized})" assert loader.chunk_text(text, return_tokenized = return_tokenized) == [], ( f"chunk_text: no chunks for empty input " f"(eos={eos_token_id}, text={text!r}, tokenized={return_tokenized})" @@ -356,7 +370,12 @@ class WhitespacePreservingTokenizer: eos_token = "" eos_token_id = 2 - def __call__(self, text, return_tensors = None, add_special_tokens = False): + def __call__( + self, + text, + return_tensors = None, + add_special_tokens = False, + ): token_ids = [ord(c) % 100 for c in text] if return_tensors == "pt": return {"input_ids": [token_ids]}