From d4db00366119430fad124a59ff1f31ec855ca6ee Mon Sep 17 00:00:00 2001 From: Dylan Chen Date: Sun, 1 Feb 2026 23:03:47 +0800 Subject: [PATCH] fix: cap SFT rendered conversation length --- scripts/chat_sft.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/chat_sft.py b/scripts/chat_sft.py index 91300b6..fb0c061 100644 --- a/scripts/chat_sft.py +++ b/scripts/chat_sft.py @@ -152,7 +152,7 @@ def sft_data_generator_bos_bestfit(split, buffer_size=100): nonlocal cursor, epoch while len(conv_buffer) < buffer_size: conversation = dataset[cursor] - ids, _ = tokenizer.render_conversation(conversation) + ids, _ = tokenizer.render_conversation(conversation, max_tokens=row_capacity) conv_buffer.append(ids) cursor += ddp_world_size if cursor >= dataset_size: