Merge a33d04dca1 into 4a87a0d19f

tasks/common.py

@@ -6,6 +6,9 @@ Example tasks: MMLU, ARC-Easy, ARC-Challenge, GSM8K, HumanEval, SmolTalk.
 """
 
 import random
+import logging
+
+logger = logging.getLogger(__name__)
 
 class Task:
     """
@@ -37,10 +40,12 @@ class Task:
         if self.stop is not None:
             num_ex = self.num_examples()
             if self.stop > num_ex:
-                raise ValueError(
+                # Warn once, then cap stop
+                logger.warning(
                     f"Stop parameter ({self.stop}) exceeds dataset size ({num_ex}). "
-                    f"Please use stop <= {num_ex} or remove the stop parameter to use the full dataset."
+                    f"Using {num_ex} examples instead."
                 )
+                self.stop = num_ex
             stop = self.stop
         else:
             stop = self.num_examples()