mirror of
https://github.com/karpathy/nanochat.git
synced 2025-12-27 06:32:22 +00:00
feat(dataset): make work share factor configurable with -f flag
This commit is contained in:
parent
ed07192724
commit
eecfdbf9f9
|
|
@ -135,7 +135,7 @@ if __name__ == "__main__":
|
||||||
type=int,
|
type=int,
|
||||||
default=8,
|
default=8,
|
||||||
help=(
|
help=(
|
||||||
"""Controls how each worker's share of shards is subdivided. CHUNK_SIZE is computed as len(ids_to_download) // (num_workers * work_share_factor), so it is the number of tasks a worker pulls per request from the main process. for example, for 240 shards and 4 workers the default value (8) produces 7 shards per request. setting it 1 gives a worker its entire share (~60 shards) in one go with minimal coordination but slow progress updates. larger work-share-factor values make the main process hand out smaller batches more often for faster feedback at a small scheduling cost."""
|
"""Controls how each worker's share of shards is subdivided. CHUNK_SIZE is computed as len(ids_to_download) // (num_workers * work_share_factor), so CHUNK_SIZE is the number of tasks a worker pulls per request from the main process. for example, for 240 shards and 4 workers the default value (8) produces 7 shards per request. setting it 1 gives a worker its entire share (~60 shards) in one go with minimal coordination but slow progress updates. larger work-share-factor values make the main process hand out smaller batches more often for faster feedback at a small scheduling cost."""
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue
Block a user