tacit/nanochat
1
0
Fork 0
mirror of https://github.com/karpathy/nanochat.git synced 2025-12-06 04:12:13 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge 02b22a5a13 into 4a87a0d19f

Browse Source
This commit is contained in:
Anton Chechetka 2025-11-23 17:57:17 +01:00 committed by GitHub
commit 490d517d89
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 4 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
scripts/tok_eval.py
View File

@ -212,9 +212,9 @@ def print_comparison(baseline_name, baseline_results, ours_results, all_text):
baseline_data = baseline_results[name] baseline_data = baseline_results[name]
ours_data = ours_results[name] ours_data = ours_results[name]
# Calculate relative difference (positive means ours is better, negative means worse) # Calculate relative difference in total tokens (negative means ours is better, positive means worse)
# Using tokens: fewer tokens is better, so we calculate (baseline_tokens - ours_tokens) / baseline_tokens # We calculate (ours_tokens - baseline_tokens) / baseline_tokens
relative_diff = ((baseline_data['tokens'] - ours_data['tokens']) / baseline_data['tokens']) * 100 relative_diff = ((ours_data['tokens'] - baseline_data['tokens']) / baseline_data['tokens']) * 100
# Determine which has better compression (higher ratio = better) # Determine which has better compression (higher ratio = better)
if baseline_data['ratio'] > ours_data['ratio']: if baseline_data['ratio'] > ours_data['ratio']:
@ -256,7 +256,7 @@ for baseline_name in ["GPT-2", "GPT-4"]:
for name, text in all_text: for name, text in all_text:
baseline_data = baseline_results[name] baseline_data = baseline_results[name]
ours_data = ours_results[name] ours_data = ours_results[name]
relative_diff = ((baseline_data['tokens'] - ours_data['tokens']) / baseline_data['tokens']) * 100 relative_diff = ((ours_data['tokens'] - baseline_data['tokens']) / baseline_data['tokens']) * 100
lines.append(f"| {name} | {baseline_data['bytes']} | {baseline_data['tokens']} | {baseline_data['ratio']:.2f} | {ours_data['tokens']} | {ours_data['ratio']:.2f} | {relative_diff:+.1f}% |") lines.append(f"| {name} | {baseline_data['bytes']} | {baseline_data['tokens']} | {baseline_data['ratio']:.2f} | {ours_data['tokens']} | {ours_data['ratio']:.2f} | {relative_diff:+.1f}% |")
lines.append("") lines.append("")
report_markdown = "\n".join(lines) report_markdown = "\n".join(lines)