ensure consistency of quotes within each statement

This commit is contained in:
svlandeg 2025-11-03 21:52:02 +01:00
parent e22fc6f2fa
commit 2ce62ec076
5 changed files with 12 additions and 12 deletions

View File

@ -37,7 +37,7 @@ from concurrent.futures import ThreadPoolExecutor, as_completed
from nanochat.common import get_base_dir from nanochat.common import get_base_dir
api_key = open("openroutertoken.txt", 'r', encoding='utf-8').read().strip() api_key = open("openroutertoken.txt", "r", encoding="utf-8").read().strip()
url = "https://openrouter.ai/api/v1/chat/completions" url = "https://openrouter.ai/api/v1/chat/completions"
headers = { headers = {
@ -45,7 +45,7 @@ headers = {
"Content-Type": "application/json" "Content-Type": "application/json"
} }
readme = open("README.md", 'r', encoding='utf-8').read().strip() readme = open("README.md", "r", encoding="utf-8").read().strip()
prompt = r""" prompt = r"""
I want to generate synthetic data for an LLM to teach it about its identity. Here is the identity I want: I want to generate synthetic data for an LLM to teach it about its identity. Here is the identity I want:

View File

@ -34,7 +34,7 @@ def save_checkpoint(checkpoint_dir, step, model_data, optimizer_data, meta_data)
log0(f"Saved optimizer file to: {optimizer_path}") log0(f"Saved optimizer file to: {optimizer_path}")
# Save the metadata dict as json # Save the metadata dict as json
meta_path = os.path.join(checkpoint_dir, f"meta_{step:06d}.json") meta_path = os.path.join(checkpoint_dir, f"meta_{step:06d}.json")
with open(meta_path, "w", encoding='utf-8') as f: with open(meta_path, "w", encoding="utf-8") as f:
json.dump(meta_data, f, indent=2) json.dump(meta_data, f, indent=2)
log0(f"Saved metadata file to: {meta_path}") log0(f"Saved metadata file to: {meta_path}")
@ -50,7 +50,7 @@ def load_checkpoint(checkpoint_dir, step, device, load_optimizer=False):
optimizer_data = torch.load(optimizer_path, map_location=device) optimizer_data = torch.load(optimizer_path, map_location=device)
# Load the metadata # Load the metadata
meta_path = os.path.join(checkpoint_dir, f"meta_{step:06d}.json") meta_path = os.path.join(checkpoint_dir, f"meta_{step:06d}.json")
with open(meta_path, "r", encoding='utf-8') as f: with open(meta_path, "r", encoding="utf-8") as f:
meta_data = json.load(f) meta_data = json.load(f)
return model_data, optimizer_data, meta_data return model_data, optimizer_data, meta_data

View File

@ -241,7 +241,7 @@ class Report:
slug = slugify(section) slug = slugify(section)
file_name = f"{slug}.md" file_name = f"{slug}.md"
file_path = os.path.join(self.report_dir, file_name) file_path = os.path.join(self.report_dir, file_name)
with open(file_path, "w", encoding='utf-8') as f: with open(file_path, "w", encoding="utf-8") as f:
f.write(f"## {section}\n") f.write(f"## {section}\n")
f.write(f"timestamp: {datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n\n") f.write(f"timestamp: {datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n\n")
for item in data: for item in data:
@ -272,11 +272,11 @@ class Report:
final_metrics = {} # the most important final metrics we'll add as table at the end final_metrics = {} # the most important final metrics we'll add as table at the end
start_time = None start_time = None
end_time = None end_time = None
with open(report_file, "w", encoding='utf-8') as out_file: with open(report_file, "w", encoding="utf-8") as out_file:
# write the header first # write the header first
header_file = os.path.join(report_dir, "header.md") header_file = os.path.join(report_dir, "header.md")
if os.path.exists(header_file): if os.path.exists(header_file):
with open(header_file, "r", encoding='utf-8') as f: with open(header_file, "r", encoding="utf-8") as f:
header_content = f.read() header_content = f.read()
out_file.write(header_content) out_file.write(header_content)
start_time = extract_timestamp(header_content, "Run started:") start_time = extract_timestamp(header_content, "Run started:")
@ -293,7 +293,7 @@ class Report:
if not os.path.exists(section_file): if not os.path.exists(section_file):
print(f"Warning: {section_file} does not exist, skipping") print(f"Warning: {section_file} does not exist, skipping")
continue continue
with open(section_file, "r", encoding='utf-8') as in_file: with open(section_file, "r", encoding="utf-8") as in_file:
section = in_file.read() section = in_file.read()
# Extract timestamp from this section (the last section's timestamp will "stick" as end_time) # Extract timestamp from this section (the last section's timestamp will "stick" as end_time)
if "rl" not in file_name: if "rl" not in file_name:
@ -373,7 +373,7 @@ class Report:
header_file = os.path.join(self.report_dir, "header.md") header_file = os.path.join(self.report_dir, "header.md")
header = generate_header() header = generate_header()
start_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") start_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
with open(header_file, "w", encoding='utf-8') as f: with open(header_file, "w", encoding="utf-8") as f:
f.write(header) f.write(header)
f.write(f"Run started: {start_time}\n\n---\n\n") f.write(f"Run started: {start_time}\n\n---\n\n")
print(f"Reset report and wrote header to {header_file}") print(f"Reset report and wrote header to {header_file}")

View File

@ -243,7 +243,7 @@ app.add_middleware(
async def root(): async def root():
"""Serve the chat UI.""" """Serve the chat UI."""
ui_html_path = os.path.join("nanochat", "ui.html") ui_html_path = os.path.join("nanochat", "ui.html")
with open(ui_html_path, "r", encoding='utf-8') as f: with open(ui_html_path, "r", encoding="utf-8") as f:
html_content = f.read() html_content = f.read()
# Replace the API_URL to use the same origin # Replace the API_URL to use the same origin
html_content = html_content.replace( html_content = html_content.replace(

View File

@ -455,13 +455,13 @@ def enwik8_path():
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def enwik8_small(enwik8_path): def enwik8_small(enwik8_path):
"""Fixture providing 100KB of enwik8 for quick tests.""" """Fixture providing 100KB of enwik8 for quick tests."""
with open(enwik8_path, "r", encoding='utf-8') as f: with open(enwik8_path, "r", encoding="utf-8") as f:
return f.read(100_000) return f.read(100_000)
@pytest.fixture(scope="module") @pytest.fixture(scope="module")
def enwik8_large(enwik8_path): def enwik8_large(enwik8_path):
"""Fixture providing 10MB of enwik8 for performance tests.""" """Fixture providing 10MB of enwik8 for performance tests."""
with open(enwik8_path, "r", encoding='utf-8') as f: with open(enwik8_path, "r", encoding="utf-8") as f:
return f.read(10**7) return f.read(10**7)
def time_function(func, *args, **kwargs): def time_function(func, *args, **kwargs):