From 5a3d8b6b5edbb98062f43a198bd27f917bce064e Mon Sep 17 00:00:00 2001 From: Jason Kneen Date: Wed, 22 Oct 2025 02:37:32 +0100 Subject: [PATCH] Update nanochat/gpt.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- nanochat/gpt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nanochat/gpt.py b/nanochat/gpt.py index 6f97ece..eeed724 100644 --- a/nanochat/gpt.py +++ b/nanochat/gpt.py @@ -41,7 +41,7 @@ def norm(x): else: # Manual RMS norm implementation variance = x.pow(2).mean(-1, keepdim=True) - return x * torch.rsqrt(variance + 1e-5) + return x * torch.rsqrt(variance + 1e-6) def apply_rotary_emb(x, cos, sin):