tacit/nanochat
1
0
Fork 0
mirror of https://github.com/karpathy/nanochat.git synced 2026-02-09 12:09:49 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

perf: remove redundant dtype conversion in apply_rotary_emb

Browse Source
This commit is contained in:
spjosyula 2025-12-24 01:56:45 +05:30
parent bc51da8bac
commit 69cc309967
1 changed files with 1 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
nanochat/gpt.py
View File

@ -44,9 +44,7 @@ def apply_rotary_emb(x, cos, sin):
x1, x2 = x[..., :d], x[..., d:] # split up last time into two halves
y1 = x1 * cos + x2 * sin # rotate pairs of dims
y2 = x1 * (-sin) + x2 * cos
out = torch.cat([y1, y2], 3) # re-assemble
out = out.to(x.dtype) # ensure input/output dtypes match
return out
return torch.cat([y1, y2], 3) # re-assemble
class CausalSelfAttention(nn.Module):
def __init__(self, config, layer_idx):