| { | |
| "block": 1024, | |
| "ffn": "swiglu", | |
| "ffn_mult": 2.667, | |
| "logit_cap": 0.0, | |
| "n_embd": 384, | |
| "n_head": 6, | |
| "n_layer": 15, | |
| "norm": "rmsnorm", | |
| "norm_eps": 1e-06, | |
| "pos": "rope", | |
| "qk_norm": true, | |
| "rope_theta": 100000.0, | |
| "tied_weights": { | |
| "tok.weight": "head.weight" | |
| }, | |
| "value_residual": true, | |
| "vocab": 32000 | |
| } | |