uploaded work done on vocab

vocab.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:04eaea28ee517ea93c73ae159633ec91ab3e5dc55aca69742e606ec155685bcc
+size 264

vocab.txt

@@ -0,0 +1,109 @@
+<unk>
+<bos>
+<eos>
+<pad>
+<msk>
+<spc>
+<tab>
+<nwl>
+A
+B
+C
+D
+E
+F
+G
+H
+I
+J
+K
+L
+M
+N
+O
+P
+Q
+R
+S
+T
+U
+V
+W
+X
+Y
+Z
+a
+b
+c
+d
+e
+f
+g
+h
+i
+j
+k
+l
+m
+n
+o
+p
+q
+r
+s
+t
+u
+v
+w
+x
+y
+z
+0
+1
+2
+3
+4
+5
+6
+7
+8
+9
+!
+"
+#
+$
+%
+&
+'
+(
+)
+*
++
+,
+-
+.
+/
+:
+;
+<
+=
+>
+?
+@
+[
+\
+]
+^
+_
+`
+{
+|
+}
+~
+—
+–
+‘
+’
+“
+”
+…

vocab_to_bin.py

@@ -0,0 +1,53 @@
+"""vocab.txt -> vocab.bin  (strict, minimal)
+
+Format: u8 length + token bytes, one entry per line, id = entry order.
+Refuses to run if the file looks wrong instead of silently corrupting.
+"""
+import sys
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent
+SRC = HERE / "vocab.txt"
+DST = HERE / "vocab.bin"
+SPECIALS = ["<unk>", "<bos>", "<eos>", "<pad>", "<msk>", "<spc>", "<tab>", "<nwl>"]
+EXPECTED_COUNT = 109  # 8 specials + 94 ascii + 7 smart punct
+
+
+def main():
+    data = SRC.read_bytes()
+    if b"\r" in data:
+        sys.exit("refusing: CR present (CRLF line endings?) — fix the file first")
+    lines = data.split(b"\n")
+    if lines and lines[-1] == b"":
+        lines.pop()
+    if len(lines) != EXPECTED_COUNT:
+        sys.exit(f"refusing: {len(lines)} tokens, expected {EXPECTED_COUNT}")
+    for i, tok in enumerate(lines):
+        if not tok:
+            sys.exit(f"refusing: empty token at line {i + 1}")
+        if len(tok) > 255:
+            sys.exit(f"refusing: token {i + 1} longer than 255 bytes")
+    for i, want in enumerate(SPECIALS):
+        if lines[i] != want.encode():
+            sys.exit(f"refusing: specials block wrong at id {i}: {lines[i]!r} != {want!r}")
+    out = bytearray()
+    for tok in lines:
+        out.append(len(tok))
+        out.extend(tok)
+    DST.write_bytes(bytes(out))
+    print(f"wrote {DST.name}: {len(out)} bytes, {len(lines)} tokens "
+          f"({sum(len(t) for t in lines)} token bytes + {len(lines)} length bytes)")
+
+    # round-trip check
+    back = bytearray()
+    p = 0
+    while p < len(out):
+        n = out[p]
+        back.extend(out[p + 1:p + 1 + n])
+        p += 1 + n
+    assert back == b"\n".join(lines) + (b"\n" if lines else b""), "round-trip failed"
+    print("round-trip: OK")
+
+
+if __name__ == "__main__":
+    main()