From 0c54e1fc74fe7677a0d4fef1c147c6e886d182db Mon Sep 17 00:00:00 2001
From: yum <yum.food.vr@gmail.com>
Date: Sun, 11 May 2025 22:22:48 -0700
Subject: code bomb

---
 tokenize_me.py | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)
 create mode 100644 tokenize_me.py

(limited to 'tokenize_me.py')

diff --git a/tokenize_me.py b/tokenize_me.py
new file mode 100644
index 0000000..83be290
--- /dev/null
+++ b/tokenize_me.py
@@ -0,0 +1,28 @@
+import argparse
+import sentencepiece as spm
+
+def get_tokenizer():
+    model_path = "./custom_unigram_tokenizer_65k/unigram.model"
+    print(f"Loading SentencePiece tokenizer from: {model_path}")
+    sp = spm.SentencePieceProcessor()
+    sp.load(model_path)
+    print(f"Successfully loaded SentencePiece model. Vocab size: {sp.get_piece_size()}")
+    return sp
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="Tokenize a given string using a SentencePiece model.")
+    parser.add_argument("text", type=str, help="The string to tokenize.")
+    args = parser.parse_args()
+    return args
+
+args = parse_args()
+tok = get_tokenizer()
+tokens = tok.encode_as_pieces(args.text)
+print("Tokens:", tokens)
+
+token_ids = tok.encode_as_ids(args.text)
+print("Token IDs:", token_ids)
+
+# Split each token ID into two 8-bit chunks (high byte, low byte)
+byte_pairs = [(tid >> 8, tid & 0xFF) for tid in token_ids]
+print("Token ID Byte Pairs:", byte_pairs)
-- 
cgit v1.2.3