yum/FastTextPager

Compressed text paging over OSC.

git clone https://git.yummers.dev/yum/FastTextPager

yumcode bomb0c54e1f

master
branchesmasterall refs →
955 B28 linesraw
1import argparse
2import sentencepiece as spm
3
4def get_tokenizer():
5    model_path = "./custom_unigram_tokenizer_65k/unigram.model"
6    print(f"Loading SentencePiece tokenizer from: {model_path}")
7    sp = spm.SentencePieceProcessor()
8    sp.load(model_path)
9    print(f"Successfully loaded SentencePiece model. Vocab size: {sp.get_piece_size()}")
10    return sp
11
12def parse_args():
13    parser = argparse.ArgumentParser(description="Tokenize a given string using a SentencePiece model.")
14    parser.add_argument("text", type=str, help="The string to tokenize.")
15    args = parser.parse_args()
16    return args
17
18args = parse_args()
19tok = get_tokenizer()
20tokens = tok.encode_as_pieces(args.text)
21print("Tokens:", tokens)
22
23token_ids = tok.encode_as_ids(args.text)
24print("Token IDs:", token_ids)
25
26# Split each token ID into two 8-bit chunks (high byte, low byte)
27byte_pairs = [(tid >> 8, tid & 0xFF) for tid in token_ids]
28print("Token ID Byte Pairs:", byte_pairs)