yum/FastTextPager

Compressed text paging over OSC.

git clone https://git.yummers.dev/yum/FastTextPager

yumcode bomb0c54e1f

master
branchesmasterall refs →
2.0 KiB61 linesraw
1import math
2import sentencepiece as spm
3
4def get_tokenizer():
5    model_path = "./custom_unigram_tokenizer_65k/unigram.model"
6    sp = spm.SentencePieceProcessor()
7    sp.load(model_path)
8    return sp
9
10tokenizer = get_tokenizer()
11
12print(f"vocabulary size: {tokenizer.get_piece_size()}")
13# Sentencepiece uses U+2581 (lower one eighth block) to indicate a space before
14# a subword.
15sp_space = chr(9601)
16tokens_with_non_ascii = set()
17subword_len_histo = dict()
18# The sum of the lengths of each subword in the vocabulary. These are rounded
19# up to 4 characters.
20vocab_len_4c_quantized = 0
21
22for i in range(tokenizer.get_piece_size()):
23    k = tokenizer.id_to_piece(i)
24    v = i
25    print(f"  Original token ({v}): {repr(k)} ({' '.join(str(ord(k_c)) for k_c in k)})")
26    for k_c in k:
27        if ord(k_c) > 127 and ord(k_c) != 9601:
28            tokens_with_non_ascii.add(k)
29            break
30    k_processed = k.replace(sp_space, ' ')
31    if not k.startswith(sp_space) and k not in ["[UNK]", "[PAD]", "[CLS]", "[SEP]", "[MASK]"]:
32        k_processed = k
33    else:
34        k_processed = k_processed
35
36    current_len = len(k_processed)
37    if current_len in subword_len_histo:
38        subword_len_histo[current_len] += 1
39    else:
40        subword_len_histo[current_len] = 1
41
42    vocab_len_4c_quantized += math.ceil(current_len / 4.0) * 4.0
43    print(f"  {v}: {k_processed}")
44
45print(f"Num tokens with non-ascii: {len(tokens_with_non_ascii)} ({100 * len(tokens_with_non_ascii) / tokenizer.get_piece_size():.2f})%")
46
47print(f"Subword length histogram:")
48avg_subword_len = 0
49total_pieces_for_avg = 0
50for k_len, v_count in sorted(subword_len_histo.items(), key=lambda x: x[0]):
51    avg_subword_len += k_len * v_count
52    total_pieces_for_avg += v_count
53    print(f"  {k_len}: {v_count}")
54
55if total_pieces_for_avg > 0:
56    avg_subword_len /= total_pieces_for_avg
57    print(f"Average subword length: {avg_subword_len:.4f}")
58else:
59    print("Average subword length: N/A (no pieces analyzed)")
60
61print(f"Sum of all subword lengths, quantized to 4 character chunks: {vocab_len_4c_quantized}")