sentencepiece_train_model_encode_decode_text_quickstart.py

1import sentencepiece as spm
2
3# train sentencepiece model from `botchan.txt` and makes `m.model` and `m.vocab`
4# `m.vocab` is just a reference. not used in the segmentation.
5spm.SentencePieceTrainer.train('--input=botchan.txt --model_prefix=m --vocab_size=2000')
6
7# makes segmenter instance and loads the model file (m.model)
8sp = spm.SentencePieceProcessor()
9sp.load('m.model')
10
11# encode: text => id
12print(sp.encode_as_pieces('This is a test'))
13print(sp.encode_as_ids('This is a test'))
14
15# decode: id => text
16print(sp.decode_pieces([' This', ' is', ' a', ' t', 'est']))
17print(sp.decode_ids([209, 31, 9, 435]))