tensorflow_text_whitespace_tokenizer_quickstart_ragged_tensors.py

1import tensorflow as tf
2import tensorflow_text as text
3
4# Define some input text
5docs = tf.constant(['Everything not saved will be lost.', 'Sad but true.'])
6
7# Initialize a tokenizer (WhitespaceTokenizer is a common starting point)
8tokenizer = text.WhitespaceTokenizer()
9
10# Tokenize the input text
11tokens = tokenizer.tokenize(docs)
12
13# Print the resulting RaggedTensor of tokens
14print(tokens)