ctranslate2_english_german_translation_quickstart_with_transformers_tokenizer.py

1import ctranslate2
2import transformers
3
4# Initialize the translator with a converted model
5# Note: You must first convert a model (e.g., from Hugging Face) using the ct2-transformer-converter
6translator = ctranslate2.Translator("translation_model_dir/")
7tokenizer = transformers.AutoTokenizer.from_pretrained("translation_model_dir/")
8
9# Prepare the input tokens
10source = ["Hello world!"]
11tokens = tokenizer.convert_ids_to_tokens(tokenizer.encode(source[0]))
12
13# Run translation
14results = translator.translate_batch([tokens])
15
16# Decode the output tokens
17output_tokens = results[0].hypotheses[0]
18output_text = tokenizer.decode(tokenizer.convert_tokens_to_ids(output_tokens))
19
20print(output_text)