s3tokenizer_load_huggingface_tokenizer_from_s3_bucket.py

1from s3tokenizer import S3Tokenizer
2from transformers import AutoTokenizer
3
4# Initialize the S3Tokenizer with your bucket name and prefix
5# It will download the tokenizer files from s3://my-bucket/my-model-prefix/
6tokenizer = S3Tokenizer.from_s3(
7    bucket_name="my-bucket",
8    prefix="my-model-prefix/",
9    region_name="us-east-1"
10)
11
12# Use it like a normal HuggingFace tokenizer
13text = "Hello, this is a test."
14encoded_input = tokenizer(text, return_tensors="pt")
15print(encoded_input)