huggingface_datasets_load_explore_and_tokenize_with_bert.py

1from datasets import load_dataset
2from transformers import AutoTokenizer
3
4# 1. Load a dataset
5dataset = load_dataset("rotten_tomatoes", split="train")
6
7# 2. Explore the dataset
8print(dataset[0])
9
10# 3. Tokenize the data
11tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
12
13def tokenize_function(examples):
14    return tokenizer(examples["text"], padding="max_length", truncation=True)
15
16tokenized_datasets = dataset.map(tokenize_function, batched=True)
17
18# 4. Preview the processed data
19print(tokenized_datasets[0])