huggingface_datasets_load_explore_and_map_preprocess.py

1from datasets import load_dataset
2
3# 1. Load a dataset from the Hub
4dataset = load_dataset("rotten_tomatoes", split="train")
5
6# 2. Explore the dataset
7print(dataset[0])
8
9# 3. Preprocess the data
10def tokenize_function(examples):
11    return {"length": [len(x) for x in examples["text"]]}
12
13dataset = dataset.map(tokenize_function, batched=True)
14
15# 4. Check the results
16print(dataset[0])