spacy_transformers_load_pipeline_access_hidden_states.py

1import spacy
2
3# Load a pipeline with a transformer component
4# Note: You must have installed the model first, e.g., 
5# python -m spacy download en_core_web_trf
6nlp = spacy.load("en_core_web_trf")
7
8# Process a text
9doc = nlp("Apple is looking at buying U.K. startup for $1 billion")
10
11# The transformer data is stored in the doc._.trf_data extension attribute
12trf_data = doc._.trf_data
13
14# Access the model output (tensors)
15print(f"Tensor shape: {trf_data.tensors[0].shape}")
16
17# Access the word-to-token alignment
18# This shows which transformer tokens correspond to which spaCy token
19print(f"Alignment for '{doc[0]}': {trf_data.align[0].data}")
20
21# Named entities are also available as usual
22for ent in doc.ents:
23    print(ent.text, ent.label_)