Transformers Frameworks

from datasets import load_dataset
# pip install datasets

# 1. Load an entire massive dataset with one line of code! (e.g., IMDB Movie Reviews)
dataset = load_dataset("imdb")

# Instantly prints metadata about the splits (train/test/unsupervised data rows)
print("Dataset Architecture:\n", dataset) 
# DatasetDict({
#     train: Dataset({ features: ['text', 'label'], num_rows: 25000 })
#     test: Dataset({ features: ['text', 'label'], num_rows: 25000 })
#     unsupervised: Dataset({ features: ['text', 'label'], num_rows: 50000 })
# })

# 2. Access a single record from the 'train' split dynamically
first_review = dataset["train"][0]

print("Review text:", first_review["text"][:100] + "...") # First 100 chars
print("Associated Label:", first_review["label"])         # 0 = Negative, 1 = Positive

from transformers import pipeline

# 1. Create an English-to-French Translation Pipeline
translator = pipeline("translation_en_to_fr")

result1 = translator("The quick brown fox jumps over the lazy dog.")
print("Translation:", result1[0]['translation_text'])
# Le rapide renard brun saute sur le chien paresseux.

# 2. Change the underlying default model dynamically
# Here we force a specific tiny model called 'bert-tiny' fine-tuned for emotion
classifier = pipeline("text-classification", model="bhadresh-savani/distilbert-base-uncased-emotion")

result2 = classifier("I am so frustrated that the code threw an error again!")
print("Classifier Output:", result2)
# [{'label': 'anger', 'score': 0.9921}]

from transformers import AutoTokenizer, AutoModelForSequenceClassification
import torch

model_name = "distilbert-base-uncased-finetuned-sst-2-english"

# Download the Tokenizer associated with the DistilBERT checkpoint
tokenizer = AutoTokenizer.from_pretrained(model_name)

# Download the actual Neural Network Architecture and Weights
# The "ForSequenceClassification" explicitly attaches a linear classification head onto the transformer body
model = AutoModelForSequenceClassification.from_pretrained(model_name)

# 1. Tokenize Input Data
text = "The graphics are stunning but the storyline is incredibly weak."

# return_tensors='pt' = Return PyTorch Tensors. (Use 'tf' for TensorFlow)
inputs = tokenizer(text, return_tensors="pt") 
print("Input IDs (Math):", inputs['input_ids'])
# tensor([[ 101, 1996, 5143, 2024, ... 102]]) # Note the [CLS] and [SEP] tokens!

# 2. Feed-Forward Neural Network Pass (No Gradient Tracking needed for Inference)
with torch.no_grad():
    outputs = model(**inputs)

# 3. Analyze Output Tensors
# The model outputs un-normalized raw scores (Logits)
logits = outputs.logits
print("Raw Logits:", logits) 

# Convert logits to percentages using Softmax
probabilities = torch.nn.functional.softmax(logits, dim=-1)
predicted_class = torch.argmax(probabilities).item()

# Assuming LABEL_0 is Negative and LABEL_1 is Positive
print(f"Prediction Class: {predicted_class}") # 0

from flair.data import Sentence
from flair.models import SequenceTagger

# Let's load a NER model (Downloads on first run)
# 'ner-fast' is optimized for CPU speed vs 'ner-large' for extreme GPU accuracy
tagger = SequenceTagger.load('ner-fast')

# You wrap strings in the special Flair 'Sentence' object 
sentence = Sentence("George Washington travelled to New York.")

# Run the neural network over the sentence
tagger.predict(sentence)

# Iterate over the tags it injected
for entity in sentence.get_spans('ner'):
    print(f"Text: {entity.text:<20} | Label: {entity.get_label('ner').value:<5} | Confidence: {entity.get_label('ner').score:.4f}")

# Outputs: 
# Text: George Washington    | Label: PER   | Confidence: 0.9997
# Text: New York             | Label: LOC   | Confidence: 0.9996

# To train an entire state-of-the-art model without writing a single line of training loop code,
# you simply provide the configuration JSON to the command line tool.
allennlp train my_custom_model_config.json -s /tmp/output_directory

import stanza

# 1. Download the English model (Requires internet, ~500MB)
# stanza.download('en')

# 2. Initialize a pipeline focusing on morphologics and dependency parsing
nlp = stanza.Pipeline('en', processors='tokenize,pos,lemma,depparse', verbose=False)

# 3. Process the text
doc = nlp("The remarkably fast fox sprinted towards the forest.")

# 4. Extract grammatical dependencies
for sentence in doc.sentences:
    for word in sentence.words:
        print(f"Word: {word.text:<12} | "
              f"Head (Parent): {sentence.words[word.head-1].text if word.head > 0 else 'root':<10} | "
              f"Dep Relation: {word.deprel}")

# Outputs the exact syntactic tree mapping adverbs (remarkably) modifying adjectives (fast)
# modifying the noun entity (fox) acting as the nominal subject (nsubj) of the verb (sprinted).