Natural Language Processing Tutorial

import re
import nltk
from nltk.corpus import stopwords
from nltk.stem import PorterStemmer, WordNetLemmatizer

nltk.download('stopwords')
nltk.download('wordnet')

def preprocess_text(text):
    # Convert to lowercase
    text = text.lower()
    
    # Remove HTML tags
    text = re.sub(r'<.*?>', '', text)
    
    # Remove special characters and digits
    text = re.sub(r'[^a-zA-Z\s]', '', text)
    
    # Tokenization
    tokens = text.split()
    
    # Remove stopwords
    stop_words = set(stopwords.words('english'))
    tokens = [word for word in tokens if word not in stop_words]
    
    # Stemming
    stemmer = PorterStemmer()
    stemmed = [stemmer.stem(word) for word in tokens]
    
    # Lemmatization
    lemmatizer = WordNetLemmatizer()
    lemmatized = [lemmatizer.lemmatize(word) for word in tokens]
    
    return {
        'original': text,
        'tokens': tokens,
        'stemmed': stemmed,
        'lemmatized': lemmatized
    }

# Example usage
text = "The cats are running quickly in the gardens!"
result = preprocess_text(text)
print(result)

from gensim.models import Word2Vec
from nltk.tokenize import word_tokenize

# Sample sentences
sentences = [
    "I love natural language processing",
    "Word embeddings capture semantic meaning",
    "NLP is fascinating and useful",
    "Deep learning powers modern NLP"
]

# Tokenize sentences
tokenized_sentences = [word_tokenize(sent.lower()) for sent in sentences]

# Train Word2Vec model
model = Word2Vec(
    sentences=tokenized_sentences,
    vector_size=100,
    window=5,
    min_count=1,
    workers=4
)

# Find similar words
similar_words = model.wv.most_similar('nlp')
print("Words similar to 'nlp':")
for word, score in similar_words:
    print(f"{word}: {score:.3f}")

# Get word vector
vector = model.wv['language']
print(f"\nVector for 'language': {vector[:5]}...")

import numpy as np

def scaled_dot_product_attention(Q, K, V):
    """
    Attention(Q,K,V) = softmax(QK^T/√d_k)V
    """
    d_k = K.shape[-1]
    scores = np.matmul(Q, K.transpose()) / np.sqrt(d_k)
    attention_weights = np.exp(scores) / np.sum(np.exp(scores), axis=-1, keepdims=True)
    output = np.matmul(attention_weights, V)
    return output, attention_weights

# Example
seq_len, d_k = 3, 4
Q = np.random.randn(seq_len, d_k)
K = np.random.randn(seq_len, d_k)
V = np.random.randn(seq_len, d_k)

output, weights = scaled_dot_product_attention(Q, K, V)
print("Attention weights:\n", weights.round(2))

from transformers import BertTokenizer, BertModel
import torch

# Load pre-trained BERT
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
model = BertModel.from_pretrained('bert-base-uncased')

# Tokenize input
text = "Natural Language Processing is fascinating!"
inputs = tokenizer(text, return_tensors='pt', 
                  padding=True, truncation=True)

# Get embeddings
with torch.no_grad():
    outputs = model(**inputs)
    
# Last hidden states
last_hidden_states = outputs.last_hidden_state
# Pooled output (CLS token)
pooled_output = outputs.pooler_output

print(f"Sequence length: {last_hidden_states.shape[1]}")
print(f"Hidden size: {last_hidden_states.shape[2]}")
print(f"Pooled output shape: {pooled_output.shape}")

from transformers import GPT2Tokenizer, GPT2LMHeadModel

# Load pre-trained GPT-2
tokenizer = GPT2Tokenizer.from_pretrained('gpt2')
model = GPT2LMHeadModel.from_pretrained('gpt2')

# Encode input
input_text = "Natural Language Processing is"
inputs = tokenizer.encode(input_text, return_tensors='pt')

# Generate text
outputs = model.generate(
    inputs,
    max_length=50,
    num_return_sequences=1,
    temperature=0.7,
    do_sample=True
)

# Decode and print
generated_text = tokenizer.decode(outputs[0], skip_special_tokens=True)
print(generated_text)