Data Science Tutorial

# Data structures commonly used in DS
import numpy as np
import pandas as pd

# Lists, tuples, dictionaries
data_list = [10, 20, 30, 40, 50]
data_tuple = (1, 2, 3, 4, 5)
data_dict = {'name': 'Alice', 'age': 25, 'city': 'New York'}

# List comprehensions
squared = [x**2 for x in data_list]
print(f"Squared: {squared}")

# Functions for data processing
def clean_data(df):
    """Basic data cleaning function"""
    # Remove duplicates
    df = df.drop_duplicates()
    # Fill missing values
    df = df.fillna(df.mean())
    return df

# Working with NumPy arrays
arr = np.array([[1, 2, 3], [4, 5, 6]])
print(f"Array shape: {arr.shape}")
print(f"Mean: {arr.mean()}, Std: {arr.std()}")

import pandas as pd
import numpy as np

# Create DataFrame
df = pd.DataFrame({
    'name': ['Alice', 'Bob', 'Charlie', 'Diana', 'Eve'],
    'age': [25, 30, 35, np.nan, 28],
    'salary': [50000, 60000, 75000, 80000, 55000],
    'department': ['HR', 'IT', 'IT', 'Finance', 'HR']
})

# Explore data
print(df.head())
print(df.info())
print(df.describe())

# Handle missing values
df['age'].fillna(df['age'].mean(), inplace=True)

# Filtering
it_employees = df[df['department'] == 'IT']
high_earners = df[df['salary'] > 60000]

# Grouping and aggregation
dept_stats = df.groupby('department').agg({
    'salary': ['mean', 'max', 'min'],
    'age': 'mean'
}).round(2)

print("\nDepartment Statistics:")
print(dept_stats)

# Apply functions
df['salary_category'] = df['salary'].apply(
    lambda x: 'High' if x > 60000 else 'Medium' if x > 50000 else 'Low'
)

# Merge and join
df2 = pd.DataFrame({
    'department': ['HR', 'IT', 'Finance'],
    'budget': [100000, 500000, 300000]
})
merged = pd.merge(df, df2, on='department', how='left')

from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import classification_report, confusion_matrix
import pandas as pd

# Load data
iris = load_iris()
X, y = iris.data, iris.target
feature_names = iris.feature_names

# Split data
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42, stratify=y
)

# Scale features
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)

# Train model
model = RandomForestClassifier(n_estimators=100, random_state=42)
model.fit(X_train_scaled, y_train)

# Predict
y_pred = model.predict(X_test_scaled)
y_proba = model.predict_proba(X_test_scaled)

# Evaluate
print("Classification Report:")
print(classification_report(y_test, y_pred, target_names=iris.target_names))

# Feature importance
importance = pd.DataFrame({
    'feature': feature_names,
    'importance': model.feature_importances_
}).sort_values('importance', ascending=False)
print("\nFeature Importance:")
print(importance)

from sklearn.cluster import KMeans
from sklearn.datasets import make_blobs
import matplotlib.pyplot as plt

# Generate sample data
X, y_true = make_blobs(n_samples=300, centers=4, 
                       cluster_std=0.60, random_state=0)

# Find optimal clusters using elbow method
inertias = []
K_range = range(1, 10)
for k in K_range:
    kmeans = KMeans(n_clusters=k, random_state=42)
    kmeans.fit(X)
    inertias.append(kmeans.inertia_)

# Plot elbow curve
plt.figure(figsize=(8, 4))
plt.plot(K_range, inertias, 'bo-')
plt.xlabel('Number of clusters (k)')
plt.ylabel('Inertia')
plt.title('Elbow Method for Optimal k')
plt.grid(True)
plt.show()

# Apply K-means with optimal k
kmeans = KMeans(n_clusters=4, random_state=42)
y_kmeans = kmeans.fit_predict(X)

# Visualize clusters
plt.figure(figsize=(8, 6))
plt.scatter(X[:, 0], X[:, 1], c=y_kmeans, cmap='viridis')
plt.scatter(kmeans.cluster_centers_[:, 0], 
            kmeans.cluster_centers_[:, 1], 
            marker='X', s=200, linewidths=3, 
            color='red', label='Centroids')
plt.title('K-Means Clustering Results')
plt.legend()
plt.show()

import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers

# Build model
model = keras.Sequential([
    layers.Dense(64, activation='relu', input_shape=(4,)),
    layers.Dropout(0.2),
    layers.Dense(32, activation='relu'),
    layers.Dropout(0.2),
    layers.Dense(3, activation='softmax')
])

# Compile model
model.compile(
    optimizer='adam',
    loss='sparse_categorical_crossentropy',
    metrics=['accuracy']
)

# Train model
history = model.fit(
    X_train, y_train,
    epochs=50,
    batch_size=16,
    validation_split=0.2,
    verbose=0
)

# Evaluate
test_loss, test_acc = model.evaluate(X_test, y_test, verbose=0)
print(f"Test accuracy: {test_acc:.3f}")

# Plot training history
import matplotlib.pyplot as plt

plt.figure(figsize=(12, 4))
plt.subplot(1, 2, 1)
plt.plot(history.history['loss'], label='Training')
plt.plot(history.history['val_loss'], label='Validation')
plt.title('Model Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()

plt.subplot(1, 2, 2)
plt.plot(history.history['accuracy'], label='Training')
plt.plot(history.history['val_accuracy'], label='Validation')
plt.title('Model Accuracy')
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
plt.legend()
plt.tight_layout()
plt.show()