Node.js File Uploads: Complete Theory & Practice Guide

1. Theory: File Upload Fundamentals

How File Uploads Work

// Conceptual overview of multipart/form-data
/*
Browser sends:
POST /upload HTTP/1.1
Content-Type: multipart/form-data; boundary=----WebKitFormBoundaryxyz

------WebKitFormBoundaryxyz
Content-Disposition: form-data; name="file"; filename="photo.jpg"
Content-Type: image/jpeg

[Binary file data here]
------WebKitFormBoundaryxyz
Content-Disposition: form-data; name="description"

My vacation photo
------WebKitFormBoundaryxyz--
*/

Key Concepts

Upload Strategies Comparison

// upload-strategies.js
const strategies = {
  // Strategy 1: Memory Storage (small files only)
  memory: {
    pros: ['Fastest', 'No disk I/O', 'Good for <5MB'],
    cons: ['Blocks event loop', 'Memory intensive', 'DDoS risk'],
    useCase: 'Profile pictures, small JSON files'
  },
  // Strategy 2: Disk Storage (most common)
  disk: {
    pros: ['Handles large files', 'Stream processing', 'Persistent'],
    cons: ['Disk space management', 'I/O bottleneck', 'Cleanup needed'],
    useCase: 'Documents, images, videos up to 500MB'
  },
  // Strategy 3: Direct to Cloud (best for scale)
  cloud: {
    pros: ['No server storage', 'CDN delivery', 'Scalable'],
    cons: ['Network latency', 'Costs', 'More complex'],
    useCase: 'User-generated content, large media files'
  },
  // Strategy 4: Streaming (real-time processing)
  streaming: {
    pros: ['Low memory', 'Process while uploading', 'Fast response'],
    cons: ['Complex error handling', 'Limited processing'],
    useCase: 'Video processing, virus scanning'
  }
};

2. Basic Upload with Express

Without External Libraries (Pure Express)

// basic-express-upload.js
const express = require('express');
const fs = require('fs');
const path = require('path');
const app = express();

// Create uploads directory if it doesn't exist
const uploadDir = path.join(__dirname, 'uploads');
if (!fs.existsSync(uploadDir)) {
  fs.mkdirSync(uploadDir, { recursive: true });
}

// Parse multipart form data manually (not recommended for production)
app.post('/upload', (req, res) => {
  let chunks = [];
  let fileSize = 0;
  req.on('data', chunk => {
    chunks.push(chunk);
    fileSize += chunk.length;
    // Prevent memory overload (limit to 10MB)
    if (fileSize > 10 * 1024 * 1024) {
      req.destroy();
      return res.status(413).json({ error: 'File too large' });
    }
  });
  req.on('end', () => {
    const buffer = Buffer.concat(chunks);
    const filename = `upload-${Date.now()}.bin`;
    const filepath = path.join(uploadDir, filename);
    fs.writeFile(filepath, buffer, err => {
      if (err) return res.status(500).json({ error: 'Save failed' });
      res.json({ message: 'File saved', filename, size: fileSize });
    });
  });
});
const multer = require('multer'); // We'll cover this next

Simple HTML Form Example

<!-- upload-form.html -->
<form action="/upload-simple" method="post" enctype="multipart/form-data">
  <input type="file" name="file" required>
  <input type="text" name="description" placeholder="File description">
  <button type="submit">Upload</button>
</form>

<!-- AJAX upload with progress (see full client script in notes) -->

3. Multer Deep Dive

Installation

npm install multer

Basic Multer Setup

// multer-basic.js
const express = require('express');
const multer = require('multer');
const path = require('path');
const fs = require('fs');
const app = express();

const storage = multer.diskStorage({
  destination: (req, file, cb) => {
    const uploadPath = path.join(__dirname, 'uploads');
    const date = new Date();
    const year = date.getFullYear();
    const month = String(date.getMonth() + 1).padStart(2, '0');
    const fullPath = path.join(uploadPath, year.toString(), month);
    fs.mkdirSync(fullPath, { recursive: true });
    cb(null, fullPath);
  },
  filename: (req, file, cb) => {
    const uniqueSuffix = Date.now() + '-' + Math.round(Math.random() * 1E9);
    const ext = path.extname(file.originalname);
    const basename = path.basename(file.originalname, ext);
    const safeBasename = basename.replace(/[^a-zA-Z0-9]/g, '-');
    cb(null, `${safeBasename}-${uniqueSuffix}${ext}`);
  }
});

const fileFilter = (req, file, cb) => {
  const allowedTypes = /jpeg|jpg|png|gif|pdf|doc|docx/;
  const extname = allowedTypes.test(path.extname(file.originalname).toLowerCase());
  const mimetype = allowedTypes.test(file.mimetype);
  if (mimetype && extname) cb(null, true);
  else cb(new Error('Invalid file type. Only images and documents are allowed.'));
};

Advanced Multer Configuration

// multer-advanced.js
const multer = require('multer');
const crypto = require('crypto');
const path = require('path');

class UploadManager {
  constructor(options = {}) {
    this.options = { maxFileSize: options.maxFileSize || 10 * 1024 * 1024, storageType: options.storageType || 'disk', ...options };
    this.upload = this.createMulterInstance();
  }
  createMulterInstance() {
    let storage;
    if (this.options.storageType === 'memory') {
      storage = multer.memoryStorage();
    } else {
      storage = multer.diskStorage({
        destination: (req, file, cb) => cb(null, this.options.uploadDir || './uploads'),
        filename: (req, file, cb) => cb(null, this.generateSecureFilename(file))
      });
    }
    return multer({
      storage,
      limits: { fileSize: this.options.maxFileSize, files: this.options.maxFiles || 10, fields: 20, parts: 50 },
      fileFilter: (req, file, cb) => this.validateFile(file, cb)
    });
  }
  validateFile(file, cb) {
    const dangerousPatterns = /\.\.\/|\.exe$|\.bat$|\.sh$|\.cmd$/i;
    if (dangerousPatterns.test(file.originalname)) return cb(new Error('Malicious filename detected'));
    cb(null, true);
  }
  single(fieldName) { return this.upload.single(fieldName); }
  array(fieldName, maxCount) { return this.upload.array(fieldName, maxCount); }
  fields(fieldsConfig) { return this.upload.fields(fieldsConfig); }
  async cleanupOldFiles(maxAge = 24 * 60 * 60 * 1000) {
    const fs = require('fs').promises;
    const now = Date.now();
    const scanDir = async (dir) => {
      const files = await fs.readdir(dir);
      for (const file of files) {
        const filePath = path.join(dir, file);
        const stat = await fs.stat(filePath);
        if (stat.isDirectory()) await scanDir(filePath);
        else if (now - stat.mtimeMs > maxAge) await fs.unlink(filePath);
      }
    };
    await scanDir(this.options.uploadDir || './uploads');
  }
}

4. Validation & Security

Comprehensive File Validation

// file-validation.js
const crypto = require('crypto');
const fs = require('fs');
const path = require('path');

class FileValidator {
  constructor(options = {}) {
    this.options = { maxSize: options.maxSize || 10 * 1024 * 1024, checkMagicNumbers: options.checkMagicNumbers !== false, ...options };
  }
  async validate(file) {
    const errors = [];
    if (file.size > this.options.maxSize) errors.push(`File too large: ${file.size}`);
    return { valid: errors.length === 0, errors, metadata: this.extractMetadata(file) };
  }
  checkMagicNumbers(buffer, declaredMime) {
    const signatures = {
      'image/jpeg': [0xFF, 0xD8, 0xFF],
      'image/png': [0x89, 0x50, 0x4E, 0x47],
      'image/gif': [0x47, 0x49, 0x46],
      'application/pdf': [0x25, 0x50, 0x44, 0x46],
      'application/zip': [0x50, 0x4B, 0x03, 0x04]
    };
    const signature = signatures[declaredMime];
    if (!signature) return true;
    for (let i = 0; i < signature.length; i++) if (buffer[i] !== signature[i]) return false;
    return true;
  }
  async scanForVirus(buffer) {
    const eicar = 'X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*';
    return !buffer.toString('binary').includes(eicar);
  }
  extractMetadata(file) {
    return {
      filename: file.originalname,
      size: file.size,
      mimetype: file.mimetype,
      extension: path.extname(file.originalname),
      uploadDate: new Date().toISOString(),
      hash: file.buffer ? crypto.createHash('sha256').update(file.buffer).digest('hex') : null
    };
  }
  sanitizeFilename(filename) { return filename.replace(/[^a-zA-Z0-9.-]/g, '_').replace(/\.{2,}/g, '.').replace(/^\.+/, '').substring(0, 255); }
  async deepInspect(filePath) {
    const buffer = fs.readFileSync(filePath);
    return {
      path: filePath,
      size: fs.statSync(filePath).size,
      hash: crypto.createHash('sha256').update(buffer).digest('hex'),
      suspicious: { hasNullBytes: buffer.includes(0) }
    };
  }
}

5. Advanced Patterns

Resumable Uploads (Chunked)

// resumable-upload.js
const express = require('express');
const fs = require('fs');
const path = require('path');

class ResumableUpload {
  constructor(uploadDir = './chunks') {
    this.uploadDir = uploadDir;
    this.chunksDir = path.join(uploadDir, 'chunks');
    this.completedDir = path.join(uploadDir, 'completed');
    [this.chunksDir, this.completedDir].forEach(dir => {
      if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
    });
  }
  async uploadChunk(req, res) {
    const { chunkNumber, totalChunks, fileIdentifier, filename } = req.body;
    const chunkDir = path.join(this.chunksDir, fileIdentifier);
    if (!fs.existsSync(chunkDir)) fs.mkdirSync(chunkDir, { recursive: true });
    fs.renameSync(req.file.path, path.join(chunkDir, `${chunkNumber}`));
    const uploadedChunks = fs.readdirSync(chunkDir).length;
    if (uploadedChunks === parseInt(totalChunks, 10)) {
      await this.assembleChunks(fileIdentifier, filename, totalChunks);
      return res.json({ status: 'complete', filename });
    }
    return res.json({ status: 'uploading', progress: (uploadedChunks / totalChunks) * 100 });
  }
  async assembleChunks(fileIdentifier, filename, totalChunks) {
    const chunkDir = path.join(this.chunksDir, fileIdentifier);
    const finalPath = path.join(this.completedDir, filename);
    const writeStream = fs.createWriteStream(finalPath);
    for (let i = 1; i <= totalChunks; i++) {
      const chunkPath = path.join(chunkDir, i.toString());
      writeStream.write(fs.readFileSync(chunkPath));
      fs.unlinkSync(chunkPath);
    }
    writeStream.end();
    fs.rmdirSync(chunkDir);
  }
  async getUploadStatus(fileIdentifier) {
    const chunkDir = path.join(this.chunksDir, fileIdentifier);
    if (!fs.existsSync(chunkDir)) return { uploadedChunks: [] };
    const chunks = fs.readdirSync(chunkDir).map(c => parseInt(c, 10)).sort((a, b) => a - b);
    return { uploadedChunks: chunks, nextChunk: chunks.length + 1 };
  }
}

Parallel Upload Handler

// parallel-uploads.js
const os = require('os');
class ParallelUploadManager {
  constructor(options = {}) {
    this.maxConcurrent = options.maxConcurrent || os.cpus().length;
    this.queue = [];
    this.active = 0;
    this.results = [];
  }
  async uploadFiles(files, uploadFn) {
    this.queue = [...files];
    this.results = [];
    const workers = Math.min(this.maxConcurrent, files.length);
    await Promise.all(Array.from({ length: workers }, () => this.worker(uploadFn)));
    return this.results;
  }
  async worker(uploadFn) {
    while (this.queue.length > 0) {
      const file = this.queue.shift();
      this.active++;
      try {
        const result = await uploadFn(file);
        this.results.push({ success: true, file: file.originalname, result });
      } catch (error) {
        this.results.push({ success: false, file: file.originalname, error: error.message });
      } finally {
        this.active--;
      }
    }
  }
  getStats() {
    return {
      processed: this.results.length,
      successful: this.results.filter(r => r.success).length,
      failed: this.results.filter(r => !r.success).length,
      active: this.active,
      queued: this.queue.length
    };
  }
}

6. Cloud Storage Integration

AWS S3 Upload

npm install aws-sdk @aws-sdk/client-s3 @aws-sdk/s3-request-presigner

// s3-upload.js
const { S3Client, PutObjectCommand, GetObjectCommand } = require('@aws-sdk/client-s3');
const { getSignedUrl } = require('@aws-sdk/s3-request-presigner');
const multer = require('multer');
const multerS3 = require('multer-s3');
const crypto = require('crypto');

class S3UploadService {
  constructor() {
    this.s3 = new S3Client({
      region: process.env.AWS_REGION || 'us-east-1',
      credentials: {
        accessKeyId: process.env.AWS_ACCESS_KEY_ID,
        secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY
      }
    });
    this.bucket = process.env.S3_BUCKET_NAME;
  }
  getMulterS3Storage() {
    return multerS3({ s3: this.s3, bucket: this.bucket, acl: 'private', contentType: multerS3.AUTO_CONTENT_TYPE });
  }
  async generatePresignedUrl(filename, contentType, expiresIn = 3600) {
    const key = `temp-uploads/${Date.now()}-${filename}`;
    const command = new PutObjectCommand({ Bucket: this.bucket, Key: key, ContentType: contentType });
    const url = await getSignedUrl(this.s3, command, { expiresIn });
    return { url, key };
  }
  async uploadBuffer(buffer, filename, contentType) {
    const key = `uploads/${Date.now()}-${filename}`;
    await this.s3.send(new PutObjectCommand({ Bucket: this.bucket, Key: key, Body: buffer, ContentType: contentType }));
    return { key, location: `https://${this.bucket}.s3.amazonaws.com/${key}` };
  }
  async generateDownloadUrl(key, expiresIn = 3600) {
    return getSignedUrl(this.s3, new GetObjectCommand({ Bucket: this.bucket, Key: key }), { expiresIn });
  }
}

Google Cloud Storage

// gcs-upload.js
const { Storage } = require('@google-cloud/storage');
class GCSUploadService {
  constructor() {
    this.storage = new Storage({ projectId: process.env.GCP_PROJECT_ID, keyFilename: process.env.GCP_KEY_FILE });
    this.bucket = this.storage.bucket(process.env.GCS_BUCKET_NAME);
  }
  getMulterGCSStorage() { return null; }
  async uploadStream(readStream, filename, options = {}) { return { filename, options }; }
  async deleteFile(filename) { return { success: true, filename }; }
  async getSignedUrl(filename, expiresIn = 3600) { return `signed-url://${filename}?exp=${expiresIn}`; }
}

7. Streaming & Performance

Stream Processing Pipeline

// streaming-upload.js
const { PassThrough, Transform } = require('stream');
const sharp = require('sharp');
const { pipeline } = require('stream/promises');

class StreamProcessor {
  static createImageOptimizer(options = {}) { return new Transform({ transform(chunk, enc, cb) { sharp(chunk).resize(options.width || 1200, options.height || 1200, { fit: 'inside' }).jpeg({ quality: options.quality || 80 }).toBuffer().then(out => cb(null, out)).catch(cb); } }); }
  static createVirusScanner() { return new Transform({ transform(chunk, enc, cb) { if (chunk.toString().includes('EICAR')) cb(new Error('Virus detected')); else cb(null, chunk); } }); }
  static createThrottle(bytesPerSecond) { return new Transform({ transform(chunk, enc, cb) { cb(null, chunk); } }); }
  static createChecksumStream() { const crypto = require('crypto'); const hash = crypto.createHash('md5'); return new Transform({ transform(chunk, enc, cb) { hash.update(chunk); cb(null, chunk); }, flush(cb) { this.checksum = hash.digest('hex'); cb(); } }); }
}

Performance Monitoring

class UploadPerfMonitor {
  constructor() {
    this.metrics = { totalUploads: 0, totalBytes: 0, averageSpeed: 0, errors: 0, operations: [] };
  }
  trackUpload(startTime, fileSize, success = true) {
    const duration = Date.now() - startTime;
    const speedMBps = (fileSize / 1024 / 1024) / (duration / 1000 || 1);
    this.metrics.totalUploads++;
    this.metrics.totalBytes += fileSize;
    if (!success) this.metrics.errors++;
    this.metrics.operations.push({ timestamp: new Date().toISOString(), size: fileSize, duration, speed: speedMBps, success });
    return { duration, speedMBps };
  }
  getMetrics() { return this.metrics; }
}

8. Best Practices

Complete Production-Ready Upload System

// production-upload-system.js
const express = require('express');
const multer = require('multer');
const rateLimit = require('express-rate-limit');
const helmet = require('helmet');
const cors = require('cors');
const { body, validationResult } = require('express-validator');
const path = require('path');
const fs = require('fs').promises;

class ProductionUploadSystem {
  constructor() {
    this.app = express();
    this.setupMiddleware();
    this.setupUploadEndpoints();
    this.setupErrorHandling();
  }
  setupMiddleware() { this.app.use(helmet()); this.app.use(cors()); this.app.use(express.json()); }
  setupUploadEndpoints() {}
  getSecureStorage() { return multer.diskStorage({ destination: (req, file, cb) => cb(null, './secure_uploads'), filename: (req, file, cb) => cb(null, Date.now() + '-' + file.originalname) }); }
  secureFileFilter(req, file, cb) { cb(null, true); }
  validateUploadRequest() { return [(req, res, next) => next()]; }
  setupErrorHandling() { this.app.use((error, req, res, next) => res.status(500).json({ error: error.message })); }
}

Upload Security Checklist

// security-checklist-uploads.js
const uploadSecurityChecklist = {
  validateFileType: (file) => {
    const allowedTypes = ['image/jpeg', 'image/png', 'application/pdf'];
    return allowedTypes.includes(file.mimetype);
  },
  enforceSizeLimits: (file, maxSize = 10 * 1024 * 1024) => file.size <= maxSize,
  sanitizeFilename: (filename) => filename.replace(/[^a-zA-Z0-9.-]/g, '_'),
  requireAuthentication: true,
  rateLimitConfig: { windowMs: 15 * 60 * 1000, max: 100, message: 'Too many upload attempts' }
};

Summary - Key Takeaways

• Never trust user input - Always validate files on the server
• Use streaming for large files - Avoid loading into memory
• Implement rate limiting - Prevent DoS attacks
• Store files outside webroot - Prevent direct access
• Generate random filenames - Prevent path traversal
• Scan for malware - Use ClamAV or similar
• Set file size limits - Multiple layers (nginx, express, multer)
• Log uploads - Audit trail for security
• Use HTTPS - Encrypt file uploads in transit
• Implement cleanup jobs - Remove old temp files

# Test upload
curl -F "file=@photo.jpg" http://localhost:3000/upload

# Multiple uploads
curl -F "files=@doc1.pdf" -F "files=@doc2.pdf" http://localhost:3000/upload/multiple

# Check upload directory
ls -la uploads/

# Monitor upload logs
tail -f logs/upload.log

10 Interview Questions + 10 MCQs

Contextual Learning Links

• Node.js Security - validation, hardening, and secure headers for upload endpoints
• Node.js HTTP Module - low-level understanding of request body streaming
• Node.js Buffers & Streams - memory-safe file handling patterns
• Node.js Environment Config - storage credentials and limits via environment variables
• Node.js Error Handling & Logging - upload observability and incident debugging