Understanding Node.js: The JavaScript Runtime

Node.js is an open-source, cross-platform JavaScript runtime environment that executes JavaScript code outside of a web browser. Created by Ryan Dahl in 2009, Node.js enables developers to use JavaScript for server-side scripting and building scalable network applications.

Node.js uses an event-driven, non-blocking I/O model that makes it lightweight and efficient, perfect for data-intensive real-time applications that run across distributed devices. It's built on Chrome's V8 JavaScript engine and uses libuv for handling asynchronous operations.

1. Node.js Basics & Fundamentals

// Basic Node.js script
// Save as app.js

console.log('Hello, Node.js!');

// Process information
console.log('Node.js version:', process.version);
console.log('Platform:', process.platform);
console.log('Current directory:', process.cwd());

// Command line arguments
console.log('Arguments:', process.argv);

// Simple function
function greet(name) {
    return `Hello, ${name}!`;
}

console.log(greet('World'));

// Running the script:
// node app.js
// node app.js argument1 argument2

// File: calculator.js
// Simple calculator module
function add(a, b) {
    return a + b;
}

function subtract(a, b) {
    return a - b;
}

function multiply(a, b) {
    return a * b;
}

function divide(a, b) {
    if (b === 0) {
        throw new Error('Division by zero is not allowed');
    }
    return a / b;
}

// Export functions
module.exports = {
    add,
    subtract,
    multiply,
    divide
};

// Using the calculator in another file
// File: main.js
const calculator = require('./calculator');

console.log('5 + 3 =', calculator.add(5, 3));
console.log('10 - 4 =', calculator.subtract(10, 4));
console.log('6 * 7 =', calculator.multiply(6, 7));
console.log('15 / 3 =', calculator.divide(15, 3));

try {
    console.log('10 / 0 =', calculator.divide(10, 0));
} catch (error) {
    console.error('Error:', error.message);
}

// File: core-modules.js

// File System Module
const fs = require('fs');
const path = require('path');

// Path operations
console.log('Current file:', __filename);
console.log('Current directory:', __dirname);
console.log('Path separator:', path.sep);
console.log('File extension:', path.extname(__filename));
console.log('Base name:', path.basename(__filename));
console.log('Directory name:', path.dirname(__filename));

// File operations
const filePath = './example.txt';

// Write file
fs.writeFileSync(filePath, 'Hello, Node.js File System!\nThis is a second line.');

// Read file
const content = fs.readFileSync(filePath, 'utf8');
console.log('File content:', content);

// Append to file
fs.appendFileSync(filePath, '\nAppended content!');

// Check if file exists
if (fs.existsSync(filePath)) {
    console.log('File exists');
}

// Read file asynchronously
fs.readFile(filePath, 'utf8', (err, data) => {
    if (err) {
        console.error('Error reading file:', err);
        return;
    }
    console.log('Async file content:', data);
});

// OS Module
const os = require('os');

console.log('OS Platform:', os.platform());
console.log('OS Architecture:', os.arch());
console.log('CPU Cores:', os.cpus().length);
console.log('Total Memory:', (os.totalmem() / 1024 / 1024 / 1024).toFixed(2), 'GB');
console.log('Free Memory:', (os.freemem() / 1024 / 1024 / 1024).toFixed(2), 'GB');
console.log('Home Directory:', os.homedir());
console.log('Uptime:', (os.uptime() / 60 / 60).toFixed(2), 'hours');

// URL Module
const url = require('url');

const exampleUrl = 'https://example.com:8080/path/name?query=string#hash';
const parsedUrl = url.parse(exampleUrl, true);

console.log('Protocol:', parsedUrl.protocol);
console.log('Host:', parsedUrl.host);
console.log('Path:', parsedUrl.pathname);
console.log('Query:', parsedUrl.query);
console.log('Hash:', parsedUrl.hash);

// Util Module
const util = require('util');

// Promisify callback-based functions
const readFileAsync = util.promisify(fs.readFile);

// Use with async/await
async function readFileExample() {
    try {
        const data = await readFileAsync(filePath, 'utf8');
        console.log('Promisified file read:', data);
    } catch (error) {
        console.error('Error:', error);
    }
}

readFileExample();

// Format strings
const formatted = util.format('Hello %s, you have %d messages', 'John', 5);
console.log('Formatted:', formatted);

// Events Module
const EventEmitter = require('events');

class MyEmitter extends EventEmitter {}

const myEmitter = new MyEmitter();

myEmitter.on('event', (data) => {
    console.log('Event received with data:', data);
});

myEmitter.on('error', (error) => {
    console.error('Error occurred:', error);
});

// Emit events
myEmitter.emit('event', { message: 'Hello Event!' });
myEmitter.emit('error', new Error('Something went wrong'));

// File: file-operations.js
const fs = require('fs').promises; // Using promises API
const path = require('path');

class FileManager {
    constructor(basePath = './data') {
        this.basePath = basePath;
    }

    // Ensure directory exists
    async ensureDirectory() {
        try {
            await fs.access(this.basePath);
        } catch (error) {
            await fs.mkdir(this.basePath, { recursive: true });
            console.log('Directory created:', this.basePath);
        }
    }

    // Create file with content
    async createFile(filename, content) {
        await this.ensureDirectory();
        const filePath = path.join(this.basePath, filename);
        
        await fs.writeFile(filePath, content);
        console.log('File created:', filePath);
        return filePath;
    }

    // Read file content
    async readFile(filename) {
        const filePath = path.join(this.basePath, filename);
        
        try {
            const content = await fs.readFile(filePath, 'utf8');
            console.log('File read:', filePath);
            return content;
        } catch (error) {
            throw new Error(`File not found: ${filePath}`);
        }
    }

    // Update file (append)
    async appendToFile(filename, content) {
        const filePath = path.join(this.basePath, filename);
        
        try {
            await fs.appendFile(filePath, content);
            console.log('Content appended to:', filePath);
        } catch (error) {
            throw new Error(`Cannot append to file: ${filePath}`);
        }
    }

    // Delete file
    async deleteFile(filename) {
        const filePath = path.join(this.basePath, filename);
        
        try {
            await fs.unlink(filePath);
            console.log('File deleted:', filePath);
        } catch (error) {
            throw new Error(`Cannot delete file: ${filePath}`);
        }
    }

    // List files in directory
    async listFiles() {
        await this.ensureDirectory();
        
        try {
            const files = await fs.readdir(this.basePath);
            console.log('Files in directory:', files);
            return files;
        } catch (error) {
            throw new Error(`Cannot read directory: ${this.basePath}`);
        }
    }

    // Get file information
    async getFileInfo(filename) {
        const filePath = path.join(this.basePath, filename);
        
        try {
            const stats = await fs.stat(filePath);
            return {
                filename,
                size: stats.size,
                created: stats.birthtime,
                modified: stats.mtime,
                isDirectory: stats.isDirectory(),
                isFile: stats.isFile()
            };
        } catch (error) {
            throw new Error(`Cannot get file info: ${filePath}`);
        }
    }

    // Copy file
    async copyFile(sourceFilename, destFilename) {
        const sourcePath = path.join(this.basePath, sourceFilename);
        const destPath = path.join(this.basePath, destFilename);
        
        try {
            await fs.copyFile(sourcePath, destPath);
            console.log('File copied:', sourcePath, '->', destPath);
        } catch (error) {
            throw new Error(`Cannot copy file: ${sourcePath} to ${destPath}`);
        }
    }

    // Watch file for changes
    async watchFile(filename, callback) {
        const filePath = path.join(this.basePath, filename);
        
        try {
            fs.watch(filePath, (eventType) => {
                console.log(`File ${filename} changed: ${eventType}`);
                callback(eventType, filename);
            });
            console.log('Watching file:', filePath);
        } catch (error) {
            throw new Error(`Cannot watch file: ${filePath}`);
        }
    }
}

// Usage example
async function demonstrateFileOperations() {
    const fileManager = new FileManager();

    try {
        // Create and manipulate files
        await fileManager.createFile('test.txt', 'Initial content\n');
        await fileManager.appendToFile('test.txt', 'Additional content\n');
        
        const content = await fileManager.readFile('test.txt');
        console.log('Current content:', content);

        const fileInfo = await fileManager.getFileInfo('test.txt');
        console.log('File info:', fileInfo);

        await fileManager.copyFile('test.txt', 'test-backup.txt');
        await fileManager.listFiles();

        // Watch for changes
        await fileManager.watchFile('test.txt', (eventType, filename) => {
            console.log(`Detected ${eventType} on ${filename}`);
        });

        // Simulate change after 2 seconds
        setTimeout(async () => {
            await fileManager.appendToFile('test.txt', 'Change from watcher\n');
        }, 2000);

        // Clean up after 5 seconds
        setTimeout(async () => {
            await fileManager.deleteFile('test.txt');
            await fileManager.deleteFile('test-backup.txt');
        }, 5000);

    } catch (error) {
        console.error('Error:', error.message);
    }
}

// demonstrateFileOperations();

// Streams for large files
const { createReadStream, createWriteStream } = require('fs');

function copyLargeFile(source, destination) {
    return new Promise((resolve, reject) => {
        const readStream = createReadStream(source);
        const writeStream = createWriteStream(destination);

        readStream.on('error', reject);
        writeStream.on('error', reject);
        writeStream.on('finish', resolve);

        readStream.pipe(writeStream);
    });
}

// Example: Process large CSV file
async function processLargeFile() {
    const readline = require('readline');
    const fs = require('fs');

    const fileStream = fs.createReadStream('./large-file.csv');
    const rl = readline.createInterface({
        input: fileStream,
        crlfDelay: Infinity
    });

    let lineCount = 0;
    
    for await (const line of rl) {
        lineCount++;
        // Process each line
        if (lineCount % 1000 === 0) {
            console.log(`Processed ${lineCount} lines`);
        }
    }

    console.log(`Total lines processed: ${lineCount}`);
}

2. Modules & NPM

// CommonJS modules (Node.js default)
// File: math.js
const PI = 3.14159;

function add(a, b) {
    return a + b;
}

function multiply(a, b) {
    return a * b;
}

// Export multiple functions
module.exports = {
    add,
    multiply,
    PI
};

// Alternative export syntax
exports.subtract = (a, b) => a - b;
exports.divide = (a, b) => a / b;

// File: app.js
const math = require('./math');
const { add, multiply } = require('./math');

console.log('5 + 3 =', math.add(5, 3));
console.log('4 * 6 =', multiply(4, 6));
console.log('PI =', math.PI);

// ES6 Modules (with .mjs extension or package.json type: "module")
// File: calculator.mjs
export const version = '1.0.0';

export function add(a, b) {
    return a + b;
}

export function subtract(a, b) {
    return a - b;
}

// Default export
export default function calculator() {
    return {
        add,
        subtract
    };
}

// File: app.mjs
import calc, { add, subtract, version } from './calculator.mjs';

console.log('Calculator version:', version);
console.log('8 + 2 =', add(8, 2));
console.log('8 - 2 =', subtract(8, 2));

// Core module imports
const path = require('path');
const fs = require('fs').promises;
const http = require('http');

// Third-party module imports (after npm install)
// const express = require('express');
// const axios = require('axios');

// Package.json example
/*
{
  "name": "my-node-app",
  "version": "1.0.0",
  "description": "A sample Node.js application",
  "main": "app.js",
  "type": "commonjs",
  "scripts": {
    "start": "node app.js",
    "dev": "nodemon app.js",
    "test": "jest",
    "build": "webpack"
  },
  "keywords": ["node", "javascript", "backend"],
  "author": "Your Name",
  "license": "MIT",
  "dependencies": {
    "express": "^4.18.0",
    "mongoose": "^6.0.0",
    "axios": "^1.0.0"
  },
  "devDependencies": {
    "nodemon": "^2.0.0",
    "jest": "^29.0.0"
  },
  "engines": {
    "node": ">=14.0.0"
  }
}
*/

// NPM commands demonstration
/*
# Initialize new project
npm init
npm init -y  # Skip questions

# Install packages
npm install express
npm install --save-dev nodemon
npm install axios@1.0.0  # Specific version

# Global installation
npm install -g nodemon

# Install from package.json
npm install

# Update packages
npm update
npm outdated  # Check for outdated packages

# Uninstall packages
npm uninstall express

# Run scripts
npm start
npm run dev
npm test

# Publish package
npm publish

# Security audit
npm audit
npm audit fix
*/

// Creating a custom module with classes
// File: user-manager.js
class User {
    constructor(id, name, email) {
        this.id = id;
        this.name = name;
        this.email = email;
        this.createdAt = new Date();
    }

    getInfo() {
        return `${this.name} (${this.email}) - ID: ${this.id}`;
    }
}

class UserManager {
    constructor() {
        this.users = new Map();
        this.nextId = 1;
    }

    addUser(name, email) {
        const id = this.nextId++;
        const user = new User(id, name, email);
        this.users.set(id, user);
        return user;
    }

    getUser(id) {
        return this.users.get(id);
    }

    getAllUsers() {
        return Array.from(this.users.values());
    }

    updateUser(id, updates) {
        const user = this.users.get(id);
        if (user) {
            Object.assign(user, updates);
            return user;
        }
        return null;
    }

    deleteUser(id) {
        return this.users.delete(id);
    }

    findUserByEmail(email) {
        return this.getAllUsers().find(user => user.email === email);
    }

    getUserCount() {
        return this.users.size;
    }
}

module.exports = {
    User,
    UserManager
};

// File: main.js
const { UserManager } = require('./user-manager');

const userManager = new UserManager();

// Add users
userManager.addUser('John Doe', 'john@example.com');
userManager.addUser('Jane Smith', 'jane@example.com');
userManager.addUser('Bob Johnson', 'bob@example.com');

// Get all users
console.log('All users:');
userManager.getAllUsers().forEach(user => {
    console.log(user.getInfo());
});

// Find user by email
const user = userManager.findUserByEmail('jane@example.com');
console.log('Found user:', user?.getInfo());

// Update user
userManager.updateUser(1, { name: 'John Updated' });
console.log('Updated user:', userManager.getUser(1).getInfo());

console.log('Total users:', userManager.getUserCount());

// Singleton pattern for shared state
// File: database-connection.js
class DatabaseConnection {
    constructor() {
        if (DatabaseConnection.instance) {
            return DatabaseConnection.instance;
        }
        
        this.connection = null;
        this.isConnected = false;
        DatabaseConnection.instance = this;
    }

    connect(config) {
        if (this.isConnected) {
            console.log('Already connected to database');
            return;
        }

        // Simulate database connection
        this.connection = {
            host: config.host,
            port: config.port,
            database: config.database
        };
        this.isConnected = true;
        console.log('Connected to database:', config.database);
    }

    disconnect() {
        this.connection = null;
        this.isConnected = false;
        console.log('Disconnected from database');
    }

    query(sql) {
        if (!this.isConnected) {
            throw new Error('Not connected to database');
        }
        console.log('Executing query:', sql);
        // Simulate query execution
        return { rows: [], count: 0 };
    }
}

module.exports = new DatabaseConnection();

// Factory pattern for object creation
// File: logger-factory.js
class ConsoleLogger {
    log(message) {
        console.log(`[CONSOLE] ${new Date().toISOString()}: ${message}`);
    }
}

class FileLogger {
    constructor(filename) {
        this.filename = filename;
    }

    log(message) {
        const fs = require('fs');
        const logMessage = `[FILE] ${new Date().toISOString()}: ${message}\n`;
        fs.appendFileSync(this.filename, logMessage);
    }
}

class DatabaseLogger {
    constructor(connection) {
        this.connection = connection;
    }

    log(message) {
        // Simulate database logging
        console.log(`[DATABASE] Logged: ${message}`);
    }
}

class LoggerFactory {
    static createLogger(type, options = {}) {
        switch (type) {
            case 'console':
                return new ConsoleLogger();
            case 'file':
                return new FileLogger(options.filename || 'app.log');
            case 'database':
                return new DatabaseLogger(options.connection);
            default:
                throw new Error(`Unknown logger type: ${type}`);
        }
    }
}

module.exports = LoggerFactory;

// Dependency Injection Container
// File: container.js
class Container {
    constructor() {
        this.services = new Map();
        this.singletons = new Map();
    }

    register(name, definition, isSingleton = false) {
        this.services.set(name, { definition, isSingleton });
    }

    get(name) {
        const service = this.services.get(name);
        
        if (!service) {
            throw new Error(`Service ${name} not found`);
        }

        if (service.isSingleton) {
            if (!this.singletons.has(name)) {
                this.singletons.set(name, this.createInstance(service.definition));
            }
            return this.singletons.get(name);
        }

        return this.createInstance(service.definition);
    }

    createInstance(definition) {
        if (typeof definition === 'function') {
            return definition(this);
        }
        return definition;
    }
}

// Configuration management
// File: config-manager.js
const fs = require('fs').promises;
const path = require('path');

class ConfigManager {
    constructor() {
        this.config = {};
        this.loaded = false;
    }

    async load(configPath = './config') {
        try {
            // Load default config
            const defaultConfig = await this.loadConfigFile(path.join(configPath, 'default.json'));
            
            // Load environment-specific config
            const env = process.env.NODE_ENV || 'development';
            const envConfig = await this.loadConfigFile(path.join(configPath, `${env}.json`));
            
            // Load local overrides
            const localConfig = await this.loadConfigFile(path.join(configPath, 'local.json'));

            // Merge configurations
            this.config = {
                ...defaultConfig,
                ...envConfig,
                ...localConfig,
                environment: env
            };

            // Override with environment variables
            this.overrideWithEnvVars();

            this.loaded = true;
            console.log('Configuration loaded for environment:', env);
        } catch (error) {
            console.error('Error loading configuration:', error.message);
            throw error;
        }
    }

    async loadConfigFile(filePath) {
        try {
            const data = await fs.readFile(filePath, 'utf8');
            return JSON.parse(data);
        } catch (error) {
            if (error.code === 'ENOENT') {
                return {};
            }
            throw error;
        }
    }

    overrideWithEnvVars() {
        for (const key in this.config) {
            const envKey = key.toUpperCase().replace(/[^A-Z0-9]/g, '_');
            if (process.env[envKey]) {
                this.config[key] = this.parseEnvValue(process.env[envKey]);
            }
        }
    }

    parseEnvValue(value) {
        // Try to parse as JSON, otherwise return as string
        try {
            return JSON.parse(value);
        } catch {
            return value;
        }
    }

    get(key, defaultValue = null) {
        if (!this.loaded) {
            throw new Error('Configuration not loaded. Call load() first.');
        }

        const keys = key.split('.');
        let value = this.config;

        for (const k of keys) {
            if (value && typeof value === 'object' && k in value) {
                value = value[k];
            } else {
                return defaultValue;
            }
        }

        return value;
    }

    set(key, value) {
        const keys = key.split('.');
        let current = this.config;

        for (let i = 0; i < keys.length - 1; i++) {
            const k = keys[i];
            if (!(k in current) || typeof current[k] !== 'object') {
                current[k] = {};
            }
            current = current[k];
        }

        current[keys[keys.length - 1]] = value;
    }
}

module.exports = new ConfigManager();

// Usage examples
async function demonstratePatterns() {
    // Singleton example
    const db1 = require('./database-connection');
    const db2 = require('./database-connection');
    
    db1.connect({ host: 'localhost', port: 5432, database: 'mydb' });
    console.log('Same instance?', db1 === db2); // true

    // Factory example
    const LoggerFactory = require('./logger-factory');
    
    const consoleLogger = LoggerFactory.createLogger('console');
    const fileLogger = LoggerFactory.createLogger('file', { filename: 'app.log' });
    
    consoleLogger.log('This goes to console');
    fileLogger.log('This goes to file');

    // Dependency Injection example
    const container = new Container();
    
    // Register services
    container.register('logger', () => LoggerFactory.createLogger('console'));
    container.register('database', (c) => {
        const db = c.get('databaseConnection');
        db.connect({ host: 'localhost', database: 'app' });
        return db;
    }, true);
    
    container.register('databaseConnection', () => require('./database-connection'));

    const logger = container.get('logger');
    const database = container.get('database');
    
    logger.log('Application started');
    database.query('SELECT * FROM users');

    // Configuration example
    const config = require('./config-manager');
    await config.load();
    
    console.log('Database host:', config.get('database.host'));
    console.log('API port:', config.get('server.port', 3000));
}

// demonstratePatterns();

3. Asynchronous Programming

// Callback-based approach (traditional)
const fs = require('fs');

function readFileCallback(filename, callback) {
    fs.readFile(filename, 'utf8', (err, data) => {
        if (err) {
            callback(err, null);
            return;
        }
        callback(null, data);
    });
}

// Using callbacks (callback hell example)
readFileCallback('file1.txt', (err, data1) => {
    if (err) {
        console.error('Error reading file1:', err);
        return;
    }
    
    readFileCallback('file2.txt', (err, data2) => {
        if (err) {
            console.error('Error reading file2:', err);
            return;
        }
        
        readFileCallback('file3.txt', (err, data3) => {
            if (err) {
                console.error('Error reading file3:', err);
                return;
            }
            
            console.log('All files read:', { data1, data2, data3 });
        });
    });
});

// Promise-based approach
function readFilePromise(filename) {
    return new Promise((resolve, reject) => {
        fs.readFile(filename, 'utf8', (err, data) => {
            if (err) {
                reject(err);
                return;
            }
            resolve(data);
        });
    });
}

// Using promises
readFilePromise('file1.txt')
    .then(data1 => {
        console.log('File1 content:', data1);
        return readFilePromise('file2.txt');
    })
    .then(data2 => {
        console.log('File2 content:', data2);
        return readFilePromise('file3.txt');
    })
    .then(data3 => {
        console.log('File3 content:', data3);
    })
    .catch(error => {
        console.error('Error reading files:', error);
    });

// Promise.all for parallel execution
Promise.all([
    readFilePromise('file1.txt'),
    readFilePromise('file2.txt'),
    readFilePromise('file3.txt')
])
.then(([data1, data2, data3]) => {
    console.log('All files read in parallel:', { data1, data2, data3 });
})
.catch(error => {
    console.error('Error in parallel read:', error);
});

// Async/await approach (modern)
async function readFilesAsync() {
    try {
        const data1 = await readFilePromise('file1.txt');
        const data2 = await readFilePromise('file2.txt');
        const data3 = await readFilePromise('file3.txt');
        
        console.log('All files read with async/await:', { data1, data2, data3 });
    } catch (error) {
        console.error('Error reading files:', error);
    }
}

// Parallel execution with async/await
async function readFilesParallel() {
    try {
        const [data1, data2, data3] = await Promise.all([
            readFilePromise('file1.txt'),
            readFilePromise('file2.txt'),
            readFilePromise('file3.txt')
        ]);
        
        console.log('Files read in parallel:', { data1, data2, data3 });
    } catch (error) {
        console.error('Error in parallel read:', error);
    }
}

// Advanced promise patterns
class AsyncOperations {
    // Retry pattern with exponential backoff
    static async retry(operation, maxRetries = 3, delay = 1000) {
        for (let attempt = 1; attempt <= maxRetries; attempt++) {
            try {
                return await operation();
            } catch (error) {
                if (attempt === maxRetries) {
                    throw error;
                }
                
                const waitTime = delay * Math.pow(2, attempt - 1);
                console.log(`Attempt ${attempt} failed. Retrying in ${waitTime}ms...`);
                
                await new Promise(resolve => setTimeout(resolve, waitTime));
            }
        }
    }

    // Timeout pattern
    static async withTimeout(promise, timeoutMs, timeoutMessage = 'Operation timed out') {
        let timeoutId;
        
        const timeoutPromise = new Promise((_, reject) => {
            timeoutId = setTimeout(() => reject(new Error(timeoutMessage)), timeoutMs);
        });

        try {
            return await Promise.race([promise, timeoutPromise]);
        } finally {
            clearTimeout(timeoutId);
        }
    }

    // Batch processing
    static async processInBatches(items, batchSize, processor) {
        const results = [];
        
        for (let i = 0; i < items.length; i += batchSize) {
            const batch = items.slice(i, i + batchSize);
            const batchPromises = batch.map(item => processor(item));
            const batchResults = await Promise.all(batchPromises);
            results.push(...batchResults);
            
            console.log(`Processed batch ${Math.floor(i / batchSize) + 1}`);
        }
        
        return results;
    }

    // Sequential processing with concurrency control
    static async processWithConcurrency(items, concurrency, processor) {
        const results = [];
        const executing = new Set();
        
        for (const item of items) {
            if (executing.size >= concurrency) {
                await Promise.race(executing);
            }
            
            const promise = processor(item).then(result => {
                executing.delete(promise);
                return result;
            });
            
            executing.add(promise);
            results.push(promise);
        }
        
        return Promise.all(results);
    }
}

// Event Emitter for complex async workflows
const EventEmitter = require('events');

class DataProcessor extends EventEmitter {
    constructor() {
        super();
        this.data = [];
        this.processing = false;
    }

    addData(item) {
        this.data.push(item);
        this.emit('dataAdded', item);
        
        if (this.data.length >= 10 && !this.processing) {
            this.processBatch();
        }
    }

    async processBatch() {
        if (this.processing) return;
        
        this.processing = true;
        this.emit('processingStarted', this.data.length);

        try {
            // Simulate async processing
            await new Promise(resolve => setTimeout(resolve, 1000));
            
            const results = this.data.map(item => `Processed: ${item}`);
            this.emit('processingCompleted', results);
            
            this.data = [];
        } catch (error) {
            this.emit('processingError', error);
        } finally {
            this.processing = false;
        }
    }
}

// Usage example
async function demonstrateAsyncPatterns() {
    const processor = new DataProcessor();

    processor.on('dataAdded', (item) => {
        console.log('Data added:', item);
    });

    processor.on('processingStarted', (count) => {
        console.log(`Processing started with ${count} items`);
    });

    processor.on('processingCompleted', (results) => {
        console.log('Processing completed:', results);
    });

    // Add data
    for (let i = 1; i <= 15; i++) {
        processor.addData(`Item ${i}`);
        await new Promise(resolve => setTimeout(resolve, 100));
    }

    // Demonstrate retry pattern
    let attempt = 0;
    const flakyOperation = () => {
        attempt++;
        return new Promise((resolve, reject) => {
            if (attempt < 3) {
                reject(new Error('Temporary failure'));
            } else {
                resolve('Success on attempt ' + attempt);
            }
        });
    };

    try {
        const result = await AsyncOperations.retry(flakyOperation, 5);
        console.log('Retry result:', result);
    } catch (error) {
        console.error('All retries failed:', error.message);
    }

    // Demonstrate timeout
    try {
        const slowOperation = new Promise(resolve => setTimeout(() => resolve('Done'), 5000));
        const result = await AsyncOperations.withTimeout(slowOperation, 2000, 'Too slow!');
        console.log('Timeout result:', result);
    } catch (error) {
        console.error('Timeout error:', error.message);
    }

    // Demonstrate batch processing
    const items = Array.from({ length: 25 }, (_, i) => i + 1);
    const results = await AsyncOperations.processInBatches(items, 5, async (item) => {
        await new Promise(resolve => setTimeout(resolve, 100));
        return item * 2;
    });
    console.log('Batch processing results:', results);
}

// demonstrateAsyncPatterns();

// Streams in Node.js
const fs = require('fs');
const { Transform, Readable, Writable, Duplex } = require('stream');
const zlib = require('zlib');

// Readable stream example
class NumberStream extends Readable {
    constructor(maxNumbers, options = {}) {
        super(options);
        this.maxNumbers = maxNumbers;
        this.currentNumber = 1;
    }

    _read() {
        if (this.currentNumber > this.maxNumbers) {
            this.push(null); // End stream
            return;
        }

        const number = this.currentNumber++;
        const chunk = `Number: ${number}\n`;
        
        // Push data to the stream
        if (!this.push(chunk)) {
            // Stream buffer is full, wait for drain
            console.log('Stream buffer full, pausing...');
        }
    }
}

// Writable stream example
class FileWriter extends Writable {
    constructor(filename, options = {}) {
        super(options);
        this.filename = filename;
    }

    _write(chunk, encoding, callback) {
        fs.appendFile(this.filename, chunk, (err) => {
            if (err) {
                callback(err);
                return;
            }
            console.log('Written chunk of size:', chunk.length);
            callback();
        });
    }

    _final(callback) {
        console.log('File writing completed');
        callback();
    }
}

// Transform stream example
class UpperCaseTransform extends Transform {
    _transform(chunk, encoding, callback) {
        const upperChunk = chunk.toString().toUpperCase();
        this.push(upperChunk);
        callback();
    }
}

// Duplex stream example
class EchoDuplex extends Duplex {
    _write(chunk, encoding, callback) {
        console.log('Received:', chunk.toString());
        this.push(chunk); // Echo back
        callback();
    }

    _read() {
        // No-op for this example
    }
}

// Using streams
function demonstrateStreams() {
    // Create streams
    const numberStream = new NumberStream(10);
    const upperCaseTransform = new UpperCaseTransform();
    const fileWriter = new FileWriter('output.txt');
    const echoDuplex = new EchoDuplex();

    // Pipe streams together
    numberStream
        .pipe(upperCaseTransform)
        .pipe(echoDuplex)
        .pipe(fileWriter);

    // Handle stream events
    numberStream.on('end', () => {
        console.log('Number stream ended');
    });

    numberStream.on('error', (error) => {
        console.error('Stream error:', error);
    });

    fileWriter.on('finish', () => {
        console.log('File writing finished');
    });
}

// demonstrateStreams();

// Working with buffers
function demonstrateBuffers() {
    // Creating buffers
    const buffer1 = Buffer.from('Hello World');
    const buffer2 = Buffer.alloc(10); // 10 bytes filled with zeros
    const buffer3 = Buffer.allocUnsafe(10); // 10 bytes, may contain old data

    console.log('Buffer 1:', buffer1);
    console.log('Buffer 1 as string:', buffer1.toString());
    console.log('Buffer 1 length:', buffer1.length);
    console.log('Buffer 1 hex:', buffer1.toString('hex'));

    // Buffer operations
    const partial = buffer1.slice(0, 5);
    console.log('Partial buffer:', partial.toString());

    // Concatenating buffers
    const buffer4 = Buffer.from('Node.js');
    const combined = Buffer.concat([buffer1, Buffer.from(' '), buffer4]);
    console.log('Combined:', combined.toString());

    // Comparing buffers
    const buffer5 = Buffer.from('Hello World');
    console.log('Buffers equal:', buffer1.equals(buffer5));

    // Writing to buffers
    const buffer6 = Buffer.alloc(50);
    buffer6.write('Hello Buffer!', 0, 'utf8');
    console.log('Written buffer:', buffer6.toString());

    // Iterating over buffer
    for (const byte of buffer1) {
        process.stdout.write(byte.toString(16) + ' ');
    }
    console.log();

    // Buffer and JSON
    const obj = { data: buffer1 };
    const json = JSON.stringify(obj);
    console.log('JSON with buffer:', json);

    const parsed = JSON.parse(json);
    const bufferFromJson = Buffer.from(parsed.data);
    console.log('Buffer from JSON:', bufferFromJson.toString());
}

// demonstrateBuffers();

// Real-world stream example: File compression
function compressFile(inputFile, outputFile) {
    return new Promise((resolve, reject) => {
        const readStream = fs.createReadStream(inputFile);
        const writeStream = fs.createWriteStream(outputFile);
        const gzip = zlib.createGzip();

        readStream
            .pipe(gzip)
            .pipe(writeStream)
            .on('finish', resolve)
            .on('error', reject);
    });
}

// Real-world example: CSV processing
const { parse } = require('csv-parse');
const { stringify } = require('csv-stringify');

function processCSV(inputFile, outputFile) {
    return new Promise((resolve, reject) => {
        const readStream = fs.createReadStream(inputFile);
        const writeStream = fs.createWriteStream(outputFile);

        const csvParser = parse({ columns: true });
        const csvStringifier = stringify({ header: true });

        const transformStream = new Transform({
            objectMode: true,
            transform(chunk, encoding, callback) {
                // Process each CSV row
                const processed = {
                    ...chunk,
                    processedAt: new Date().toISOString(),
                    name: chunk.name?.toUpperCase()
                };
                callback(null, processed);
            }
        });

        readStream
            .pipe(csvParser)
            .pipe(transformStream)
            .pipe(csvStringifier)
            .pipe(writeStream)
            .on('finish', resolve)
            .on('error', reject);
    });
}

// Backpressure handling
class ControlledStream extends Readable {
    constructor(options = {}) {
        super(options);
        this.data = Array.from({ length: 100 }, (_, i) => `Data ${i}`);
        this.index = 0;
    }

    _read(size) {
        // Simulate slow data production
        setTimeout(() => {
            if (this.index >= this.data.length) {
                this.push(null);
                return;
            }

            const chunk = this.data[this.index++];
            if (!this.push(chunk)) {
                console.log('Backpressure: consumer cannot handle more data');
                // In real scenario, you might want to pause production
            }
        }, 100);
    }
}

// Custom writable with backpressure handling
class ControlledWriter extends Writable {
    constructor(options = {}) {
        super(options);
        this.writeCount = 0;
    }

    _write(chunk, encoding, callback) {
        this.writeCount++;
        
        // Simulate slow processing
        setTimeout(() => {
            console.log(`Processed ${this.writeCount}: ${chunk.toString()}`);
            callback();
        }, 200);
    }
}

function demonstrateBackpressure() {
    const producer = new ControlledStream();
    const consumer = new ControlledWriter();

    producer.pipe(consumer);
}

// demonstrateBackpressure();

// Stream utilities
class StreamUtils {
    static createReadableFromArray(array) {
        return Readable.from(array);
    }

    static createWritableToFunction(processFn) {
        return new Writable({
            objectMode: true,
            write(chunk, encoding, callback) {
                processFn(chunk);
                callback();
            }
        });
    }

    static createTransform(transformFn) {
        return new Transform({
            objectMode: true,
            transform(chunk, encoding, callback) {
                try {
                    const result = transformFn(chunk);
                    callback(null, result);
                } catch (error) {
                    callback(error);
                }
            }
        });
    }
}

// Usage example
async function demonstrateStreamUtilities() {
    const data = [1, 2, 3, 4, 5];
    
    const readable = StreamUtils.createReadableFromArray(data);
    const transform = StreamUtils.createTransform(n => n * 2);
    const writable = StreamUtils.createWritableToFunction(console.log);

    readable.pipe(transform).pipe(writable);
}

// demonstrateStreamUtilities();

4. Web Development

// Basic HTTP server with Node.js
const http = require('http');
const url = require('url');
const querystring = require('querystring');

const server = http.createServer((req, res) => {
    const parsedUrl = url.parse(req.url, true);
    const pathname = parsedUrl.pathname;
    const query = parsedUrl.query;
    
    // Set CORS headers
    res.setHeader('Access-Control-Allow-Origin', '*');
    res.setHeader('Access-Control-Allow-Methods', 'GET, POST, PUT, DELETE, OPTIONS');
    res.setHeader('Access-Control-Allow-Headers', 'Content-Type, Authorization');

    // Handle preflight requests
    if (req.method === 'OPTIONS') {
        res.writeHead(200);
        res.end();
        return;
    }

    // Router
    if (pathname === '/' && req.method === 'GET') {
        res.writeHead(200, { 'Content-Type': 'text/html' });
        res.end(\`
            <html>
                <body>
                    <h1>Welcome to Node.js Server</h1>
                    <p>Current time: ${new Date().toISOString()}</p>
                </body>
            </html>
        \`);
    } else if (pathname === '/api/users' && req.method === 'GET') {
        res.writeHead(200, { 'Content-Type': 'application/json' });
        res.end(JSON.stringify({
            users: [
                { id: 1, name: 'John Doe', email: 'john@example.com' },
                { id: 2, name: 'Jane Smith', email: 'jane@example.com' }
            ]
        }));
    } else if (pathname === '/api/users' && req.method === 'POST') {
        let body = '';
        
        req.on('data', chunk => {
            body += chunk.toString();
        });
        
        req.on('end', () => {
            try {
                const userData = JSON.parse(body);
                res.writeHead(201, { 'Content-Type': 'application/json' });
                res.end(JSON.stringify({
                    message: 'User created successfully',
                    user: { id: Date.now(), ...userData }
                }));
            } catch (error) {
                res.writeHead(400, { 'Content-Type': 'application/json' });
                res.end(JSON.stringify({ error: 'Invalid JSON' }));
            }
        });
    } else {
        res.writeHead(404, { 'Content-Type': 'application/json' });
        res.end(JSON.stringify({ error: 'Route not found' }));
    }
});

// server.listen(3000, () => {
//     console.log('Server running on http://localhost:3000');
// });

// Express.js web application
const express = require('express');
const path = require('path');

const app = express();

// Middleware
app.use(express.json()); // Parse JSON bodies
app.use(express.urlencoded({ extended: true })); // Parse URL-encoded bodies
app.use(express.static('public')); // Serve static files

// Custom middleware
app.use((req, res, next) => {
    console.log(`${new Date().toISOString()} - ${req.method} ${req.path}`);
    next();
});

// Authentication middleware
const authMiddleware = (req, res, next) => {
    const token = req.headers.authorization;
    
    if (!token || !token.startsWith('Bearer ')) {
        return res.status(401).json({ error: 'Authentication required' });
    }
    
    // In real app, validate JWT token
    const user = { id: 1, name: 'John Doe' };
    req.user = user;
    next();
};

// Routes
app.get('/', (req, res) => {
    res.send(\`
        <html>
            <head>
                <title>Express App</title>
                <link href="/styles.css" rel="stylesheet">
            </head>
            <body>
                <h1>Welcome to Express.js</h1>
                <p>This is a sample Express application</p>
            </body>
        </html>
    \`);
});

// API Routes
const router = express.Router();

// GET /api/users
router.get('/users', (req, res) => {
    const users = [
        { id: 1, name: 'John Doe', email: 'john@example.com' },
        { id: 2, name: 'Jane Smith', email: 'jane@example.com' },
        { id: 3, name: 'Bob Johnson', email: 'bob@example.com' }
    ];
    
    // Filter and paginate
    const page = parseInt(req.query.page) || 1;
    const limit = parseInt(req.query.limit) || 10;
    const startIndex = (page - 1) * limit;
    const endIndex = startIndex + limit;
    
    const result = {
        page,
        limit,
        total: users.length,
        data: users.slice(startIndex, endIndex)
    };
    
    res.json(result);
});

// GET /api/users/:id
router.get('/users/:id', (req, res) => {
    const userId = parseInt(req.params.id);
    const users = [
        { id: 1, name: 'John Doe', email: 'john@example.com' },
        { id: 2, name: 'Jane Smith', email: 'jane@example.com' }
    ];
    
    const user = users.find(u => u.id === userId);
    
    if (!user) {
        return res.status(404).json({ error: 'User not found' });
    }
    
    res.json(user);
});

// POST /api/users
router.post('/users', (req, res) => {
    const { name, email } = req.body;
    
    if (!name || !email) {
        return res.status(400).json({ error: 'Name and email are required' });
    }
    
    const newUser = {
        id: Date.now(),
        name,
        email,
        createdAt: new Date().toISOString()
    };
    
    res.status(201).json(newUser);
});

// PUT /api/users/:id
router.put('/users/:id', authMiddleware, (req, res) => {
    const userId = parseInt(req.params.id);
    const { name, email } = req.body;
    
    if (!name || !email) {
        return res.status(400).json({ error: 'Name and email are required' });
    }
    
    const updatedUser = {
        id: userId,
        name,
        email,
        updatedAt: new Date().toISOString(),
        updatedBy: req.user.name
    };
    
    res.json(updatedUser);
});

// DELETE /api/users/:id
router.delete('/users/:id', authMiddleware, (req, res) => {
    const userId = parseInt(req.params.id);
    
    res.json({ message: `User ${userId} deleted successfully` });
});

// Mount router
app.use('/api', router);

// Error handling middleware
app.use((err, req, res, next) => {
    console.error('Error:', err.stack);
    
    if (err instanceof SyntaxError && err.status === 400 && 'body' in err) {
        return res.status(400).json({ error: 'Invalid JSON' });
    }
    
    res.status(500).json({ error: 'Something went wrong!' });
});

// 404 handler
app.use((req, res) => {
    res.status(404).json({ error: 'Route not found' });
});

// Start server
// app.listen(3000, () => {
//     console.log('Express server running on http://localhost:3000');
// });

// Advanced Express features
class ExpressApp {
    constructor() {
        this.app = express();
        this.setupMiddleware();
        this.setupRoutes();
        this.setupErrorHandling();
    }

    setupMiddleware() {
        // Security middleware
        this.app.use(require('helmet')());
        this.app.use(require('cors')());
        
        // Rate limiting
        const rateLimit = require('express-rate-limit');
        const limiter = rateLimit({
            windowMs: 15 * 60 * 1000, // 15 minutes
            max: 100, // limit each IP to 100 requests per windowMs
            message: 'Too many requests from this IP'
        });
        this.app.use(limiter);
        
        // Body parsing
        this.app.use(express.json({ limit: '10mb' }));
        this.app.use(express.urlencoded({ extended: true }));
        
        // Static files
        this.app.use(express.static('public'));
        
        // Logging
        const morgan = require('morgan');
        this.app.use(morgan('combined'));
    }

    setupRoutes() {
        // Health check
        this.app.get('/health', (req, res) => {
            res.json({ 
                status: 'OK', 
                timestamp: new Date().toISOString(),
                uptime: process.uptime()
            });
        });

        // API versioning
        const v1Router = express.Router();
        this.setupV1Routes(v1Router);
        this.app.use('/api/v1', v1Router);
    }

    setupV1Routes(router) {
        // Sample CRUD operations for products
        const products = [];
        
        router.get('/products', (req, res) => {
            res.json({ products });
        });
        
        router.post('/products', (req, res) => {
            const product = {
                id: products.length + 1,
                ...req.body,
                createdAt: new Date().toISOString()
            };
            products.push(product);
            res.status(201).json(product);
        });
        
        router.get('/products/:id', (req, res) => {
            const product = products.find(p => p.id === parseInt(req.params.id));
            if (!product) {
                return res.status(404).json({ error: 'Product not found' });
            }
            res.json(product);
        });
    }

    setupErrorHandling() {
        // 404 handler
        this.app.use((req, res) => {
            res.status(404).json({ error: 'Route not found' });
        });

        // Global error handler
        this.app.use((err, req, res, next) => {
            console.error('Unhandled error:', err);
            
            const response = {
                error: 'Internal server error',
                ...(process.env.NODE_ENV === 'development' && { stack: err.stack })
            };
            
            res.status(err.status || 500).json(response);
        });
    }

    start(port = 3000) {
        return new Promise((resolve, reject) => {
            this.app.listen(port, () => {
                console.log(`Server running on port ${port}`);
                resolve();
            }).on('error', reject);
        });
    }
}

// Usage
// const myApp = new ExpressApp();
// myApp.start(3000);

// RESTful API with Express
const express = require('express');
const app = express();

app.use(express.json());

// In-memory database for demonstration
let books = [
    { id: 1, title: 'The Great Gatsby', author: 'F. Scott Fitzgerald', year: 1925 },
    { id: 2, title: 'To Kill a Mockingbird', author: 'Harper Lee', year: 1960 },
    { id: 3, title: '1984', author: 'George Orwell', year: 1949 }
];

// GET /books - Get all books with filtering and pagination
app.get('/books', (req, res) => {
    let filteredBooks = [...books];
    
    // Filter by author if provided
    if (req.query.author) {
        filteredBooks = filteredBooks.filter(book => 
            book.author.toLowerCase().includes(req.query.author.toLowerCase())
        );
    }
    
    // Filter by year if provided
    if (req.query.year) {
        filteredBooks = filteredBooks.filter(book => book.year == req.query.year);
    }
    
    // Pagination
    const page = parseInt(req.query.page) || 1;
    const limit = parseInt(req.query.limit) || 10;
    const startIndex = (page - 1) * limit;
    const endIndex = startIndex + limit;
    
    const result = {
        page,
        limit,
        total: filteredBooks.length,
        totalPages: Math.ceil(filteredBooks.length / limit),
        data: filteredBooks.slice(startIndex, endIndex)
    };
    
    res.json(result);
});

// GET /books/:id - Get single book
app.get('/books/:id', (req, res) => {
    const book = books.find(b => b.id === parseInt(req.params.id));
    
    if (!book) {
        return res.status(404).json({ error: 'Book not found' });
    }
    
    res.json(book);
});

// POST /books - Create new book
app.post('/books', (req, res) => {
    const { title, author, year } = req.body;
    
    // Validation
    if (!title || !author || !year) {
        return res.status(400).json({ 
            error: 'Title, author, and year are required' 
        });
    }
    
    const newBook = {
        id: books.length > 0 ? Math.max(...books.map(b => b.id)) + 1 : 1,
        title,
        author,
        year: parseInt(year),
        createdAt: new Date().toISOString()
    };
    
    books.push(newBook);
    res.status(201).json(newBook);
});

// PUT /books/:id - Update entire book
app.put('/books/:id', (req, res) => {
    const bookIndex = books.findIndex(b => b.id === parseInt(req.params.id));
    
    if (bookIndex === -1) {
        return res.status(404).json({ error: 'Book not found' });
    }
    
    const { title, author, year } = req.body;
    
    if (!title || !author || !year) {
        return res.status(400).json({ 
            error: 'Title, author, and year are required' 
        });
    }
    
    books[bookIndex] = {
        ...books[bookIndex],
        title,
        author,
        year: parseInt(year),
        updatedAt: new Date().toISOString()
    };
    
    res.json(books[bookIndex]);
});

// PATCH /books/:id - Partial update
app.patch('/books/:id', (req, res) => {
    const bookIndex = books.findIndex(b => b.id === parseInt(req.params.id));
    
    if (bookIndex === -1) {
        return res.status(404).json({ error: 'Book not found' });
    }
    
    books[bookIndex] = {
        ...books[bookIndex],
        ...req.body,
        updatedAt: new Date().toISOString()
    };
    
    res.json(books[bookIndex]);
});

// DELETE /books/:id - Delete book
app.delete('/books/:id', (req, res) => {
    const bookIndex = books.findIndex(b => b.id === parseInt(req.params.id));
    
    if (bookIndex === -1) {
        return res.status(404).json({ error: 'Book not found' });
    }
    
    const deletedBook = books.splice(bookIndex, 1)[0];
    res.json({ message: 'Book deleted successfully', book: deletedBook });
});

// WebSocket server with Socket.io
const http = require('http');
const socketIo = require('socket.io');

class ChatServer {
    constructor() {
        this.app = express();
        this.server = http.createServer(this.app);
        this.io = socketIo(this.server, {
            cors: {
                origin: "*",
                methods: ["GET", "POST"]
            }
        });
        
        this.users = new Map(); // socket.id -> user data
        this.rooms = new Map(); // room name -> users in room
        
        this.setupExpress();
        this.setupSocketIO();
    }

    setupExpress() {
        this.app.use(express.static('public'));
        
        this.app.get('/', (req, res) => {
            res.sendFile(__dirname + '/public/index.html');
        });
        
        this.app.get('/api/users', (req, res) => {
            res.json(Array.from(this.users.values()));
        });
        
        this.app.get('/api/rooms', (req, res) => {
            const roomsInfo = Array.from(this.rooms.entries()).map(([name, users]) => ({
                name,
                userCount: users.size,
                users: Array.from(users.values())
            }));
            res.json(roomsInfo);
        });
    }

    setupSocketIO() {
        this.io.on('connection', (socket) => {
            console.log('User connected:', socket.id);

            // User joins the server
            socket.on('user_join', (userData) => {
                const user = {
                    id: socket.id,
                    username: userData.username,
                    joinedAt: new Date().toISOString()
                };
                
                this.users.set(socket.id, user);
                
                // Notify all clients
                socket.broadcast.emit('user_joined', user);
                socket.emit('user_list', Array.from(this.users.values()));
                
                console.log(`User ${user.username} joined`);
            });

            // Join room
            socket.on('join_room', (roomName) => {
                socket.join(roomName);
                
                if (!this.rooms.has(roomName)) {
                    this.rooms.set(roomName, new Map());
                }
                
                const user = this.users.get(socket.id);
                if (user) {
                    this.rooms.get(roomName).set(socket.id, user);
                }
                
                socket.to(roomName).emit('user_joined_room', {
                    user: this.users.get(socket.id),
                    room: roomName
                });
                
                socket.emit('room_users', {
                    room: roomName,
                    users: Array.from(this.rooms.get(roomName).values())
                });
            });

            // Leave room
            socket.on('leave_room', (roomName) => {
                socket.leave(roomName);
                
                if (this.rooms.has(roomName)) {
                    this.rooms.get(roomName).delete(socket.id);
                    
                    socket.to(roomName).emit('user_left_room', {
                        user: this.users.get(socket.id),
                        room: roomName
                    });
                }
            });

            // Send message
            socket.on('send_message', (data) => {
                const message = {
                    id: Date.now(),
                    user: this.users.get(socket.id),
                    content: data.content,
                    room: data.room,
                    timestamp: new Date().toISOString(),
                    type: data.type || 'text'
                };
                
                if (data.room) {
                    // Room message
                    socket.to(data.room).emit('new_message', message);
                } else {
                    // Broadcast to all
                    socket.broadcast.emit('new_message', message);
                }
                
                // Echo back to sender
                socket.emit('new_message', message);
            });

            // Typing indicators
            socket.on('typing_start', (data) => {
                socket.to(data.room).emit('user_typing', {
                    user: this.users.get(socket.id),
                    room: data.room
                });
            });

            socket.on('typing_stop', (data) => {
                socket.to(data.room).emit('user_stopped_typing', {
                    user: this.users.get(socket.id),
                    room: data.room
                });
            });

            // Handle disconnection
            socket.on('disconnect', () => {
                const user = this.users.get(socket.id);
                
                if (user) {
                    // Remove from all rooms
                    this.rooms.forEach((users, roomName) => {
                        if (users.has(socket.id)) {
                            users.delete(socket.id);
                            socket.to(roomName).emit('user_left_room', {
                                user,
                                room: roomName
                            });
                        }
                    });
                    
                    // Remove from users
                    this.users.delete(socket.id);
                    
                    // Notify all clients
                    socket.broadcast.emit('user_left', user);
                    
                    console.log(`User ${user.username} disconnected`);
                }
            });
        });
    }

    start(port = 3000) {
        this.server.listen(port, () => {
            console.log(`Chat server running on port ${port}`);
        });
    }
}

// Real-time collaboration example
class CollaborationServer {
    constructor() {
        this.app = express();
        this.server = http.createServer(this.app);
        this.io = socketIo(this.server);
        
        this.documents = new Map(); // documentId -> content
        this.setupSocketIO();
    }

    setupSocketIO() {
        this.io.on('connection', (socket) => {
            console.log('Client connected for collaboration:', socket.id);

            // Join document room
            socket.on('join_document', (documentId) => {
                socket.join(documentId);
                
                if (!this.documents.has(documentId)) {
                    this.documents.set(documentId, '');
                }
                
                // Send current document state to new client
                socket.emit('document_state', {
                    content: this.documents.get(documentId),
                    documentId
                });
                
                console.log(`Client ${socket.id} joined document ${documentId}`);
            });

            // Handle text changes
            socket.on('text_change', (data) => {
                const { documentId, changes, version } = data;
                
                if (this.documents.has(documentId)) {
                    // Apply changes (simplified - in real app use operational transforms)
                    this.documents.set(documentId, changes);
                    
                    // Broadcast to other clients in the same document
                    socket.to(documentId).emit('text_update', {
                        changes,
                        version,
                        from: socket.id
                    });
                }
            });

            // Cursor position
            socket.on('cursor_move', (data) => {
                const { documentId, position } = data;
                
                socket.to(documentId).emit('cursor_update', {
                    userId: socket.id,
                    position
                });
            });

            // Selection changes
            socket.on('selection_change', (data) => {
                const { documentId, selection } = data;
                
                socket.to(documentId).emit('selection_update', {
                    userId: socket.id,
                    selection
                });
            });

            // Leave document
            socket.on('leave_document', (documentId) => {
                socket.leave(documentId);
                socket.to(documentId).emit('user_left_document', {
                    userId: socket.id
                });
            });
        });
    }

    start(port = 3001) {
        this.server.listen(port, () => {
            console.log(`Collaboration server running on port ${port}`);
        });
    }
}

// Usage examples
// const chatServer = new ChatServer();
// chatServer.start(3000);

// const collaborationServer = new CollaborationServer();
// collaborationServer.start(3001);

// REST API with authentication
const jwt = require('jsonwebtoken');
const bcrypt = require('bcrypt');

class AuthAPI {
    constructor() {
        this.app = express();
        this.secretKey = process.env.JWT_SECRET || 'your-secret-key';
        this.users = new Map(); // In production, use database
        
        this.setupRoutes();
    }

    setupRoutes() {
        this.app.use(express.json());

        // Register
        this.app.post('/auth/register', async (req, res) => {
            try {
                const { username, email, password } = req.body;
                
                if (!username || !email || !password) {
                    return res.status(400).json({ error: 'All fields are required' });
                }
                
                if (this.users.has(email)) {
                    return res.status(409).json({ error: 'User already exists' });
                }
                
                // Hash password
                const hashedPassword = await bcrypt.hash(password, 10);
                
                const user = {
                    id: Date.now(),
                    username,
                    email,
                    password: hashedPassword,
                    createdAt: new Date().toISOString()
                };
                
                this.users.set(email, user);
                
                // Generate JWT token
                const token = jwt.sign(
                    { userId: user.id, email: user.email },
                    this.secretKey,
                    { expiresIn: '24h' }
                );
                
                res.status(201).json({
                    message: 'User registered successfully',
                    token,
                    user: { id: user.id, username: user.username, email: user.email }
                });
            } catch (error) {
                res.status(500).json({ error: 'Internal server error' });
            }
        });

        // Login
        this.app.post('/auth/login', async (req, res) => {
            try {
                const { email, password } = req.body;
                
                if (!email || !password) {
                    return res.status(400).json({ error: 'Email and password are required' });
                }
                
                const user = this.users.get(email);
                if (!user) {
                    return res.status(401).json({ error: 'Invalid credentials' });
                }
                
                // Verify password
                const isValidPassword = await bcrypt.compare(password, user.password);
                if (!isValidPassword) {
                    return res.status(401).json({ error: 'Invalid credentials' });
                }
                
                // Generate JWT token
                const token = jwt.sign(
                    { userId: user.id, email: user.email },
                    this.secretKey,
                    { expiresIn: '24h' }
                );
                
                res.json({
                    message: 'Login successful',
                    token,
                    user: { id: user.id, username: user.username, email: user.email }
                });
            } catch (error) {
                res.status(500).json({ error: 'Internal server error' });
            }
        });

        // Protected route example
        this.app.get('/auth/profile', this.authenticateToken, (req, res) => {
            res.json({ user: req.user });
        });

        // Middleware to verify JWT token
        this.authenticateToken = (req, res, next) => {
            const authHeader = req.headers['authorization'];
            const token = authHeader && authHeader.split(' ')[1]; // Bearer TOKEN
            
            if (!token) {
                return res.status(401).json({ error: 'Access token required' });
            }
            
            jwt.verify(token, this.secretKey, (err, user) => {
                if (err) {
                    return res.status(403).json({ error: 'Invalid or expired token' });
                }
                
                req.user = user;
                next();
            });
        };
    }

    start(port = 3002) {
        this.app.listen(port, () => {
            console.log(`Auth API running on port ${port}`);
        });
    }
}

// const authAPI = new AuthAPI();
// authAPI.start(3002);

5. Database Integration

const mongoose = require('mongoose');

// Connect to MongoDB
async function connectDB() {
    try {
        await mongoose.connect(process.env.MONGODB_URI || 'mongodb://localhost:27017/mydatabase', {
            useNewUrlParser: true,
            useUnifiedTopology: true,
        });
        console.log('Connected to MongoDB');
    } catch (error) {
        console.error('MongoDB connection error:', error);
        process.exit(1);
    }
}

// User Schema
const userSchema = new mongoose.Schema({
    username: {
        type: String,
        required: [true, 'Username is required'],
        unique: true,
        trim: true,
        minlength: [3, 'Username must be at least 3 characters'],
        maxlength: [30, 'Username cannot exceed 30 characters']
    },
    email: {
        type: String,
        required: [true, 'Email is required'],
        unique: true,
        lowercase: true,
        validate: {
            validator: function(email) {
                return /^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(email);
            },
            message: 'Please provide a valid email'
        }
    },
    password: {
        type: String,
        required: [true, 'Password is required'],
        minlength: [6, 'Password must be at least 6 characters']
    },
    profile: {
        firstName: String,
        lastName: String,
        bio: {
            type: String,
            maxlength: [500, 'Bio cannot exceed 500 characters']
        },
        avatar: String
    },
    role: {
        type: String,
        enum: ['user', 'admin', 'moderator'],
        default: 'user'
    },
    isActive: {
        type: Boolean,
        default: true
    },
    lastLogin: Date
}, {
    timestamps: true, // Adds createdAt and updatedAt
    toJSON: {
        transform: function(doc, ret) {
            delete ret.password; // Remove password from JSON output
            return ret;
        }
    }
});

// Index for better query performance
userSchema.index({ email: 1 });
userSchema.index({ username: 1 });
userSchema.index({ 'profile.firstName': 1, 'profile.lastName': 1 });

// Virtual for full name
userSchema.virtual('fullName').get(function() {
    return `${this.profile.firstName} ${this.profile.lastName}`.trim();
});

// Instance methods
userSchema.methods.getPublicProfile = function() {
    return {
        id: this._id,
        username: this.username,
        email: this.email,
        profile: this.profile,
        role: this.role,
        createdAt: this.createdAt
    };
};

userSchema.methods.updateLastLogin = function() {
    this.lastLogin = new Date();
    return this.save();
};

// Static methods
userSchema.statics.findByEmail = function(email) {
    return this.findOne({ email: email.toLowerCase() });
};

userSchema.statics.findActiveUsers = function() {
    return this.find({ isActive: true });
};

// Middleware
userSchema.pre('save', async function(next) {
    if (!this.isModified('password')) return next();
    
    // In real app, hash password here
    // this.password = await bcrypt.hash(this.password, 12);
    next();
});

userSchema.post('save', function(doc, next) {
    console.log(`User ${doc.username} was saved`);
    next();
});

const User = mongoose.model('User', userSchema);

// Product Schema with references
const productSchema = new mongoose.Schema({
    name: {
        type: String,
        required: true,
        trim: true
    },
    description: String,
    price: {
        type: Number,
        required: true,
        min: [0, 'Price cannot be negative']
    },
    category: {
        type: String,
        required: true,
        enum: ['electronics', 'books', 'clothing', 'home', 'sports']
    },
    tags: [String],
    inventory: {
        stock: {
            type: Number,
            default: 0,
            min: 0
        },
        reserved: {
            type: Number,
            default: 0
        }
    },
    images: [String],
    specifications: Map, // Flexible key-value pairs
    createdBy: {
        type: mongoose.Schema.Types.ObjectId,
        ref: 'User',
        required: true
    },
    isPublished: {
        type: Boolean,
        default: false
    }
}, {
    timestamps: true
});

// Compound index
productSchema.index({ category: 1, price: 1 });
productSchema.index({ tags: 1 });

// Virtual for available stock
productSchema.virtual('availableStock').get(function() {
    return this.inventory.stock - this.inventory.reserved;
});

// Instance method
productSchema.methods.reserveStock = function(quantity) {
    if (this.availableStock < quantity) {
        throw new Error('Insufficient stock');
    }
    this.inventory.reserved += quantity;
    return this.save();
};

productSchema.methods.releaseStock = function(quantity) {
    this.inventory.reserved = Math.max(0, this.inventory.reserved - quantity);
    return this.save();
};

const Product = mongoose.model('Product', productSchema);

// Order Schema with relationships
const orderSchema = new mongoose.Schema({
    orderNumber: {
        type: String,
        unique: true,
        required: true
    },
    user: {
        type: mongoose.Schema.Types.ObjectId,
        ref: 'User',
        required: true
    },
    items: [{
        product: {
            type: mongoose.Schema.Types.ObjectId,
            ref: 'Product',
            required: true
        },
        quantity: {
            type: Number,
            required: true,
            min: 1
        },
        price: {
            type: Number,
            required: true,
            min: 0
        }
    }],
    totalAmount: {
        type: Number,
        required: true,
        min: 0
    },
    status: {
        type: String,
        enum: ['pending', 'confirmed', 'shipped', 'delivered', 'cancelled'],
        default: 'pending'
    },
    shippingAddress: {
        street: String,
        city: String,
        state: String,
        zipCode: String,
        country: String
    },
    paymentStatus: {
        type: String,
        enum: ['pending', 'paid', 'failed', 'refunded'],
        default: 'pending'
    }
}, {
    timestamps: true
});

// Pre-save middleware to generate order number
orderSchema.pre('save', async function(next) {
    if (this.isNew) {
        const count = await this.constructor.countDocuments();
        this.orderNumber = `ORD-${String(count + 1).padStart(6, '0')}`;
    }
    next();
});

// Instance method to calculate total
orderSchema.methods.calculateTotal = function() {
    this.totalAmount = this.items.reduce((total, item) => {
        return total + (item.quantity * item.price);
    }, 0);
    return this.save();
};

const Order = mongoose.model('Order', orderSchema);

// Database operations class
class DatabaseService {
    constructor() {
        this.User = User;
        this.Product = Product;
        this.Order = Order;
    }

    // User operations
    async createUser(userData) {
        try {
            const user = new this.User(userData);
            await user.save();
            return user.getPublicProfile();
        } catch (error) {
            if (error.code === 11000) {
                throw new Error('Username or email already exists');
            }
            throw error;
        }
    }

    async findUserById(userId) {
        return this.User.findById(userId);
    }

    async findUserByEmail(email) {
        return this.User.findByEmail(email);
    }

    async updateUser(userId, updates) {
        const user = await this.User.findByIdAndUpdate(
            userId, 
            updates, 
            { new: true, runValidators: true }
        );
        return user ? user.getPublicProfile() : null;
    }

    // Product operations
    async createProduct(productData) {
        const product = new this.Product(productData);
        await product.save();
        return product;
    }

    async findProducts(filters = {}, options = {}) {
        const {
            page = 1,
            limit = 10,
            sort = '-createdAt',
            category,
            minPrice,
            maxPrice,
            tags,
            search
        } = filters;

        const query = { isPublished: true };

        if (category) query.category = category;
        if (tags) query.tags = { $in: Array.isArray(tags) ? tags : [tags] };
        if (minPrice || maxPrice) {
            query.price = {};
            if (minPrice) query.price.$gte = Number(minPrice);
            if (maxPrice) query.price.$lte = Number(maxPrice);
        }
        if (search) {
            query.$or = [
                { name: { $regex: search, $options: 'i' } },
                { description: { $regex: search, $options: 'i' } }
            ];
        }

        const products = await this.Product.find(query)
            .populate('createdBy', 'username email')
            .sort(sort)
            .limit(limit * 1)
            .skip((page - 1) * limit);

        const total = await this.Product.countDocuments(query);

        return {
            products,
            totalPages: Math.ceil(total / limit),
            currentPage: page,
            total
        };
    }

    // Order operations
    async createOrder(orderData) {
        const session = await mongoose.startSession();
        session.startTransaction();

        try {
            // Validate products and stock
            for (const item of orderData.items) {
                const product = await this.Product.findById(item.product);
                if (!product) {
                    throw new Error(`Product ${item.product} not found`);
                }
                if (product.availableStock < item.quantity) {
                    throw new Error(`Insufficient stock for ${product.name}`);
                }
            }

            // Reserve stock
            for (const item of orderData.items) {
                await this.Product.findByIdAndUpdate(
                    item.product,
                    { $inc: { 'inventory.reserved': item.quantity } }
                );
            }

            const order = new this.Order(orderData);
            await order.save({ session });
            await order.calculateTotal();

            await session.commitTransaction();
            return order;
        } catch (error) {
            await session.abortTransaction();
            throw error;
        } finally {
            session.endSession();
        }
    }

    async getOrderWithDetails(orderId) {
        return this.Order.findById(orderId)
            .populate('user', 'username email profile')
            .populate('items.product', 'name price images');
    }

    // Aggregation examples
    async getSalesReport(startDate, endDate) {
        return this.Order.aggregate([
            {
                $match: {
                    createdAt: {
                        $gte: new Date(startDate),
                        $lte: new Date(endDate)
                    },
                    status: { $ne: 'cancelled' }
                }
            },
            {
                $unwind: '$items'
            },
            {
                $lookup: {
                    from: 'products',
                    localField: 'items.product',
                    foreignField: '_id',
                    as: 'productDetails'
                }
            },
            {
                $unwind: '$productDetails'
            },
            {
                $group: {
                    _id: '$productDetails.category',
                    totalSales: { $sum: { $multiply: ['$items.quantity', '$items.price'] } },
                    totalQuantity: { $sum: '$items.quantity' },
                    orderCount: { $sum: 1 }
                }
            },
            {
                $sort: { totalSales: -1 }
            }
        ]);
    }

    async getUserStatistics(userId) {
        return this.Order.aggregate([
            {
                $match: { user: mongoose.Types.ObjectId(userId) }
            },
            {
                $group: {
                    _id: '$status',
                    count: { $sum: 1 },
                    totalAmount: { $sum: '$totalAmount' }
                }
            }
        ]);
    }
}

// Usage example
async function demonstrateMongoDB() {
    await connectDB();
    const dbService = new DatabaseService();

    try {
        // Create a user
        const user = await dbService.createUser({
            username: 'johndoe',
            email: 'john@example.com',
            password: 'securepassword',
            profile: {
                firstName: 'John',
                lastName: 'Doe'
            }
        });
        console.log('Created user:', user);

        // Create products
        const product = await dbService.createProduct({
            name: 'Wireless Mouse',
            description: 'High-quality wireless mouse',
            price: 29.99,
            category: 'electronics',
            tags: ['wireless', 'computer', 'accessories'],
            inventory: { stock: 100 },
            createdBy: user.id
        });
        console.log('Created product:', product);

        // Create order
        const order = await dbService.createOrder({
            user: user.id,
            items: [{
                product: product._id,
                quantity: 2,
                price: product.price
            }],
            shippingAddress: {
                street: '123 Main St',
                city: 'New York',
                state: 'NY',
                zipCode: '10001',
                country: 'USA'
            }
        });
        console.log('Created order:', order);

        // Get sales report
        const report = await dbService.getSalesReport(
            '2023-01-01',
            '2023-12-31'
        );
        console.log('Sales report:', report);

    } catch (error) {
        console.error('Error:', error.message);
    }
}

// demonstrateMongoDB();

const { Sequelize, DataTypes, Model, Op } = require('sequelize');
const bcrypt = require('bcrypt');

// Database connection
const sequelize = new Sequelize({
    dialect: 'mysql', // or 'postgres', 'sqlite', 'mariadb', 'mssql'
    host: process.env.DB_HOST || 'localhost',
    port: process.env.DB_PORT || 3306,
    database: process.env.DB_NAME || 'mydatabase',
    username: process.env.DB_USER || 'root',
    password: process.env.DB_PASSWORD || '',
    logging: process.env.NODE_ENV === 'development' ? console.log : false,
    pool: {
        max: 5,
        min: 0,
        acquire: 30000,
        idle: 10000
    },
    define: {
        timestamps: true,
        underscored: true,
        freezeTableName: true
    }
});

// User model
class User extends Model {
    // Instance methods
    async validatePassword(password) {
        return bcrypt.compare(password, this.password_hash);
    }

    getFullName() {
        return `${this.first_name} ${this.last_name}`;
    }

    // Static methods
    static async findByEmail(email) {
        return this.findOne({ where: { email } });
    }

    static async findActiveUsers() {
        return this.findAll({ where: { is_active: true } });
    }
}

User.init({
    id: {
        type: DataTypes.INTEGER,
        primaryKey: true,
        autoIncrement: true
    },
    first_name: {
        type: DataTypes.STRING(50),
        allowNull: false,
        validate: {
            notEmpty: true,
            len: [2, 50]
        }
    },
    last_name: {
        type: DataTypes.STRING(50),
        allowNull: false,
        validate: {
            notEmpty: true,
            len: [2, 50]
        }
    },
    email: {
        type: DataTypes.STRING(100),
        allowNull: false,
        unique: true,
        validate: {
            isEmail: true,
            notEmpty: true
        }
    },
    password_hash: {
        type: DataTypes.STRING(255),
        allowNull: false
    },
    role: {
        type: DataTypes.ENUM('user', 'admin', 'moderator'),
        defaultValue: 'user'
    },
    is_active: {
        type: DataTypes.BOOLEAN,
        defaultValue: true
    },
    last_login: {
        type: DataTypes.DATE,
        allowNull: true
    },
    preferences: {
        type: DataTypes.JSON,
        defaultValue: {}
    }
}, {
    sequelize,
    modelName: 'user',
    tableName: 'users',
    hooks: {
        beforeSave: async (user) => {
            if (user.changed('password_hash')) {
                user.password_hash = await bcrypt.hash(user.password_hash, 12);
            }
        },
        afterCreate: (user) => {
            console.log(`User ${user.email} created`);
        }
    },
    defaultScope: {
        attributes: { exclude: ['password_hash'] }
    },
    scopes: {
        withPassword: {
            attributes: { include: ['password_hash'] }
        },
        active: {
            where: { is_active: true }
        }
    }
});

// Product model
class Product extends Model {
    // Virtual field
    get formattedPrice() {
        return `${	his.price.toFixed(2)}`;
    }

    // Instance method
    async reserveStock(quantity) {
        if (this.stock_quantity < quantity) {
            throw new Error('Insufficient stock');
        }
        this.stock_quantity -= quantity;
        return this.save();
    }
}

Product.init({
    id: {
        type: DataTypes.INTEGER,
        primaryKey: true,
        autoIncrement: true
    },
    name: {
        type: DataTypes.STRING(100),
        allowNull: false,
        validate: {
            notEmpty: true,
            len: [3, 100]
        }
    },
    description: {
        type: DataTypes.TEXT,
        allowNull: true
    },
    price: {
        type: DataTypes.DECIMAL(10, 2),
        allowNull: false,
        validate: {
            min: 0
        }
    },
    category: {
        type: DataTypes.ENUM('electronics', 'books', 'clothing', 'home', 'sports'),
        allowNull: false
    },
    stock_quantity: {
        type: DataTypes.INTEGER,
        defaultValue: 0,
        validate: {
            min: 0
        }
    },
    sku: {
        type: DataTypes.STRING(50),
        unique: true,
        allowNull: false
    },
    specifications: {
        type: DataTypes.JSON,
        defaultValue: {}
    },
    is_published: {
        type: DataTypes.BOOLEAN,
        defaultValue: false
    },
    published_at: {
        type: DataTypes.DATE,
        allowNull: true
    }
}, {
    sequelize,
    modelName: 'product',
    tableName: 'products',
    hooks: {
        beforeUpdate: (product) => {
            if (product.changed('is_published') && product.is_published) {
                product.published_at = new Date();
            }
        }
    },
    indexes: [
        {
            fields: ['category', 'price']
        },
        {
            fields: ['sku'],
            unique: true
        }
    ]
});

// Order model
class Order extends Model {
    // Instance method
    calculateTotal() {
        return this.sequelize.models.OrderItem.findAll({
            where: { order_id: this.id },
            include: ['product']
        }).then(items => {
            const total = items.reduce((sum, item) => {
                return sum + (item.quantity * item.product.price);
            }, 0);
            this.total_amount = total;
            return this.save();
        });
    }
}

Order.init({
    id: {
        type: DataTypes.INTEGER,
        primaryKey: true,
        autoIncrement: true
    },
    order_number: {
        type: DataTypes.STRING(20),
        unique: true,
        allowNull: false
    },
    total_amount: {
        type: DataTypes.DECIMAL(10, 2),
        allowNull: false,
        defaultValue: 0,
        validate: {
            min: 0
        }
    },
    status: {
        type: DataTypes.ENUM('pending', 'confirmed', 'shipped', 'delivered', 'cancelled'),
        defaultValue: 'pending'
    },
    shipping_address: {
        type: DataTypes.JSON,
        allowNull: false
    },
    payment_status: {
        type: DataTypes.ENUM('pending', 'paid', 'failed', 'refunded'),
        defaultValue: 'pending'
    }
}, {
    sequelize,
    modelName: 'order',
    tableName: 'orders',
    hooks: {
        beforeCreate: async (order) => {
            const count = await Order.count();
            order.order_number = `ORD-${String(count + 1).padStart(6, '0')}`;
        }
    }
});

// OrderItem model for the join table
class OrderItem extends Model {}

OrderItem.init({
    id: {
        type: DataTypes.INTEGER,
        primaryKey: true,
        autoIncrement: true
    },
    quantity: {
        type: DataTypes.INTEGER,
        allowNull: false,
        validate: {
            min: 1
        }
    },
    unit_price: {
        type: DataTypes.DECIMAL(10, 2),
        allowNull: false,
        validate: {
            min: 0
        }
    }
}, {
    sequelize,
    modelName: 'order_item',
    tableName: 'order_items'
});

// Define relationships
User.hasMany(Order, { foreignKey: 'user_id', as: 'orders' });
Order.belongsTo(User, { foreignKey: 'user_id', as: 'user' });

Order.hasMany(OrderItem, { foreignKey: 'order_id', as: 'items' });
OrderItem.belongsTo(Order, { foreignKey: 'order_id', as: 'order' });

Product.hasMany(OrderItem, { foreignKey: 'product_id', as: 'order_items' });
OrderItem.belongsTo(Product, { foreignKey: 'product_id', as: 'product' });

// Database service class
class SQLDatabaseService {
    constructor() {
        this.sequelize = sequelize;
        this.User = User;
        this.Product = Product;
        this.Order = Order;
        this.OrderItem = OrderItem;
    }

    async connect() {
        try {
            await this.sequelize.authenticate();
            console.log('Database connection established');
            
            // Sync models (in production, use migrations instead)
            if (process.env.NODE_ENV === 'development') {
                await this.sequelize.sync({ force: false });
                console.log('Database synced');
            }
        } catch (error) {
            console.error('Database connection failed:', error);
            throw error;
        }
    }

    // User operations
    async createUser(userData) {
        const transaction = await this.sequelize.transaction();
        
        try {
            const user = await this.User.create(userData, { transaction });
            await transaction.commit();
            return user;
        } catch (error) {
            await transaction.rollback();
            throw error;
        }
    }

    async findUserWithOrders(userId) {
        return this.User.findByPk(userId, {
            include: [{
                model: this.Order,
                as: 'orders',
                include: [{
                    model: this.OrderItem,
                    as: 'items',
                    include: ['product']
                }]
            }]
        });
    }

    // Product operations
    async findProducts(filters = {}) {
        const {
            page = 1,
            limit = 10,
            category,
            minPrice,
            maxPrice,
            search
        } = filters;

        const where = { is_published: true };
        const offset = (page - 1) * limit;

        if (category) where.category = category;
        if (minPrice || maxPrice) {
            where.price = {};
            if (minPrice) where.price[Op.gte] = minPrice;
            if (maxPrice) where.price[Op.lte] = maxPrice;
        }
        if (search) {
            where[Op.or] = [
                { name: { [Op.like]: `%${search}%` } },
                { description: { [Op.like]: `%${search}%` } }
            ];
        }

        const { count, rows } = await this.Product.findAndCountAll({
            where,
            limit,
            offset,
            order: [['created_at', 'DESC']]
        });

        return {
            products: rows,
            totalPages: Math.ceil(count / limit),
            currentPage: page,
            total: count
        };
    }

    // Order operations
    async createOrder(orderData) {
        const transaction = await this.sequelize.transaction();
        
        try {
            // Validate products and stock
            for (const item of orderData.items) {
                const product = await this.Product.findByPk(item.product_id);
                if (!product) {
                    throw new Error(`Product ${item.product_id} not found`);
                }
                if (product.stock_quantity < item.quantity) {
                    throw new Error(`Insufficient stock for ${product.name}`);
                }
            }

            // Create order
            const order = await this.Order.create({
                user_id: orderData.user_id,
                shipping_address: orderData.shipping_address
            }, { transaction });

            // Create order items and update stock
            for (const item of orderData.items) {
                const product = await this.Product.findByPk(item.product_id);
                
                await this.OrderItem.create({
                    order_id: order.id,
                    product_id: item.product_id,
                    quantity: item.quantity,
                    unit_price: product.price
                }, { transaction });

                // Update product stock
                await product.decrement('stock_quantity', { 
                    by: item.quantity,
                    transaction 
                });
            }

            // Calculate total
            await order.calculateTotal();

            await transaction.commit();
            return order;
        } catch (error) {
            await transaction.rollback();
            throw error;
        }
    }

    // Complex queries with raw SQL
    async getSalesReport(startDate, endDate) {
        const [results] = await this.sequelize.query(`
            SELECT 
                p.category,
                COUNT(oi.id) as total_orders,
                SUM(oi.quantity) as total_quantity,
                SUM(oi.quantity * oi.unit_price) as total_sales
            FROM order_items oi
            JOIN products p ON oi.product_id = p.id
            JOIN orders o ON oi.order_id = o.id
            WHERE o.created_at BETWEEN ? AND ?
                AND o.status != 'cancelled'
            GROUP BY p.category
            ORDER BY total_sales DESC
        `, {
            replacements: [startDate, endDate],
            type: this.sequelize.QueryTypes.SELECT
        });

        return results;
    }

    // Bulk operations
    async bulkCreateProducts(products) {
        return this.Product.bulkCreate(products, {
            validate: true,
            individualHooks: true
        });
    }

    async updateProductPrices(category, percentageIncrease) {
        return this.Product.update({
            price: this.sequelize.literal(`price * ${1 + percentageIncrease / 100}`)
        }, {
            where: { category }
        });
    }

    // Close connection
    async close() {
        await this.sequelize.close();
    }
}

// Usage example
async function demonstrateSequelize() {
    const dbService = new SQLDatabaseService();
    
    try {
        await dbService.connect();

        // Create user
        const user = await dbService.createUser({
            first_name: 'John',
            last_name: 'Doe',
            email: 'john@example.com',
            password_hash: 'plaintextpassword' // Will be hashed by hook
        });
        console.log('Created user:', user.getFullName());

        // Create product
        const product = await dbService.Product.create({
            name: 'Laptop',
            description: 'High-performance laptop',
            price: 999.99,
            category: 'electronics',
            sku: 'LAPTOP-001',
            stock_quantity: 50
        });
        console.log('Created product:', product.formattedPrice);

        // Create order
        const order = await dbService.createOrder({
            user_id: user.id,
            shipping_address: {
                street: '123 Main St',
                city: 'New York'
            },
            items: [{
                product_id: product.id,
                quantity: 1
            }]
        });
        console.log('Created order:', order.order_number);

        // Get user with orders
        const userWithOrders = await dbService.findUserWithOrders(user.id);
        console.log('User orders:', userWithOrders.orders.length);

        // Get sales report
        const report = await dbService.getSalesReport(
            '2023-01-01',
            '2023-12-31'
        );
        console.log('Sales report:', report);

    } catch (error) {
        console.error('Error:', error.message);
    } finally {
        await dbService.close();
    }
}

// demonstrateSequelize();

6. Advanced Concepts

const express = require('express');
const cluster = require('cluster');
const os = require('os');
const redis = require('redis');

// Clustering for utilizing multiple CPU cores
if (cluster.isMaster) {
    const numCPUs = os.cpus().length;
    console.log(`Master ${process.pid} is running`);
    console.log(`Forking for ${numCPUs} CPUs`);

    // Fork workers
    for (let i = 0; i < numCPUs; i++) {
        cluster.fork();
    }

    cluster.on('exit', (worker, code, signal) => {
        console.log(`Worker ${worker.process.pid} died. Forking new worker...`);
        cluster.fork();
    });
} else {
    // Worker process
    const app = express();
    
    // Redis client for caching
    const redisClient = redis.createClient({
        url: process.env.REDIS_URL || 'redis://localhost:6379'
    });

    redisClient.on('error', (err) => {
        console.error('Redis error:', err);
    });

    // Connect to Redis
    async function connectRedis() {
        await redisClient.connect();
    }

    // Caching middleware
    function cacheMiddleware(duration = 300) { // 5 minutes default
        return async (req, res, next) => {
            if (req.method !== 'GET') {
                return next();
            }

            const key = `cache:${req.originalUrl}`;
            
            try {
                const cachedData = await redisClient.get(key);
                if (cachedData) {
                    console.log('Cache hit for:', key);
                    return res.json(JSON.parse(cachedData));
                }

                // Override res.json to cache the response
                const originalJson = res.json;
                res.json = function(data) {
                    redisClient.setEx(key, duration, JSON.stringify(data))
                        .catch(err => console.error('Cache set error:', err));
                    originalJson.call(this, data);
                };

                next();
            } catch (error) {
                console.error('Cache error:', error);
                next();
            }
        };
    }

    // Database connection pool (using generic pool)
    const { createPool } = require('generic-pool');
    const mysql = require('mysql2/promise');

    const dbPool = createPool({
        create: async () => {
            return await mysql.createConnection({
                host: process.env.DB_HOST,
                user: process.env.DB_USER,
                password: process.env.DB_PASSWORD,
                database: process.env.DB_NAME,
                connectionLimit: 10
            });
        },
        destroy: (connection) => connection.end(),
        validate: (connection) => connection.query('SELECT 1').then(() => true),
        acquireTimeoutMillis: 30000,
        idleTimeoutMillis: 30000,
        max: 10,
        min: 2
    });

    // Rate limiting with Redis
    function rateLimitMiddleware(windowMs = 60000, maxRequests = 100) {
        return async (req, res, next) => {
            const key = `rate_limit:${req.ip}`;
            
            try {
                const current = await redisClient.get(key);
                const currentCount = current ? parseInt(current) : 0;

                if (currentCount >= maxRequests) {
                    return res.status(429).json({
                        error: 'Too many requests',
                        retryAfter: Math.ceil(windowMs / 1000)
                    });
                }

                if (currentCount === 0) {
                    await redisClient.setEx(key, Math.ceil(windowMs / 1000), '1');
                } else {
                    await redisClient.incr(key);
                }

                res.set('X-RateLimit-Limit', maxRequests.toString());
                res.set('X-RateLimit-Remaining', (maxRequests - currentCount - 1).toString());
                
                next();
            } catch (error) {
                console.error('Rate limit error:', error);
                next();
            }
        };
    }

    // Compression middleware
    const compression = require('compression');
    app.use(compression({
        level: 6,
        threshold: 1024 // Only compress responses larger than 1KB
    }));

    // Helmet for security headers
    const helmet = require('helmet');
    app.use(helmet({
        contentSecurityPolicy: false, // Configure based on your needs
        crossOriginEmbedderPolicy: false
    }));

    // Body parser with limits
    app.use(express.json({ limit: '10mb' }));
    app.use(express.urlencoded({ extended: true, limit: '10mb' }));

    // Static files with caching
    app.use(express.static('public', {
        maxAge: '1d', // Cache for 1 day
        etag: true,
        lastModified: true
    }));

    // Routes with caching and rate limiting
    app.get('/api/products', 
        rateLimitMiddleware(60000, 60), // 60 requests per minute
        cacheMiddleware(300), // Cache for 5 minutes
        async (req, res) => {
            try {
                const connection = await dbPool.acquire();
                
                try {
                    const [products] = await connection.execute(`
                        SELECT * FROM products 
                        WHERE is_published = true 
                        ORDER BY created_at DESC 
                        LIMIT 100
                    `);
                    
                    res.json({
                        products,
                        timestamp: new Date().toISOString()
                    });
                } finally {
                    dbPool.release(connection);
                }
            } catch (error) {
                console.error('Database error:', error);
                res.status(500).json({ error: 'Internal server error' });
            }
        }
    );

    // Batch processing endpoint
    app.post('/api/batch', async (req, res) => {
        const { operations } = req.body;
        
        if (!Array.isArray(operations) || operations.length > 100) {
            return res.status(400).json({ error: 'Invalid batch operations' });
        }

        const results = [];
        const connection = await dbPool.acquire();

        try {
            await connection.beginTransaction();

            for (const op of operations) {
                try {
                    let result;
                    
                    switch (op.type) {
                        case 'create':
                            [result] = await connection.execute(
                                'INSERT INTO products (name, price) VALUES (?, ?)',
                                [op.data.name, op.data.price]
                            );
                            break;
                        case 'update':
                            [result] = await connection.execute(
                                'UPDATE products SET price = ? WHERE id = ?',
                                [op.data.price, op.data.id]
                            );
                            break;
                        default:
                            throw new Error(`Unknown operation type: ${op.type}`);
                    }
                    
                    results.push({ success: true, result });
                } catch (error) {
                    results.push({ success: false, error: error.message });
                }
            }

            await connection.commit();
            res.json({ results });
        } catch (error) {
            await connection.rollback();
            res.status(500).json({ error: 'Batch operation failed' });
        } finally {
            dbPool.release(connection);
        }
    });

    // Memory usage monitoring
    function monitorMemoryUsage() {
        const used = process.memoryUsage();
        console.log('Memory usage:');
        for (let key in used) {
            console.log(`${key}: ${Math.round(used[key] / 1024 / 1024 * 100) / 100} MB`);
        }
    }

    // Set up memory monitoring interval
    setInterval(monitorMemoryUsage, 60000); // Every minute

    // Garbage collection hint (if needed)
    if (global.gc) {
        setInterval(() => {
            global.gc();
            console.log('Forced garbage collection');
        }, 300000); // Every 5 minutes
    }

    // Process event handlers for graceful shutdown
    process.on('SIGTERM', async () => {
        console.log('SIGTERM received, shutting down gracefully');
        
        // Stop accepting new requests
        server.close(() => {
            console.log('HTTP server closed');
        });
        
        // Close database connections
        await dbPool.drain();
        await dbPool.clear();
        
        // Close Redis connection
        await redisClient.quit();
        
        process.exit(0);
    });

    process.on('uncaughtException', (error) => {
        console.error('Uncaught Exception:', error);
        // In production, you might want to restart the process
        process.exit(1);
    });

    process.on('unhandledRejection', (reason, promise) => {
        console.error('Unhandled Rejection at:', promise, 'reason:', reason);
        // In production, you might want to restart the process
        process.exit(1);
    });

    // Start server
    const PORT = process.env.PORT || 3000;
    const server = app.listen(PORT, async () => {
        await connectRedis();
        console.log(`Worker ${process.pid} started on port ${PORT}`);
    });

    // Health check endpoint
    app.get('/health', async (req, res) => {
        const health = {
            status: 'OK',
            timestamp: new Date().toISOString(),
            uptime: process.uptime(),
            memory: process.memoryUsage(),
            database: 'unknown',
            redis: 'unknown'
        };

        try {
            // Check database connection
            const connection = await dbPool.acquire();
            await connection.execute('SELECT 1');
            dbPool.release(connection);
            health.database = 'connected';
        } catch (error) {
            health.database = 'disconnected';
            health.status = 'ERROR';
        }

        try {
            // Check Redis connection
            await redisClient.ping();
            health.redis = 'connected';
        } catch (error) {
            health.redis = 'disconnected';
            health.status = 'ERROR';
        }

        res.status(health.status === 'OK' ? 200 : 503).json(health);
    });
}

// Advanced caching strategies
class CacheManager {
    constructor(redisClient) {
        this.redis = redisClient;
        this.localCache = new Map();
        this.localCacheTTL = new Map();
    }

    // Multi-level caching (Redis + Memory)
    async get(key) {
        // Check local cache first
        if (this.localCache.has(key)) {
            const ttl = this.localCacheTTL.get(key);
            if (ttl > Date.now()) {
                console.log('Local cache hit for:', key);
                return this.localCache.get(key);
            } else {
                // Local cache expired
                this.localCache.delete(key);
                this.localCacheTTL.delete(key);
            }
        }

        // Check Redis
        try {
            const value = await this.redis.get(key);
            if (value) {
                console.log('Redis cache hit for:', key);
                const parsedValue = JSON.parse(value);
                
                // Store in local cache with shorter TTL
                this.localCache.set(key, parsedValue);
                this.localCacheTTL.set(key, Date.now() + 30000); // 30 seconds
                
                return parsedValue;
            }
        } catch (error) {
            console.error('Redis get error:', error);
        }

        return null;
    }

    async set(key, value, ttlSeconds = 300) {
        try {
            // Store in Redis
            await this.redis.setEx(key, ttlSeconds, JSON.stringify(value));
            
            // Store in local cache with shorter TTL
            this.localCache.set(key, value);
            this.localCacheTTL.set(key, Date.now() + Math.min(ttlSeconds * 1000, 30000));
        } catch (error) {
            console.error('Redis set error:', error);
        }
    }

    async invalidate(pattern) {
        try {
            const keys = await this.redis.keys(pattern);
            if (keys.length > 0) {
                await this.redis.del(keys);
                
                // Also clear local cache for these keys
                keys.forEach(key => {
                    this.localCache.delete(key);
                    this.localCacheTTL.delete(key);
                });
                
                console.log(`Invalidated ${keys.length} cache keys matching ${pattern}`);
            }
        } catch (error) {
            console.error('Cache invalidation error:', error);
        }
    }
}

// Connection pooling with different databases
class DatabaseManager {
    constructor() {
        this.pools = new Map();
    }

    addPool(name, config) {
        const pool = mysql.createPool(config);
        this.pools.set(name, pool);
        return pool;
    }

    async query(poolName, sql, params = []) {
        const pool = this.pools.get(poolName);
        if (!pool) {
            throw new Error(`Pool ${poolName} not found`);
        }

        const connection = await pool.getConnection();
        try {
            const [results] = await connection.execute(sql, params);
            return results;
        } finally {
            connection.release();
        }
    }

    async transaction(poolName, callback) {
        const pool = this.pools.get(poolName);
        const connection = await pool.getConnection();
        
        await connection.beginTransaction();
        
        try {
            const result = await callback(connection);
            await connection.commit();
            return result;
        } catch (error) {
            await connection.rollback();
            throw error;
        } finally {
            connection.release();
        }
    }
}

// Performance monitoring
const performanceMonitor = {
    requests: new Map(),
    errors: new Map(),

    recordRequest(method, path, duration, statusCode) {
        const key = `${method} ${path}`;
        
        if (!this.requests.has(key)) {
            this.requests.set(key, {
                count: 0,
                totalDuration: 0,
                minDuration: Infinity,
                maxDuration: 0,
                statusCodes: {}
            });
        }

        const stats = this.requests.get(key);
        stats.count++;
        stats.totalDuration += duration;
        stats.minDuration = Math.min(stats.minDuration, duration);
        stats.maxDuration = Math.max(stats.maxDuration, duration);
        
        if (!stats.statusCodes[statusCode]) {
            stats.statusCodes[statusCode] = 0;
        }
        stats.statusCodes[statusCode]++;
    },

    recordError(type, error) {
        if (!this.errors.has(type)) {
            this.errors.set(type, []);
        }
        this.errors.get(type).push({
            message: error.message,
            timestamp: new Date().toISOString(),
            stack: error.stack
        });

        // Keep only last 100 errors per type
        const errors = this.errors.get(type);
        if (errors.length > 100) {
            this.errors.set(type, errors.slice(-100));
        }
    },

    getStats() {
        const requestStats = {};
        for (let [key, stats] of this.requests) {
            requestStats[key] = {
                ...stats,
                averageDuration: stats.totalDuration / stats.count
            };
        }

        return {
            requests: requestStats,
            errors: Object.fromEntries(this.errors)
        };
    }
};

// Middleware to record performance
function performanceMiddleware(req, res, next) {
    const start = Date.now();
    
    res.on('finish', () => {
        const duration = Date.now() - start;
        performanceMonitor.recordRequest(
            req.method, 
            req.path, 
            duration, 
            res.statusCode
        );
    });
    
    next();
}

const express = require('express');
const amqp = require('amqplib');
const redis = require('redis');

// Microservice: User Service
class UserService {
    constructor() {
        this.app = express();
        this.redisClient = redis.createClient();
        this.setupRoutes();
        this.connectQueue();
    }

    setupRoutes() {
        this.app.use(express.json());

        // Get user by ID
        this.app.get('/users/:id', async (req, res) => {
            try {
                const user = await this.getUser(req.params.id);
                if (!user) {
                    return res.status(404).json({ error: 'User not found' });
                }
                res.json(user);
            } catch (error) {
                res.status(500).json({ error: 'Internal server error' });
            }
        });

        // Create user
        this.app.post('/users', async (req, res) => {
            try {
                const user = await this.createUser(req.body);
                
                // Publish user created event
                await this.publishMessage('user_created', user);
                
                res.status(201).json(user);
            } catch (error) {
                res.status(500).json({ error: 'Internal server error' });
            }
        });

        // Health check
        this.app.get('/health', (req, res) => {
            res.json({ status: 'OK', service: 'user-service' });
        });
    }

    async getUser(userId) {
        // Check cache first
        const cachedUser = await this.redisClient.get(`user:${userId}`);
        if (cachedUser) {
            return JSON.parse(cachedUser);
        }

        // Simulate database call
        const user = {
            id: userId,
            name: 'John Doe',
            email: 'john@example.com',
            createdAt: new Date().toISOString()
        };

        // Cache for 5 minutes
        await this.redisClient.setEx(
            `user:${userId}`, 
            300, 
            JSON.stringify(user)
        );

        return user;
    }

    async createUser(userData) {
        const user = {
            id: Date.now().toString(),
            ...userData,
            createdAt: new Date().toISOString()
        };

        // In real application, save to database
        await this.redisClient.setEx(
            `user:${user.id}`,
            300,
            JSON.stringify(user)
        );

        return user;
    }

    async connectQueue() {
        try {
            this.connection = await amqp.connect('amqp://localhost');
            this.channel = await this.connection.createChannel();
            
            // Assert exchanges and queues
            await this.channel.assertExchange('user_events', 'topic', { durable: true });
            await this.channel.assertQueue('user_created_notifications', { durable: true });
            await this.channel.bindQueue('user_created_notifications', 'user_events', 'user.created');
            
            console.log('Connected to RabbitMQ');
        } catch (error) {
            console.error('RabbitMQ connection failed:', error);
        }
    }

    async publishMessage(routingKey, message) {
        if (!this.channel) return;

        try {
            await this.channel.publish(
                'user_events',
                routingKey,
                Buffer.from(JSON.stringify(message)),
                { persistent: true }
            );
            console.log(`Message published with routing key: ${routingKey}`);
        } catch (error) {
            console.error('Failed to publish message:', error);
        }
    }

    start(port = 3001) {
        this.redisClient.connect().then(() => {
            this.app.listen(port, () => {
                console.log(`User service running on port ${port}`);
            });
        });
    }
}

// Microservice: Notification Service
class NotificationService {
    constructor() {
        this.app = express();
        this.setupRoutes();
        this.connectQueue();
    }

    setupRoutes() {
        this.app.use(express.json());

        this.app.get('/health', (req, res) => {
            res.json({ status: 'OK', service: 'notification-service' });
        });
    }

    async connectQueue() {
        try {
            this.connection = await amqp.connect('amqp://localhost');
            this.channel = await this.connection.createChannel();
            
            // Assert exchange and queue
            await this.channel.assertExchange('user_events', 'topic', { durable: true });
            await this.channel.assertQueue('user_created_notifications', { durable: true });
            await this.channel.bindQueue('user_created_notifications', 'user_events', 'user.created');
            
            // Consume messages
            await this.channel.consume('user_created_notifications', async (msg) => {
                if (msg !== null) {
                    try {
                        const user = JSON.parse(msg.content.toString());
                        await this.handleUserCreated(user);
                        this.channel.ack(msg);
                    } catch (error) {
                        console.error('Error processing message:', error);
                        this.channel.nack(msg, false, false); // Don't requeue
                    }
                }
            });

            console.log('Notification service connected to RabbitMQ');
        } catch (error) {
            console.error('RabbitMQ connection failed:', error);
        }
    }

    async handleUserCreated(user) {
        console.log(`Sending welcome email to: ${user.email}`);
        
        // Simulate email sending
        await new Promise(resolve => setTimeout(resolve, 1000));
        console.log(`Welcome email sent to ${user.email}`);

        // You could also send push notifications, SMS, etc.
    }

    start(port = 3002) {
        this.app.listen(port, () => {
            console.log(`Notification service running on port ${port}`);
        });
    }
}

// API Gateway
class ApiGateway {
    constructor() {
        this.app = express();
        this.services = {
            user: process.env.USER_SERVICE_URL || 'http://localhost:3001',
            notification: process.env.NOTIFICATION_SERVICE_URL || 'http://localhost:3002',
            order: process.env.ORDER_SERVICE_URL || 'http://localhost:3003'
        };
        this.setupRoutes();
        this.setupMiddleware();
    }

    setupMiddleware() {
        // Rate limiting
        this.app.use(require('express-rate-limit')({
            windowMs: 15 * 60 * 1000, // 15 minutes
            max: 100 // limit each IP to 100 requests per windowMs
        }));

        // CORS
        this.app.use(require('cors')());
        
        // Body parsing
        this.app.use(express.json());
        
        // Logging
        this.app.use((req, res, next) => {
            console.log(`${new Date().toISOString()} - ${req.method} ${req.path}`);
            next();
        });
    }

    setupRoutes() {
        // Health check aggregator
        this.app.get('/health', async (req, res) => {
            const healthChecks = {};
            
            for (const [serviceName, serviceUrl] of Object.entries(this.services)) {
                try {
                    const response = await fetch(`${serviceUrl}/health`);
                    healthChecks[serviceName] = await response.json();
                } catch (error) {
                    healthChecks[serviceName] = { status: 'ERROR', error: error.message };
                }
            }

            const allHealthy = Object.values(healthChecks).every(
                check => check.status === 'OK'
            );

            res.status(allHealthy ? 200 : 503).json(healthChecks);
        });

        // User service routes
        this.app.use('/users', async (req, res) => {
            await this.proxyRequest(req, res, 'user');
        });

        // Notification service routes
        this.app.use('/notifications', async (req, res) => {
            await this.proxyRequest(req, res, 'notification');
        });
    }

    async proxyRequest(req, res, serviceName) {
        const serviceUrl = this.services[serviceName];
        const targetUrl = `${serviceUrl}${req.path}`;

        try {
            const response = await fetch(targetUrl, {
                method: req.method,
                headers: {
                    'Content-Type': 'application/json',
                    ...req.headers
                },
                body: req.method !== 'GET' ? JSON.stringify(req.body) : undefined
            });

            const data = await response.json();
            res.status(response.status).json(data);
        } catch (error) {
            res.status(502).json({ 
                error: `${serviceName} service unavailable` 
            });
        }
    }

    start(port = 3000) {
        this.app.listen(port, () => {
            console.log(`API Gateway running on port ${port}`);
        });
    }
}

// Message Queue Manager
class MessageQueueManager {
    constructor() {
        this.connection = null;
        this.channel = null;
        this.queues = new Map();
    }

    async connect(url = 'amqp://localhost') {
        this.connection = await amqp.connect(url);
        this.channel = await this.connection.createChannel();
        return this;
    }

    async createQueue(queueName, options = { durable: true }) {
        await this.channel.assertQueue(queueName, options);
        this.queues.set(queueName, true);
        return queueName;
    }

    async createExchange(exchangeName, type = 'direct', options = { durable: true }) {
        await this.channel.assertExchange(exchangeName, type, options);
        return exchangeName;
    }

    async bindQueueToExchange(queueName, exchangeName, routingKey = '') {
        await this.channel.bindQueue(queueName, exchangeName, routingKey);
    }

    async publishToExchange(exchangeName, routingKey, message, options = {}) {
        await this.channel.publish(
            exchangeName,
            routingKey,
            Buffer.from(JSON.stringify(message)),
            { persistent: true, ...options }
        );
    }

    async sendToQueue(queueName, message, options = {}) {
        await this.channel.sendToQueue(
            queueName,
            Buffer.from(JSON.stringify(message)),
            { persistent: true, ...options }
        );
    }

    async consumeQueue(queueName, handler, options = { noAck: false }) {
        await this.channel.consume(queueName, async (msg) => {
            if (msg !== null) {
                try {
                    const message = JSON.parse(msg.content.toString());
                    await handler(message);
                    this.channel.ack(msg);
                } catch (error) {
                    console.error('Error processing message:', error);
                    this.channel.nack(msg, false, false);
                }
            }
        }, options);
    }

    async close() {
        if (this.channel) await this.channel.close();
        if (this.connection) await this.connection.close();
    }
}

// Event-Driven Service Example
class OrderService {
    constructor() {
        this.mqManager = new MessageQueueManager();
        this.orders = new Map();
        this.setupEventHandlers();
    }

    async setupEventHandlers() {
        await this.mqManager.connect();
        
        // Create exchanges and queues
        await this.mqManager.createExchange('order_events', 'topic');
        await this.mqManager.createQueue('order_processing');
        await this.mqManager.bindQueueToExchange('order_processing', 'order_events', 'order.created');
        
        // Consume order creation events
        await this.mqManager.consumeQueue('order_processing', async (order) => {
            await this.processOrder(order);
        });
    }

    async createOrder(orderData) {
        const order = {
            id: Date.now().toString(),
            ...orderData,
            status: 'created',
            createdAt: new Date().toISOString()
        };

        this.orders.set(order.id, order);

        // Publish order created event
        await this.mqManager.publishToExchange(
            'order_events',
            'order.created',
            order
        );

        return order;
    }

    async processOrder(order) {
        console.log(`Processing order: ${order.id}`);
        
        // Simulate order processing
        await new Promise(resolve => setTimeout(resolve, 2000));
        
        order.status = 'processed';
        order.processedAt = new Date().toISOString();
        this.orders.set(order.id, order);

        // Publish order processed event
        await this.mqManager.publishToExchange(
            'order_events',
            'order.processed',
            order
        );

        console.log(`Order ${order.id} processed successfully`);
    }

    async getOrder(orderId) {
        return this.orders.get(orderId);
    }
}

// Service Discovery (simple implementation)
class ServiceRegistry {
    constructor() {
        this.services = new Map();
        this.healthCheckInterval = 30000; // 30 seconds
    }

    registerService(serviceName, serviceUrl, metadata = {}) {
        this.services.set(serviceName, {
            url: serviceUrl,
            metadata,
            lastHealthCheck: Date.now(),
            isHealthy: true
        });

        console.log(`Service registered: ${serviceName} at ${serviceUrl}`);
    }

    unregisterService(serviceName) {
        this.services.delete(serviceName);
        console.log(`Service unregistered: ${serviceName}`);
    }

    getService(serviceName) {
        const service = this.services.get(serviceName);
        return service && service.isHealthy ? service : null;
    }

    startHealthChecks() {
        setInterval(async () => {
            for (const [serviceName, service] of this.services) {
                try {
                    const response = await fetch(`${service.url}/health`);
                    if (response.ok) {
                        service.isHealthy = true;
                        service.lastHealthCheck = Date.now();
                    } else {
                        service.isHealthy = false;
                    }
                } catch (error) {
                    service.isHealthy = false;
                    console.log(`Service ${serviceName} health check failed`);
                }
            }
        }, this.healthCheckInterval);
    }
}

// Circuit Breaker pattern
class CircuitBreaker {
    constructor(timeout = 10000) {
        this.state = 'CLOSED'; // CLOSED, OPEN, HALF_OPEN
        this.failureCount = 0;
        this.successCount = 0;
        this.nextAttempt = Date.now();
        this.timeout = timeout;
        this.failureThreshold = 5;
        this.successThreshold = 3;
    }

    async call(serviceCall) {
        if (this.state === 'OPEN') {
            if (Date.now() < this.nextAttempt) {
                throw new Error('Circuit breaker is OPEN');
            }
            this.state = 'HALF_OPEN';
        }

        try {
            const result = await serviceCall();
            this.onSuccess();
            return result;
        } catch (error) {
            this.onFailure();
            throw error;
        }
    }

    onSuccess() {
        this.failureCount = 0;
        if (this.state === 'HALF_OPEN') {
            this.successCount++;
            if (this.successCount >= this.successThreshold) {
                this.state = 'CLOSED';
                this.successCount = 0;
            }
        }
    }

    onFailure() {
        this.failureCount++;
        if (this.failureCount >= this.failureThreshold) {
            this.state = 'OPEN';
            this.nextAttempt = Date.now() + this.timeout;
        }
    }
}

// Usage example
async function demonstrateMicroservices() {
    // Start services
    const userService = new UserService();
    const notificationService = new NotificationService();
    const apiGateway = new ApiGateway();
    const orderService = new OrderService();
    
    // Start services
    userService.start(3001);
    notificationService.start(3002);
    apiGateway.start(3000);

    // Wait for services to start
    await new Promise(resolve => setTimeout(resolve, 1000));

    // Demonstrate service communication
    const circuitBreaker = new CircuitBreaker();
    
    try {
        const user = await circuitBreaker.call(async () => {
            const response = await fetch('http://localhost:3000/users/123');
            return response.json();
        });
        console.log('User fetched:', user);
    } catch (error) {
        console.error('Failed to fetch user:', error.message);
    }

    // Create a user through API Gateway
    try {
        const response = await fetch('http://localhost:3000/users', {
            method: 'POST',
            headers: { 'Content-Type': 'application/json' },
            body: JSON.stringify({
                name: 'Jane Smith',
                email: 'jane@example.com'
            })
        });
        const newUser = await response.json();
        console.log('User created:', newUser);
    } catch (error) {
        console.error('Failed to create user:', error);
    }
}

// demonstrateMicroservices();

7. Testing & Debugging

// math.js - Module to test
function add(a, b) {
    return a + b;
}

function subtract(a, b) {
    return a - b;
}

function multiply(a, b) {
    return a * b;
}

function divide(a, b) {
    if (b === 0) {
        throw new Error('Division by zero');
    }
    return a / b;
}

function factorial(n) {
    if (n < 0) throw new Error('Factorial is not defined for negative numbers');
    if (n === 0 || n === 1) return 1;
    return n * factorial(n - 1);
}

function isPrime(num) {
    if (num <= 1) return false;
    if (num <= 3) return true;
    if (num % 2 === 0 || num % 3 === 0) return false;
    
    for (let i = 5; i * i <= num; i += 6) {
        if (num % i === 0 || num % (i + 2) === 0) return false;
    }
    return true;
}

module.exports = {
    add,
    subtract,
    multiply,
    divide,
    factorial,
    isPrime
};

// math.test.js - Unit tests
const {
    add,
    subtract,
    multiply,
    divide,
    factorial,
    isPrime
} = require('./math');

describe('Math operations', () => {
    describe('add', () => {
        test('adds two positive numbers correctly', () => {
            expect(add(2, 3)).toBe(5);
        });

        test('adds negative numbers correctly', () => {
            expect(add(-1, -2)).toBe(-3);
        });

        test('adds zero correctly', () => {
            expect(add(5, 0)).toBe(5);
        });
    });

    describe('subtract', () => {
        test('subtracts numbers correctly', () => {
            expect(subtract(5, 3)).toBe(2);
        });

        test('handles negative results', () => {
            expect(subtract(3, 5)).toBe(-2);
        });
    });

    describe('multiply', () => {
        test('multiplies numbers correctly', () => {
            expect(multiply(4, 5)).toBe(20);
        });

        test('multiplies by zero', () => {
            expect(multiply(5, 0)).toBe(0);
        });
    });

    describe('divide', () => {
        test('divides numbers correctly', () => {
            expect(divide(10, 2)).toBe(5);
        });

        test('throws error when dividing by zero', () => {
            expect(() => divide(10, 0)).toThrow('Division by zero');
        });
    });

    describe('factorial', () => {
        test('calculates factorial of 0', () => {
            expect(factorial(0)).toBe(1);
        });

        test('calculates factorial of 5', () => {
            expect(factorial(5)).toBe(120);
        });

        test('throws error for negative numbers', () => {
            expect(() => factorial(-1)).toThrow('Factorial is not defined for negative numbers');
        });
    });

    describe('isPrime', () => {
        test('identifies prime numbers', () => {
            expect(isPrime(2)).toBe(true);
            expect(isPrime(17)).toBe(true);
            expect(isPrime(97)).toBe(true);
        });

        test('identifies non-prime numbers', () => {
            expect(isPrime(1)).toBe(false);
            expect(isPrime(4)).toBe(false);
            expect(isPrime(100)).toBe(false);
        });
    });
});

// user-service.js - Service to test
class UserService {
    constructor(database) {
        this.database = database;
        this.users = new Map();
    }

    async createUser(userData) {
        const { username, email, password } = userData;
        
        if (!username || !email || !password) {
            throw new Error('All fields are required');
        }

        if (this.users.has(email)) {
            throw new Error('User already exists');
        }

        const user = {
            id: Date.now().toString(),
            username,
            email,
            password, // In real app, hash this
            createdAt: new Date().toISOString()
        };

        this.users.set(email, user);
        return user;
    }

    async getUserByEmail(email) {
        const user = this.users.get(email);
        if (!user) {
            throw new Error('User not found');
        }
        return user;
    }

    async updateUser(email, updates) {
        const user = this.users.get(email);
        if (!user) {
            throw new Error('User not found');
        }

        const updatedUser = { ...user, ...updates, updatedAt: new Date().toISOString() };
        this.users.set(email, updatedUser);
        return updatedUser;
    }

    async deleteUser(email) {
        if (!this.users.has(email)) {
            throw new Error('User not found');
        }
        this.users.delete(email);
        return true;
    }

    async listUsers() {
        return Array.from(this.users.values());
    }
}

// user-service.test.js - Service tests
describe('UserService', () => {
    let userService;

    beforeEach(() => {
        userService = new UserService();
    });

    afterEach(() => {
        // Clean up after each test
        userService.users.clear();
    });

    describe('createUser', () => {
        test('creates user successfully with valid data', async () => {
            const userData = {
                username: 'john_doe',
                email: 'john@example.com',
                password: 'securepassword'
            };

            const user = await userService.createUser(userData);

            expect(user).toHaveProperty('id');
            expect(user.username).toBe(userData.username);
            expect(user.email).toBe(userData.email);
            expect(user.createdAt).toBeDefined();
        });

        test('throws error when required fields are missing', async () => {
            const invalidData = { username: 'john' };

            await expect(userService.createUser(invalidData))
                .rejects.toThrow('All fields are required');
        });

        test('throws error when user already exists', async () => {
            const userData = {
                username: 'john_doe',
                email: 'john@example.com',
                password: 'securepassword'
            };

            await userService.createUser(userData);

            await expect(userService.createUser(userData))
                .rejects.toThrow('User already exists');
        });
    });

    describe('getUserByEmail', () => {
        test('returns user when found', async () => {
            const userData = {
                username: 'jane_doe',
                email: 'jane@example.com',
                password: 'password123'
            };

            await userService.createUser(userData);
            const user = await userService.getUserByEmail('jane@example.com');

            expect(user.email).toBe('jane@example.com');
        });

        test('throws error when user not found', async () => {
            await expect(userService.getUserByEmail('nonexistent@example.com'))
                .rejects.toThrow('User not found');
        });
    });

    describe('updateUser', () => {
        test('updates user successfully', async () => {
            const userData = {
                username: 'original',
                email: 'user@example.com',
                password: 'password'
            };

            await userService.createUser(userData);
            const updated = await userService.updateUser('user@example.com', {
                username: 'updated'
            });

            expect(updated.username).toBe('updated');
            expect(updated.updatedAt).toBeDefined();
        });
    });

    describe('deleteUser', () => {
        test('deletes user successfully', async () => {
            const userData = {
                username: 'todelete',
                email: 'delete@example.com',
                password: 'password'
            };

            await userService.createUser(userData);
            const result = await userService.deleteUser('delete@example.com');

            expect(result).toBe(true);
            await expect(userService.getUserByEmail('delete@example.com'))
                .rejects.toThrow('User not found');
        });
    });
});

// API endpoint testing with Supertest
const request = require('supertest');
const express = require('express');

function createTestApp() {
    const app = express();
    app.use(express.json());

    const userService = new UserService();

    app.post('/users', async (req, res) => {
        try {
            const user = await userService.createUser(req.body);
            res.status(201).json(user);
        } catch (error) {
            res.status(400).json({ error: error.message });
        }
    });

    app.get('/users/:email', async (req, res) => {
        try {
            const user = await userService.getUserByEmail(req.params.email);
            res.json(user);
        } catch (error) {
            res.status(404).json({ error: error.message });
        }
    });

    app.put('/users/:email', async (req, res) => {
        try {
            const user = await userService.updateUser(req.params.email, req.body);
            res.json(user);
        } catch (error) {
            res.status(404).json({ error: error.message });
        }
    });

    app.delete('/users/:email', async (req, res) => {
        try {
            await userService.deleteUser(req.params.email);
            res.status(204).send();
        } catch (error) {
            res.status(404).json({ error: error.message });
        }
    });

    return app;
}

// API tests
describe('User API', () => {
    let app;

    beforeAll(() => {
        app = createTestApp();
    });

    describe('POST /users', () => {
        test('creates user successfully', async () => {
            const userData = {
                username: 'api_user',
                email: 'api@example.com',
                password: 'password'
            };

            const response = await request(app)
                .post('/users')
                .send(userData)
                .expect(201);

            expect(response.body).toHaveProperty('id');
            expect(response.body.email).toBe(userData.email);
        });

        test('returns 400 for invalid data', async () => {
            const invalidData = { username: 'incomplete' };

            const response = await request(app)
                .post('/users')
                .send(invalidData)
                .expect(400);

            expect(response.body).toHaveProperty('error');
        });
    });

    describe('GET /users/:email', () => {
        test('returns user when found', async () => {
            const userData = {
                username: 'get_user',
                email: 'get@example.com',
                password: 'password'
            };

            await request(app).post('/users').send(userData);

            const response = await request(app)
                .get('/users/get@example.com')
                .expect(200);

            expect(response.body.email).toBe('get@example.com');
        });

        test('returns 404 when user not found', async () => {
            const response = await request(app)
                .get('/users/nonexistent@example.com')
                .expect(404);

            expect(response.body).toHaveProperty('error');
        });
    });
});

// Mocking and spies
const axios = require('axios');

class WeatherService {
    constructor(apiKey) {
        this.apiKey = apiKey;
        this.baseURL = 'https://api.weatherapi.com/v1';
    }

    async getCurrentWeather(city) {
        const response = await axios.get(`${this.baseURL}/current.json`, {
            params: {
                key: this.apiKey,
                q: city
            }
        });
        return response.data;
    }

    async getForecast(city, days = 3) {
        const response = await axios.get(`${this.baseURL}/forecast.json`, {
            params: {
                key: this.apiKey,
                q: city,
                days: days
            }
        });
        return response.data;
    }
}

// Weather service tests with mocking
jest.mock('axios');
const mockedAxios = axios;

describe('WeatherService', () => {
    let weatherService;

    beforeEach(() => {
        weatherService = new WeatherService('test-api-key');
        jest.clearAllMocks();
    });

    test('getCurrentWeather returns weather data', async () => {
        const mockWeatherData = {
            location: { name: 'London' },
            current: { temp_c: 15, condition: { text: 'Cloudy' } }
        };

        mockedAxios.get.mockResolvedValue({ data: mockWeatherData });

        const weather = await weatherService.getCurrentWeather('London');

        expect(weather).toEqual(mockWeatherData);
        expect(mockedAxios.get).toHaveBeenCalledWith(
            'https://api.weatherapi.com/v1/current.json',
            {
                params: {
                    key: 'test-api-key',
                    q: 'London'
                }
            }
        );
    });

    test('getForecast returns forecast data', async () => {
        const mockForecastData = {
            location: { name: 'Paris' },
            forecast: { forecastday: [] }
        };

        mockedAxios.get.mockResolvedValue({ data: mockForecastData });

        const forecast = await weatherService.getForecast('Paris', 5);

        expect(forecast).toEqual(mockForecastData);
        expect(mockedAxios.get).toHaveBeenCalledWith(
            'https://api.weatherapi.com/v1/forecast.json',
            {
                params: {
                    key: 'test-api-key',
                    q: 'Paris',
                    days: 5
                }
            }
        );
    });

    test('handles API errors', async () => {
        const errorMessage = 'API Error';
        mockedAxios.get.mockRejectedValue(new Error(errorMessage));

        await expect(weatherService.getCurrentWeather('InvalidCity'))
            .rejects.toThrow(errorMessage);
    });
});

// Test utilities and helpers
class TestUtils {
    static async retry(fn, retries = 3, delay = 1000) {
        for (let i = 0; i < retries; i++) {
            try {
                return await fn();
            } catch (error) {
                if (i === retries - 1) throw error;
                await new Promise(resolve => setTimeout(resolve, delay));
            }
        }
    }

    static generateTestUser(overrides = {}) {
        const baseUser = {
            username: `testuser_${Date.now()}`,
            email: `test_${Date.now()}@example.com`,
            password: 'testpassword123',
            profile: {
                firstName: 'Test',
                lastName: 'User'
            }
        };

        return { ...baseUser, ...overrides };
    }

    static async waitForCondition(condition, timeout = 5000) {
        const startTime = Date.now();
        
        while (Date.now() - startTime < timeout) {
            if (await condition()) return true;
            await new Promise(resolve => setTimeout(resolve, 100));
        }
        
        throw new Error('Condition not met within timeout');
    }
}

// Integration tests
describe('Integration Tests', () => {
    let app;
    let server;

    beforeAll(async () => {
        app = createTestApp();
        server = app.listen(0); // Random port
    });

    afterAll(async () => {
        await new Promise(resolve => server.close(resolve));
    });

    test('full user lifecycle', async () => {
        const userData = TestUtils.generateTestUser();

        // Create user
        const createResponse = await request(app)
            .post('/users')
            .send(userData)
            .expect(201);

        const userId = createResponse.body.id;

        // Get user
        const getResponse = await request(app)
            .get(`/users/${userData.email}`)
            .expect(200);

        expect(getResponse.body.id).toBe(userId);

        // Update user
        const updateResponse = await request(app)
            .put(`/users/${userData.email}`)
            .send({ username: 'updated_username' })
            .expect(200);

        expect(updateResponse.body.username).toBe('updated_username');

        // Delete user
        await request(app)
            .delete(`/users/${userData.email}`)
            .expect(204);

        // Verify deletion
        await request(app)
            .get(`/users/${userData.email}`)
            .expect(404);
    });
});

// Performance testing
describe('Performance Tests', () => {
    test('user creation performance', async () => {
        const userService = new UserService();
        const startTime = Date.now();
        const iterations = 1000;

        for (let i = 0; i < iterations; i++) {
            await userService.createUser({
                username: `perfuser_${i}`,
                email: `perf_${i}@example.com`,
                password: 'password'
            });
        }

        const endTime = Date.now();
        const duration = endTime - startTime;
        const operationsPerSecond = iterations / (duration / 1000);

        console.log(`User creation: ${operationsPerSecond.toFixed(2)} ops/sec`);
        
        expect(operationsPerSecond).toBeGreaterThan(100); // At least 100 ops/sec
    }, 30000); // 30 second timeout
});

// Test configuration
module.exports = {
    testEnvironment: 'node',
    setupFilesAfterEnv: ['<rootDir>/test-setup.js'],
    collectCoverageFrom: [
        '**/*.js',
        '!**/node_modules/**',
        '!**/coverage/**',
        '!**/test/**'
    ],
    coverageThreshold: {
        global: {
            branches: 80,
            functions: 80,
            lines: 80,
            statements: 80
        }
    },
    testMatch: [
        '**/__tests__/**/*.js',
        '**/*.test.js'
    ],
    verbose: true,
    forceExit: true,
    clearMocks: true,
    resetMocks: true,
    restoreMocks: true
};

const express = require('express');
const app = express();

// Debug module for conditional logging
const debug = require('debug');

// Create debuggers for different modules
const debugApp = debug('app:server');
const debugDB = debug('app:database');
const debugAuth = debug('app:auth');
const debugError = debug('app:error');

// Enable debugging: DEBUG=app:* node app.js

// Morgan for HTTP request logging
const morgan = require('morgan');

// Custom morgan token for request ID
morgan.token('id', (req) => req.id || '-');

// Custom format
const morganFormat = ':id :method :url :status :res[content-length] - :response-time ms';

app.use((req, res, next) => {
    req.id = Date.now().toString(36) + Math.random().toString(36).substr(2);
    next();
});

app.use(morgan(morganFormat, {
    stream: {
        write: (message) => {
            debugApp(message.trim());
        }
    }
}));

// Winston for structured logging
const winston = require('winston');

const logger = winston.createLogger({
    level: 'info',
    format: winston.format.combine(
        winston.format.timestamp(),
        winston.format.errors({ stack: true }),
        winston.format.json()
    ),
    defaultMeta: { service: 'user-service' },
    transports: [
        new winston.transports.File({ filename: 'error.log', level: 'error' }),
        new winston.transports.File({ filename: 'combined.log' }),
        new winston.transports.Console({
            format: winston.format.combine(
                winston.format.colorize(),
                winston.format.simple()
            )
        })
    ]
});

// Custom middleware for request logging
app.use((req, res, next) => {
    const start = Date.now();
    
    res.on('finish', () => {
        const duration = Date.now() - start;
        
        logger.info('HTTP request', {
            method: req.method,
            url: req.url,
            status: res.statusCode,
            duration: duration,
            userAgent: req.get('User-Agent'),
            ip: req.ip
        });
        
        if (res.statusCode >= 400) {
            logger.warn('HTTP error', {
                method: req.method,
                url: req.url,
                status: res.statusCode,
                duration: duration
            });
        }
    });
    
    next();
});

// Error handling middleware
app.use((error, req, res, next) => {
    logger.error('Unhandled error', {
        error: error.message,
        stack: error.stack,
        url: req.url,
        method: req.method,
        ip: req.ip
    });
    
    res.status(500).json({
        error: 'Internal server error',
        ...(process.env.NODE_ENV === 'development' && { details: error.message })
    });
});

// Performance monitoring
const performanceMonitor = {
    requests: new Map(),
    errors: new Map(),
    
    recordRequest(method, path, duration, statusCode) {
        const key = `${method} ${path}`;
        
        if (!this.requests.has(key)) {
            this.requests.set(key, {
                count: 0,
                totalDuration: 0,
                minDuration: Infinity,
                maxDuration: 0,
                statusCodes: {}
            });
        }
        
        const stats = this.requests.get(key);
        stats.count++;
        stats.totalDuration += duration;
        stats.minDuration = Math.min(stats.minDuration, duration);
        stats.maxDuration = Math.max(stats.maxDuration, duration);
        
        if (!stats.statusCodes[statusCode]) {
            stats.statusCodes[statusCode] = 0;
        }
        stats.statusCodes[statusCode]++;
    },
    
    getStats() {
        const stats = {};
        for (let [key, data] of this.requests) {
            stats[key] = {
                ...data,
                averageDuration: data.totalDuration / data.count
            };
        }
        return stats;
    },
    
    reset() {
        this.requests.clear();
        this.errors.clear();
    }
};

// Memory usage monitoring
function setupMemoryMonitoring() {
    const used = process.memoryUsage();
    
    setInterval(() => {
        const current = process.memoryUsage();
        const changes = {};
        
        for (let key in used) {
            changes[key] = {
                current: Math.round(current[key] / 1024 / 1024 * 100) / 100,
                previous: Math.round(used[key] / 1024 / 1024 * 100) / 100,
                change: Math.round((current[key] - used[key]) / 1024 / 1024 * 100) / 100
            };
        }
        
        logger.debug('Memory usage', changes);
        Object.assign(used, current);
    }, 30000); // Every 30 seconds
}

// CPU profiling
const v8 = require('v8');
const fs = require('fs').promises;

async function takeHeapSnapshot() {
    const snapshot = v8.getHeapSnapshot();
    const filename = `heap-snapshot-${Date.now()}.heapsnapshot`;
    await fs.writeFile(filename, JSON.stringify(snapshot));
    logger.info(`Heap snapshot saved: ${filename}`);
}

// Process monitoring
function monitorProcess() {
    // Monitor event loop lag
    let lastLoop = Date.now();
    
    setInterval(() => {
        const now = Date.now();
        const loopLag = now - lastLoop - 1000; // Should be close to 0
        lastLoop = now;
        
        if (loopLag > 100) {
            logger.warn('Event loop lag detected', { lag: loopLag });
        }
    }, 1000);
    
    // Monitor active handles and requests
    setInterval(() => {
        const activeHandles = process._getActiveHandles().length;
        const activeRequests = process._getActiveRequests().length;
        
        logger.debug('Process activity', {
            activeHandles,
            activeRequests,
            memory: process.memoryUsage(),
            uptime: process.uptime()
        });
    }, 60000); // Every minute
}

// Debugging endpoints
app.get('/debug/performance', (req, res) => {
    res.json(performanceMonitor.getStats());
});

app.get('/debug/memory', (req, res) => {
    res.json({
        memory: process.memoryUsage(),
        uptime: process.uptime(),
        version: process.version
    });
});

app.post('/debug/heap-snapshot', async (req, res) => {
    try {
        await takeHeapSnapshot();
        res.json({ message: 'Heap snapshot created' });
    } catch (error) {
        logger.error('Failed to create heap snapshot', { error: error.message });
        res.status(500).json({ error: 'Failed to create snapshot' });
    }
});

// Node.js Inspector
// Start with: node --inspect app.js
// Then open chrome://inspect in Chrome

// Clinic.js for performance profiling
// clinic doctor -- node app.js
// clinic flame -- node app.js
// clinic bubbleprof -- node app.js

// Custom debugging utilities
class DebugHelper {
    static async measureExecutionTime(fn, context = 'unknown') {
        const start = process.hrtime.bigint();
        const result = await fn();
        const end = process.hrtime.bigint();
        const duration = Number(end - start) / 1000000; // Convert to milliseconds
        
        logger.debug('Execution time', {
            context,
            duration: duration.toFixed(2) + 'ms'
        });
        
        return { result, duration };
    }
    
    static createTracer(moduleName) {
        return {
            enter: (functionName, args = {}) => {
                logger.debug('Function entered', {
                    module: moduleName,
                    function: functionName,
                    args: this.sanitizeArgs(args)
                });
            },
            
            exit: (functionName, result) => {
                logger.debug('Function exited', {
                    module: moduleName,
                    function: functionName,
                    result: this.sanitizeResult(result)
                });
            },
            
            error: (functionName, error) => {
                logger.error('Function error', {
                    module: moduleName,
                    function: functionName,
                    error: error.message,
                    stack: error.stack
                });
            }
        };
    }
    
    static sanitizeArgs(args) {
        // Remove sensitive data from logging
        const sanitized = { ...args };
        if (sanitized.password) sanitized.password = '***';
        if (sanitized.token) sanitized.token = '***';
        return sanitized;
    }
    
    static sanitizeResult(result) {
        // Sanitize sensitive data in results
        if (result && typeof result === 'object') {
            const sanitized = { ...result };
            if (sanitized.password) sanitized.password = '***';
            if (sanitized.token) sanitized.token = '***';
            return sanitized;
        }
        return result;
    }
}

// Example usage with tracing
class UserServiceWithTracing {
    constructor() {
        this.tracer = DebugHelper.createTracer('UserService');
        this.users = new Map();
    }
    
    async createUser(userData) {
        this.tracer.enter('createUser', userData);
        
        try {
            if (!userData.username || !userData.email) {
                throw new Error('Username and email are required');
            }
            
            const user = {
                id: Date.now().toString(),
                ...userData,
                createdAt: new Date().toISOString()
            };
            
            this.users.set(user.email, user);
            this.tracer.exit('createUser', user);
            return user;
        } catch (error) {
            this.tracer.error('createUser', error);
            throw error;
        }
    }
}

// Async context tracking for request tracing
const { AsyncLocalStorage } = require('async_hooks');

const asyncLocalStorage = new AsyncLocalStorage();

app.use((req, res, next) => {
    const store = {
        requestId: req.id,
        startTime: Date.now(),
        user: req.user // if authenticated
    };
    
    asyncLocalStorage.run(store, () => {
        next();
    });
});

// Enhanced logger with context
const contextLogger = {
    info(message, meta = {}) {
        const store = asyncLocalStorage.getStore();
        logger.info(message, { ...store, ...meta });
    },
    
    error(message, error, meta = {}) {
        const store = asyncLocalStorage.getStore();
        logger.error(message, { 
            ...store, 
            ...meta,
            error: error.message,
            stack: error.stack 
        });
    },
    
    debug(message, meta = {}) {
        const store = asyncLocalStorage.getStore();
        logger.debug(message, { ...store, ...meta });
    }
};

// Usage in route handlers
app.get('/api/users/:id', async (req, res) => {
    const { measureExecutionTime } = DebugHelper;
    
    try {
        const { result: user, duration } = await measureExecutionTime(
            () => getUserFromDatabase(req.params.id),
            'getUser'
        );
        
        contextLogger.info('User retrieved', { userId: req.params.id, duration });
        
        if (!user) {
            contextLogger.debug('User not found', { userId: req.params.id });
            return res.status(404).json({ error: 'User not found' });
        }
        
        res.json(user);
    } catch (error) {
        contextLogger.error('Failed to get user', error, { userId: req.params.id });
        res.status(500).json({ error: 'Internal server error' });
    }
});

// Mock function for demonstration
async function getUserFromDatabase(userId) {
    await new Promise(resolve => setTimeout(resolve, 100)); // Simulate DB call
    return { id: userId, name: 'John Doe', email: 'john@example.com' };
}

// Health check with detailed metrics
app.get('/health/detailed', async (req, res) => {
    const health = {
        status: 'OK',
        timestamp: new Date().toISOString(),
        uptime: process.uptime(),
        memory: process.memoryUsage(),
        versions: process.versions,
        performance: performanceMonitor.getStats()
    };
    
    try {
        // Check database connection
        health.database = 'connected';
    } catch (error) {
        health.database = 'disconnected';
        health.status = 'ERROR';
    }
    
    // Check memory pressure
    const memoryUsage = process.memoryUsage();
    const memoryPressure = memoryUsage.heapUsed / memoryUsage.heapTotal;
    health.memoryPressure = memoryPressure;
    
    if (memoryPressure > 0.8) {
        health.status = 'WARNING';
        health.message = 'High memory pressure';
    }
    
    res.json(health);
});

// Graceful shutdown with debugging
function setupGracefulShutdown() {
    const shutdownSignals = ['SIGTERM', 'SIGINT', 'SIGUSR2'];
    
    shutdownSignals.forEach(signal => {
        process.on(signal, async () => {
            contextLogger.info(`Received ${signal}, starting graceful shutdown`);
            
            // Stop accepting new requests
            server.close(() => {
                contextLogger.info('HTTP server closed');
            });
            
            // Close database connections, etc.
            await closeAllConnections();
            
            // Take final heap snapshot in development
            if (process.env.NODE_ENV === 'development') {
                await takeHeapSnapshot();
            }
            
            contextLogger.info('Graceful shutdown completed');
            process.exit(0);
        });
    });
}

async function closeAllConnections() {
    // Close database connections, Redis connections, etc.
    contextLogger.info('Closing all connections');
    await new Promise(resolve => setTimeout(resolve, 1000));
}

// Start the application
const PORT = process.env.PORT || 3000;
const server = app.listen(PORT, () => {
    logger.info(`Server started on port ${PORT}`, {
        environment: process.env.NODE_ENV,
        nodeVersion: process.version
    });
    
    // Start monitoring
    setupMemoryMonitoring();
    monitorProcess();
    setupGracefulShutdown();
});

module.exports = app;

8. Deployment & DevOps

# Dockerfile for Node.js application
# Multi-stage build for optimized production image

# Stage 1: Dependencies and build
FROM node:18-alpine AS builder

# Install build dependencies
RUN apk add --no-cache     python3     make     g++     git

# Set working directory
WORKDIR /app

# Copy package files
COPY package*.json ./
COPY yarn.lock ./

# Install dependencies
RUN npm ci --only=production && npm cache clean --force

# Stage 2: Production image
FROM node:18-alpine AS production

# Install runtime dependencies
RUN apk add --no-cache     tini     curl

# Create app user
RUN addgroup -g 1001 -S nodejs
RUN adduser -S nodejs -u 1001

# Set working directory
WORKDIR /app

# Copy node_modules from builder stage
COPY --from=builder --chown=nodejs:nodejs /app/node_modules ./node_modules

# Copy application code
COPY --chown=nodejs:nodejs . .

# Create necessary directories
RUN mkdir -p logs tmp && chown -R nodejs:nodejs logs tmp

# Switch to non-root user
USER nodejs

# Expose port
EXPOSE 3000

# Health check
HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
    CMD curl -f http://localhost:3000/health || exit 1

# Use tini as init process
ENTRYPOINT ["/sbin/tini", "--"]

# Start application
CMD ["node", "server.js"]

# docker-compose.yml for development
version: '3.8'

services:
  app:
    build:
      context: .
      target: development
    ports:
      - "3000:3000"
    environment:
      - NODE_ENV=development
      - DATABASE_URL=postgresql://user:password@db:5432/mydb
      - REDIS_URL=redis://redis:6379
    volumes:
      - .:/app
      - /app/node_modules
    depends_on:
      - db
      - redis
    command: npm run dev

  db:
    image: postgres:14-alpine
    environment:
      - POSTGRES_DB=mydb
      - POSTGRES_USER=user
      - POSTGRES_PASSWORD=password
    ports:
      - "5432:5432"
    volumes:
      - postgres_data:/var/lib/postgresql/data

  redis:
    image: redis:7-alpine
    ports:
      - "6379:6379"
    command: redis-server --appendonly yes
    volumes:
      - redis_data:/data

  nginx:
    image: nginx:alpine
    ports:
      - "80:80"
    volumes:
      - ./nginx.conf:/etc/nginx/nginx.conf
    depends_on:
      - app

volumes:
  postgres_data:
  redis_data:

# docker-compose.prod.yml for production
version: '3.8'

services:
  app:
    build:
      context: .
      target: production
    environment:
      - NODE_ENV=production
      - DATABASE_URL=postgresql://user:password@db:5432/mydb
      - REDIS_URL=redis://redis:6379
    deploy:
      replicas: 3
      restart_policy:
        condition: on-failure
      resources:
        limits:
          memory: 512M
        reservations:
          memory: 256M
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:3000/health"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 40s

  nginx:
    image: nginx:alpine
    ports:
      - "80:80"
    volumes:
      - ./nginx.prod.conf:/etc/nginx/nginx.conf
    deploy:
      replicas: 2
      resources:
        limits:
          memory: 128M

# nginx.conf for load balancing
events {
    worker_connections 1024;
}

http {
    upstream nodejs_app {
        server app:3000;
    }

    server {
        listen 80;
        
        # Security headers
        add_header X-Frame-Options "SAMEORIGIN" always;
        add_header X-XSS-Protection "1; mode=block" always;
        add_header X-Content-Type-Options "nosniff" always;
        add_header Referrer-Policy "no-referrer-when-downgrade" always;
        
        # Gzip compression
        gzip on;
        gzip_vary on;
        gzip_min_length 1024;
        gzip_types text/plain text/css text/xml text/javascript application/javascript application/xml+rss application/json;

        location / {
            proxy_pass http://nodejs_app;
            proxy_http_version 1.1;
            proxy_set_header Upgrade $http_upgrade;
            proxy_set_header Connection 'upgrade';
            proxy_set_header Host $host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Proto $scheme;
            proxy_cache_bypass $http_upgrade;
            proxy_connect_timeout 30s;
            proxy_send_timeout 30s;
            proxy_read_timeout 30s;
        }

        location /health {
            proxy_pass http://nodejs_app;
            access_log off;
        }
    }
}

# package.json scripts for Docker
/*
{
  "scripts": {
    "dev": "nodemon server.js",
    "start": "node server.js",
    "docker:build": "docker build -t my-app .",
    "docker:run": "docker run -p 3000:3000 my-app",
    "docker:compose:dev": "docker-compose up",
    "docker:compose:prod": "docker-compose -f docker-compose.prod.yml up",
    "docker:compose:build": "docker-compose build",
    "docker:compose:down": "docker-compose down"
  }
}
*/

# Dockerfile for development
FROM node:18-alpine AS development

# Install development tools
RUN apk add --no-cache \
    python3 \
    make \
    g++ \
    git \
    curl

WORKDIR /app

# Copy package files
COPY package*.json ./
COPY yarn.lock ./

# Install all dependencies (including dev dependencies)
RUN npm install

# Copy source code
COPY . .

# Expose port
EXPOSE 3000

# Development command
CMD ["npm", "run", "dev"]

# Kubernetes deployment files
# deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: nodejs-app
  labels:
    app: nodejs-app
spec:
  replicas: 3
  selector:
    matchLabels:
      app: nodejs-app
  template:
    metadata:
      labels:
        app: nodejs-app
    spec:
      containers:
      - name: nodejs-app
        image: my-registry/nodejs-app:latest
        ports:
        - containerPort: 3000
        env:
        - name: NODE_ENV
          value: "production"
        - name: DATABASE_URL
          valueFrom:
            secretKeyRef:
              name: app-secrets
              key: database-url
        resources:
          requests:
            memory: "256Mi"
            cpu: "250m"
          limits:
            memory: "512Mi"
            cpu: "500m"
        livenessProbe:
          httpGet:
            path: /health
            port: 3000
          initialDelaySeconds: 30
          periodSeconds: 10
        readinessProbe:
          httpGet:
            path: /health
            port: 3000
          initialDelaySeconds: 5
          periodSeconds: 5
---
# service.yaml
apiVersion: v1
kind: Service
metadata:
  name: nodejs-app-service
spec:
  selector:
    app: nodejs-app
  ports:
  - port: 80
    targetPort: 3000
  type: LoadBalancer
---
# ingress.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: nodejs-app-ingress
  annotations:
    nginx.ingress.kubernetes.io/rewrite-target: /
spec:
  rules:
  - host: app.example.com
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: nodejs-app-service
            port:
              number: 80

# Docker optimization tips
# 1. Use .dockerignore
node_modules
npm-debug.log
Dockerfile*
docker-compose*
.git
.gitignore
README.md
.env
.nyc_output
coverage
.logs

# 2. Multi-stage build to reduce image size
# 3. Use specific version tags, not 'latest'
# 4. Run as non-root user for security
# 5. Use health checks
# 6. Set resource limits
# 7. Use volume for logs and temporary files

# Environment-specific configuration
class Config {
    static get database() {
        return {
            host: process.env.DB_HOST || 'localhost',
            port: parseInt(process.env.DB_PORT) || 5432,
            database: process.env.DB_NAME || 'mydb',
            username: process.env.DB_USER || 'user',
            password: process.env.DB_PASSWORD || 'password',
            ssl: process.env.NODE_ENV === 'production' ? { rejectUnauthorized: false } : false
        };
    }

    static get redis() {
        return {
            url: process.env.REDIS_URL || 'redis://localhost:6379',
            ...(process.env.NODE_ENV === 'production' && {
                password: process.env.REDIS_PASSWORD
            })
        };
    }

    static get server() {
        return {
            port: parseInt(process.env.PORT) || 3000,
            environment: process.env.NODE_ENV || 'development',
            logLevel: process.env.LOG_LEVEL || 'info'
        };
    }
}

# Health check implementation
app.get('/health', async (req, res) => {
    const health = {
        status: 'OK',
        timestamp: new Date().toISOString(),
        uptime: process.uptime(),
        memory: process.memoryUsage(),
        environment: process.env.NODE_ENV
    };

    try {
        // Check database connection
        await checkDatabaseConnection();
        health.database = 'connected';
    } catch (error) {
        health.database = 'disconnected';
        health.status = 'ERROR';
    }

    try {
        // Check Redis connection
        await checkRedisConnection();
        health.redis = 'connected';
    } catch (error) {
        health.redis = 'disconnected';
        health.status = 'ERROR';
    }

    res.status(health.status === 'OK' ? 200 : 503).json(health);
});

// CI/CD pipeline example (.github/workflows/deploy.yml)
\`name: Deploy Node.js App

on:
  push:
    branches: [ main ]
  pull_request:
    branches: [ main ]

jobs:
  test:
    runs-on: ubuntu-latest
    
    strategy:
      matrix:
        node-version: [16.x, 18.x]

    steps:
    - uses: actions/checkout@v3
    
    - name: Use Node.js ${{ matrix.node-version }}
      uses: actions/setup-node@v3
      with:
        node-version: ${{ matrix.node-version }}
        cache: 'npm'
    
    - name: Install dependencies
      run: npm ci
    
    - name: Run tests
      run: npm test
    
    - name: Run security audit
      run: npm audit --audit-level moderate
    
    - name: Upload coverage
      uses: codecov/codecov-action@v3

  build-and-deploy:
    needs: test
    runs-on: ubuntu-latest
    if: github.ref == 'refs/heads/main'
    
    steps:
    - uses: actions/checkout@v3
    
    - name: Build Docker image
      run: |
        docker build -t my-app:${{ github.sha }} .
        docker tag my-app:${{ github.sha }} my-registry/my-app:latest
    
    - name: Push to Docker Registry
      run: |
        echo ${{ secrets.DOCKER_PASSWORD }} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin
        docker push my-registry/my-app:latest
    
    - name: Deploy to Kubernetes
      run: |
        kubectl set image deployment/nodejs-app nodejs-app=my-registry/my-app:${{ github.sha }}
\`

// Environment configuration management
const fs = require('fs').promises;
const path = require('path');

class EnvironmentManager {
    constructor() {
        this.configs = new Map();
    }

    async loadEnvironment(env = 'development') {
        const envFile = path.join(__dirname, `.env.${env}`);
        
        try {
            const content = await fs.readFile(envFile, 'utf8');
            const lines = content.split('
');
            
            for (const line of lines) {
                const [key, ...valueParts] = line.split('=');
                if (key && !key.startsWith('#')) {
                    const value = valueParts.join('=').trim();
                    process.env[key] = this.parseValue(value);
                }
            }
            
            console.log(`Loaded environment: ${env}`);
        } catch (error) {
            console.warn(`Environment file not found: ${envFile}`);
        }
    }

    parseValue(value) {
        if (value === 'true') return true;
        if (value === 'false') return false;
        if (value === 'null') return null;
        if (value === 'undefined') return undefined;
        if (!isNaN(value) && value.trim() !== '') return Number(value);
        return value;
    }

    validateRequired(requiredVars) {
        const missing = requiredVars.filter(varName => !process.env[varName]);
        
        if (missing.length > 0) {
            throw new Error(`Missing required environment variables: ${missing.join(', ')}`);
        }
    }
}

// Production deployment checklist
const deploymentChecklist = {
    security: [
        'Use HTTPS in production',
        'Set secure headers (Helmet.js)',
        'Validate and sanitize all inputs',
        'Use environment variables for secrets',
        'Implement rate limiting',
        'Use CSRF protection',
        'Sanitize error messages',
        'Use secure cookies'
    ],
    
    performance: [
        'Enable gzip compression',
        'Implement caching strategies',
        'Use CDN for static assets',
        'Optimize database queries',
        'Use connection pooling',
        'Implement request compression',
        'Enable HTTP/2',
        'Use reverse proxy (Nginx)'
    ],
    
    monitoring: [
        'Set up logging (Winston/Morgan)',
        'Implement health checks',
        'Set up application metrics',
        'Configure error tracking',
        'Monitor resource usage',
        'Set up alerts',
        'Use APM tools (New Relic, Datadog)',
        'Log aggregation (ELK stack)'
    ],
    
    reliability: [
        'Use process manager (PM2)',
        'Implement graceful shutdown',
        'Set up load balancing',
        'Use database connection retries',
        'Implement circuit breakers',
        'Set up backup strategies',
        'Use multiple availability zones',
        'Implement auto-scaling'
    ]
};

// PM2 ecosystem file for process management
const pm2Config = {
    apps: [{
        name: 'nodejs-app',
        script: './server.js',
        instances: 'max',
        exec_mode: 'cluster',
        env: {
            NODE_ENV: 'development',
            PORT: 3000
        },
        env_production: {
            NODE_ENV: 'production',
            PORT: 3000
        },
        error_file: './logs/err.log',
        out_file: './logs/out.log',
        log_file: './logs/combined.log',
        time: true,
        max_memory_restart: '1G',
        node_args: '--max-old-space-size=1024',
        watch: false,
        merge_logs: true,
        kill_timeout: 3000,
        wait_ready: true,
        listen_timeout: 3000
    }]
};

// Load testing with Artillery
const artilleryConfig = {
    config: {
        target: 'http://localhost:3000',
        phases: [
            { duration: 60, arrivalRate: 10 },
            { duration: 120, arrivalRate: 20 },
            { duration: 60, arrivalRate: 5 }
        ],
        defaults: {
            headers: {
                'Content-Type': 'application/json'
            }
        }
    },
    scenarios: [
        {
            name: 'Health check',
            flow: [
                { get: { url: '/health' } }
            ]
        },
        {
            name: 'User operations',
            flow: [
                { post: { 
                    url: '/api/users',
                    json: {
                        username: 'testuser',
                        email: 'test@example.com',
                        password: 'password123'
                    }
                }},
                { get: { url: '/api/users/test@example.com' } }
            ]
        }
    ]
};

// SSL/TLS configuration with Express
const https = require('https');
const fs = require('fs');

function createHttpsServer(app) {
    const options = {
        key: fs.readFileSync('/path/to/private-key.pem'),
        cert: fs.readFileSync('/path/to/certificate.pem'),
        ca: fs.readFileSync('/path/to/ca-bundle.pem'),
        secureProtocol: 'TLSv1_2_method',
        ciphers: [
            'ECDHE-RSA-AES128-GCM-SHA256',
            'ECDHE-RSA-AES256-GCM-SHA384'
        ].join(':'),
        honorCipherOrder: true
    };

    return https.createServer(options, app);
}

// Zero-downtime deployment strategies
class DeploymentManager {
    constructor() {
        this.isShuttingDown = false;
    }

    setupGracefulShutdown(server) {
        process.on('SIGTERM', () => {
            console.log('Received SIGTERM, starting graceful shutdown');
            this.gracefulShutdown(server);
        });

        process.on('SIGINT', () => {
            console.log('Received SIGINT, starting graceful shutdown');
            this.gracefulShutdown(server);
        });
    }

    async gracefulShutdown(server) {
        this.isShuttingDown = true;

        // Stop accepting new connections
        server.close(() => {
            console.log('HTTP server closed');
        });

        // Close database connections
        await this.closeDatabaseConnections();

        // Close Redis connections
        await this.closeRedisConnections();

        // Wait for ongoing requests to complete
        await this.waitForOngoingRequests();

        console.log('Graceful shutdown completed');
        process.exit(0);
    }

    middleware() {
        return (req, res, next) => {
            if (this.isShuttingDown) {
                res.setHeader('Connection', 'close');
                res.status(503).json({ error: 'Server is shutting down' });
                return;
            }
            next();
        };
    }
}

// AWS Lambda function for serverless deployment
const awsServerlessExpress = require('aws-serverless-express');
const app = require('./app');

const server = awsServerlessExpress.createServer(app);

exports.handler = (event, context) => {
    return awsServerlessExpress.proxy(server, event, context, 'PROMISE').promise;
};

// Serverless framework configuration (serverless.yml)
\`
service: nodejs-app

provider:
  name: aws
  runtime: nodejs18.x
  region: us-east-1
  environment:
    NODE_ENV: production
    DATABASE_URL: ${env:DATABASE_URL}

functions:
  api:
    handler: handler.handler
    events:
      - http:
          path: /
          method: ANY
          cors: true
      - http:
          path: /{proxy+}
          method: ANY
          cors: true
    environment:
      NODE_ENV: production

plugins:
  - serverless-offline

custom:
  serverless-offline:
    httpPort: 4000
\`

// AWS SDK integration for cloud services
const AWS = require('aws-sdk');

class CloudServices {
    constructor() {
        this.s3 = new AWS.S3();
        this.sns = new AWS.SNS();
        this.sqs = new AWS.SQS();
        this.cloudWatch = new AWS.CloudWatch();
    }

    async uploadToS3(bucket, key, data) {
        const params = {
            Bucket: bucket,
            Key: key,
            Body: data,
            ContentType: 'application/json'
        };

        return this.s3.upload(params).promise();
    }

    async publishToSNS(topicArn, message) {
        const params = {
            Message: JSON.stringify(message),
            TopicArn: topicArn
        };

        return this.sns.publish(params).promise();
    }

    async sendToSQS(queueUrl, message) {
        const params = {
            QueueUrl: queueUrl,
            MessageBody: JSON.stringify(message)
        };

        return this.sqs.sendMessage(params).promise();
    }

    async putMetric(metricName, value, dimensions = []) {
        const params = {
            MetricData: [
                {
                    MetricName: metricName,
                    Value: value,
                    Timestamp: new Date(),
                    Dimensions: dimensions
                }
            ],
            Namespace: 'NodeJSApp'
        };

        return this.cloudWatch.putMetricData(params).promise();
    }
}

// Infrastructure as Code with Terraform
\`
# main.tf
terraform {
  required_providers {
    aws = {
      source  = "hashicorp/aws"
      version = "~> 4.0"
    }
  }
}

provider "aws" {
  region = "us-east-1"
}

resource "aws_ecr_repository" "app" {
  name = "nodejs-app"
}

resource "aws_ecs_cluster" "main" {
  name = "nodejs-app-cluster"
}

resource "aws_ecs_task_definition" "app" {
  family                   = "nodejs-app"
  network_mode             = "awsvpc"
  requires_compatibilities = ["FARGATE"]
  cpu                      = 512
  memory                   = 1024
  execution_role_arn       = aws_iam_role.ecs_task_execution_role.arn

  container_definitions = jsonencode([{
    name      = "nodejs-app"
    image     = "${aws_ecr_repository.app.repository_url}:latest"
    cpu       = 512
    memory    = 1024
    essential = true
    portMappings = [{
      containerPort = 3000
      hostPort      = 3000
    }]
    environment = [
      { name = "NODE_ENV", value = "production" }
    ]
    logConfiguration = {
      logDriver = "awslogs"
      options = {
        awslogs-group         = "/ecs/nodejs-app"
        awslogs-region        = "us-east-1"
        awslogs-stream-prefix = "ecs"
      }
    }
  }])
}

resource "aws_ecs_service" "app" {
  name            = "nodejs-app-service"
  cluster         = aws_ecs_cluster.main.id
  task_definition = aws_ecs_task_definition.app.arn
  desired_count   = 2
  launch_type     = "FARGATE"

  network_configuration {
    subnets          = aws_subnet.private.*.id
    security_groups  = [aws_security_group.ecs_tasks.id]
    assign_public_ip = false
  }

  load_balancer {
    target_group_arn = aws_lb_target_group.app.arn
    container_name   = "nodejs-app"
    container_port   = 3000
  }

  depends_on = [aws_lb_listener.app]
}
\`

// Monitoring and alerting configuration
const monitoringConfig = {
    application: {
        metrics: {
            http_requests_total: {
                type: 'counter',
                help: 'Total HTTP requests'
            },
            http_request_duration_seconds: {
                type: 'histogram',
                help: 'HTTP request duration in seconds',
                buckets: [0.1, 0.5, 1, 2, 5]
            },
            memory_usage_bytes: {
                type: 'gauge',
                help: 'Memory usage in bytes'
            }
        },
        
        alerts: {
            high_error_rate: {
                condition: 'rate(http_requests_total{status=~"5.."}[5m]) > 0.1',
                duration: '2m',
                severity: 'critical'
            },
            high_latency: {
                condition: 'histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m])) > 2',
                duration: '5m',
                severity: 'warning'
            },
            memory_leak: {
                condition: 'increase(memory_usage_bytes[1h]) > 100 * 1024 * 1024',
                severity: 'critical'
            }
        }
    }
};

// Log aggregation with structured logging
const winston = require('winston');
const { ElasticsearchTransport } = require('winston-elasticsearch');

const logger = winston.createLogger({
    level: 'info',
    format: winston.format.combine(
        winston.format.timestamp(),
        winston.format.errors({ stack: true }),
        winston.format.json()
    ),
    defaultMeta: { service: 'nodejs-app', environment: process.env.NODE_ENV },
    transports: [
        new winston.transports.Console({
            format: winston.format.combine(
                winston.format.colorize(),
                winston.format.simple()
            )
        }),
        new winston.transports.File({ 
            filename: 'logs/error.log', 
            level: 'error' 
        }),
        new winston.transports.File({ 
            filename: 'logs/combined.log' 
        }),
        new ElasticsearchTransport({
            level: 'info',
            index: 'nodejs-app-logs',
            clientOpts: {
                node: process.env.ELASTICSEARCH_URL || 'http://localhost:9200'
            }
        })
    ]
});

// Distributed tracing with OpenTelemetry
const { NodeTracerProvider } = require('@opentelemetry/node');
const { SimpleSpanProcessor } = require('@opentelemetry/tracing');
const { JaegerExporter } = require('@opentelemetry/exporter-jaeger');

const tracerProvider = new NodeTracerProvider();
tracerProvider.addSpanProcessor(
    new SimpleSpanProcessor(
        new JaegerExporter({
            serviceName: 'nodejs-app',
            tags: [],
            host: process.env.JAEGER_HOST || 'localhost',
            port: parseInt(process.env.JAEGER_PORT) || 6832
        })
    )
);
tracerProvider.register();

// Feature flags for controlled rollouts
class FeatureFlags {
    constructor() {
        this.flags = new Map();
    }

    setFlag(name, enabled, percentage = 100) {
        this.flags.set(name, { enabled, percentage });
    }

    isEnabled(name, userId = null) {
        const flag = this.flags.get(name);
        
        if (!flag) return false;
        if (!flag.enabled) return false;
        if (flag.percentage === 100) return true;
        
        // Percentage-based rollout
        if (userId) {
            const hash = this.hashCode(userId.toString());
            return (hash % 100) < flag.percentage;
        }
        
        return Math.random() * 100 < flag.percentage;
    }

    hashCode(str) {
        let hash = 0;
        for (let i = 0; i < str.length; i++) {
            hash = ((hash << 5) - hash) + str.charCodeAt(i);
            hash |= 0; // Convert to 32bit integer
        }
        return Math.abs(hash);
    }
}

// Usage in application
const featureFlags = new FeatureFlags();

// Enable new feature for 10% of users
featureFlags.setFlag('new_ui', true, 10);

app.get('/api/data', (req, res) => {
    const userId = req.user?.id;
    
    if (featureFlags.isEnabled('new_ui', userId)) {
        // New implementation
        return newImplementation(req, res);
    } else {
        // Old implementation
        return oldImplementation(req, res);
    }
});

// Blue-green deployment strategy
class DeploymentStrategy {
    constructor() {
        this.activeEnvironment = 'blue';
        this.weights = { blue: 100, green: 0 };
    }

    switchTraffic(blueWeight, greenWeight) {
        this.weights.blue = blueWeight;
        this.weights.green = greenWeight;
        
        console.log(`Traffic weights updated - Blue: ${blueWeight}%, Green: ${greenWeight}%`);
    }

    getTargetEnvironment() {
        const random = Math.random() * 100;
        return random < this.weights.blue ? 'blue' : 'green';
    }

    async performRollout() {
        // Step 1: Deploy to green environment
        console.log('Deploying to green environment...');
        await this.deployToGreen();
        
        // Step 2: Start routing small percentage to green
        this.switchTraffic(95, 5);
        await this.waitForStabilization(300000); // 5 minutes
        
        // Step 3: Increase traffic to green
        this.switchTraffic(50, 50);
        await this.waitForStabilization(300000);
        
        // Step 4: Route all traffic to green
        this.switchTraffic(0, 100);
        this.activeEnvironment = 'green';
        
        // Step 5: Keep blue for rollback capability
        setTimeout(() => {
            this.cleanupBlueEnvironment();
        }, 3600000); // Cleanup after 1 hour
    }

    async rollback() {
        console.log('Initiating rollback...');
        this.switchTraffic(100, 0);
        this.activeEnvironment = 'blue';
        await this.cleanupGreenEnvironment();
    }
}

// Security best practices implementation
const helmet = require('helmet');
const rateLimit = require('express-rate-limit');
const cors = require('cors');

// Security middleware configuration
app.use(helmet({
    contentSecurityPolicy: {
        directives: {
            defaultSrc: ["'self'"],
            styleSrc: ["'self'", "'unsafe-inline'"],
            scriptSrc: ["'self'"],
            imgSrc: ["'self'", "data:", "https:"]
        }
    },
    hsts: {
        maxAge: 31536000,
        includeSubDomains: true,
        preload: true
    }
}));

// Rate limiting
const limiter = rateLimit({
    windowMs: 15 * 60 * 1000, // 15 minutes
    max: 100, // limit each IP to 100 requests per windowMs
    message: 'Too many requests from this IP',
    standardHeaders: true,
    legacyHeaders: false
});

app.use(limiter);

// CORS configuration
app.use(cors({
    origin: process.env.ALLOWED_ORIGINS?.split(',') || ['http://localhost:3000'],
    credentials: true,
    methods: ['GET', 'POST', 'PUT', 'DELETE', 'OPTIONS']
}));

// Input validation and sanitization
const Joi = require('joi');

const userSchema = Joi.object({
    username: Joi.string().alphanum().min(3).max(30).required(),
    email: Joi.string().email().required(),
    password: Joi.string().pattern(new RegExp('^[a-zA-Z0-9]{8,30}$')).required(),
    role: Joi.string().valid('user', 'admin').default('user')
});

app.post('/api/users', async (req, res) => {
    try {
        const validatedData = await userSchema.validateAsync(req.body);
        // Process validated data
        const user = await userService.createUser(validatedData);
        res.status(201).json(user);
    } catch (error) {
        res.status(400).json({ error: error.details[0].message });
    }
});

// Final application startup with all configurations
async function startApplication() {
    try {
        // Load environment configuration
        const envManager = new EnvironmentManager();
        await envManager.loadEnvironment(process.env.NODE_ENV);
        
        // Validate required environment variables
        envManager.validateRequired([
            'DATABASE_URL',
            'REDIS_URL',
            'JWT_SECRET'
        ]);
        
        // Initialize database connections
        await initializeDatabase();
        
        // Initialize Redis connections
        await initializeRedis();
        
        // Setup monitoring
        setupMonitoring();
        
        // Start the server
        const PORT = process.env.PORT || 3000;
        const server = app.listen(PORT, () => {
            logger.info(`Application started successfully on port ${PORT}`, {
                environment: process.env.NODE_ENV,
                nodeVersion: process.version,
                pid: process.pid
            });
        });
        
        // Setup graceful shutdown
        const deploymentManager = new DeploymentManager();
        deploymentManager.setupGracefulShutdown(server);
        
        // Setup health checks
        setupHealthChecks();
        
        return server;
    } catch (error) {
        logger.error('Failed to start application', error);
        process.exit(1);
    }
}

// Start the application
if (require.main === module) {
    startApplication().catch(error => {
        console.error('Application startup failed:', error);
        process.exit(1);
    });
}

module.exports = app;

🎯 Conclusion: Mastering Node.js