'use strict'; /** * Cursor-Based Pagination Helper (Stripe API Style) * * Provides efficient pagination for large datasets using cursor-based approach * instead of common offset-based (skip/limit) approache which degrades performance on deep pagination. * * Benefits: * - Constant-time performance regardless of page depth * - Uses MongoDB indexes efficiently (_id index) * - Prevents Chrome DevTools cache eviction on large responses * - Compatible with Stripe API pagination patterns * * Usage: * const { buildCursorQuery, processCursorResults } = require('./helpers/cursor_pagination'); * * const pagination = buildCursorQuery(req.body, { defaultLimit: 1000 }); * const records = await Model.find(pagination.filter, null, pagination.options); * const result = processCursorResults(records, pagination.limit); */ const mongoose = require('mongoose'); const env = require('./env'); const { AppInputError } = require('./app_error'); const { Errors } = require('./constants'); // Pagination limits from environment variables const DEFAULT_PAGE_LIMIT = env.PAGINATION_DEFAULT_LIMIT; const MAX_PAGE_LIMIT = env.PAGINATION_MAX_LIMIT; /** * Build cursor-based query filter and options * @param {Object} params - Request parameters * @param {string} [params.startingAfter] - Cursor to start after (forward pagination) * @param {string} [params.endingBefore] - Cursor to end before (backward pagination) * @param {number} [params.limit] - Number of records per page (use -1 or 0 for all records) * @param {boolean} [params.returnAll] - If true, return all records without pagination * @param {Object} baseFilter - Base filter to apply (e.g., { fileId: '123' }) * @param {Object} options - Configuration options * @param {number} [options.defaultLimit] - Default limit if not specified (uses env.PAGINATION_DEFAULT_LIMIT) * @param {number} [options.maxLimit] - Maximum allowed limit (uses env.PAGINATION_MAX_LIMIT) * @param {string} [options.cursorField='_id'] - Field to use for cursor (must be indexed) * @param {boolean} [options.allowReturnAll=true] - Whether to allow returning all records * @returns {Object} Query configuration { filter, options, limit, isBackward, hasStartingAfter, hasEndingBefore, returnAll } */ function buildCursorQuery(params, baseFilter = {}, options = {}) { const { defaultLimit = DEFAULT_PAGE_LIMIT, maxLimit = MAX_PAGE_LIMIT, cursorField = '_id', allowReturnAll = true } = options; const startingAfter = params.startingAfter; const endingBefore = params.endingBefore; // Check if client wants all records const requestedLimit = parseInt(params.limit); const returnAll = params.returnAll === true || (allowReturnAll && (requestedLimit === -1 || requestedLimit === 0)); // If returning all, set limit to null, otherwise apply default and max const limit = returnAll ? null : Math.min(requestedLimit || defaultLimit, maxLimit); // Validate that only one cursor is used at a time if (startingAfter && endingBefore) { AppInputError.throw(undefined, 'Cannot use both startingAfter and endingBefore simultaneously'); } // Cursors are ignored when returning all records if (returnAll && (startingAfter || endingBefore)) { AppInputError.throw(undefined, 'Cannot use cursors when requesting all records (returnAll or limit=-1)'); } // Clone base filter to avoid mutation const filter = { ...baseFilter }; // Add cursor to filter (only if not returning all) if (!returnAll) { if (startingAfter) { // Fetch records AFTER the cursor (forward pagination) filter[cursorField] = { $gt: mongoose.Types.ObjectId(startingAfter) }; } else if (endingBefore) { // Fetch records BEFORE the cursor (backward pagination) filter[cursorField] = { $lt: mongoose.Types.ObjectId(endingBefore) }; } } // Determine sort direction (reverse for backward pagination) const sortDirection = endingBefore ? -1 : 1; const sortOption = { [cursorField]: sortDirection }; // Build query options const queryOptions = { lean: true, sort: sortOption }; // Only set limit if not returning all if (!returnAll) { queryOptions.limit = limit + 1; // Fetch one extra to determine if there are more records } return { filter, options: queryOptions, limit, isBackward: !!endingBefore, hasStartingAfter: !!startingAfter, hasEndingBefore: !!endingBefore, cursorField, returnAll }; } /** * Process cursor query results and extract pagination metadata * @param {Array} records - Query results * @param {number|null} limit - Requested limit (null if returning all) * @param {boolean} isBackward - Whether this is backward pagination * @param {string} [cursorField='_id'] - Field used for cursor * @param {boolean} [returnAll=false] - Whether all records are being returned * @returns {Object} Processed results { data, hasMore, startingAfter, endingBefore } */ function processCursorResults(records, limit, isBackward = false, cursorField = '_id', returnAll = false) { // If returning all records, no pagination metadata needed if (returnAll) { return { data: records, hasMore: false, returnAll: true, total: records.length }; } // Check if there are more records const hasMore = records.length > limit; // Remove the extra record if present if (hasMore) { records.pop(); } // For backward pagination, reverse the results to maintain correct order if (isBackward) { records.reverse(); } const result = { data: records, hasMore: hasMore }; // Provide cursor tokens for next/previous pages if (records.length > 0) { result.startingAfter = records[records.length - 1][cursorField]; // Cursor for next page result.endingBefore = records[0][cursorField]; // Cursor for previous page } return result; } /** * Complete cursor pagination helper - combines query building and result processing * @param {Object} Model - Mongoose model to query * @param {Object} params - Request parameters * @param {Object} baseFilter - Base filter to apply * @param {Object} options - Configuration options * @returns {Promise} Paginated results with metadata */ async function paginateWithCursor(Model, params, baseFilter = {}, options = {}) { // Validate parameters if (params.startingAfter && params.endingBefore) { AppInputError.throw(undefined, 'Cannot use both startingAfter and endingBefore simultaneously'); } // Build query const queryConfig = buildCursorQuery(params, baseFilter, options); // Execute query const records = await Model.find(queryConfig.filter, null, queryConfig.options); // Process results const result = processCursorResults( records, queryConfig.limit, queryConfig.isBackward, queryConfig.cursorField, queryConfig.returnAll ); return result; } /** * Validate cursor pagination parameters * @param {Object} params - Request parameters to validate * @returns {Object} Validation result { valid: boolean, error: string|null } */ function validateCursorParams(params) { if (params.startingAfter && params.endingBefore) { return { valid: false, error: 'Cannot use both startingAfter and endingBefore simultaneously' }; } if (params.startingAfter && !mongoose.Types.ObjectId.isValid(params.startingAfter)) { return { valid: false, error: 'Invalid startingAfter cursor: must be a valid ObjectId' }; } if (params.endingBefore && !mongoose.Types.ObjectId.isValid(params.endingBefore)) { return { valid: false, error: 'Invalid endingBefore cursor: must be a valid ObjectId' }; } if (params.limit && (isNaN(params.limit) || params.limit < 1)) { return { valid: false, error: 'Invalid limit: must be a positive number' }; } return { valid: true, error: null }; } /** * Express middleware for cursor pagination parameter validation * @param {Object} options - Configuration options * @returns {Function} Express middleware function */ function cursorPaginationMiddleware(options = {}) { return (req, res, next) => { const params = req.body || req.query; const validation = validateCursorParams(params); if (!validation.valid) { return AppInputError.create(undefined, validation.error); } next(); }; } module.exports = { buildCursorQuery, processCursorResults, paginateWithCursor, validateCursorParams, cursorPaginationMiddleware, DEFAULT_PAGE_LIMIT, MAX_PAGE_LIMIT };