'use strict';

/**
 * Job ID Migration and Copy Worker
 * 
 * This script provides two main functions:
 * 
 * 1. Migrate job IDs within the same database:
 *    - Takes specific job IDs and maps them to new IDs
 *    - Updates references in job_logs, job_assigns, applications, app_files, app_details, and invoices
 *    - Creates new jobs with new IDs and deletes the old ones in a transaction
 *    - Maintains all relationships
 * 
 * 2. Copy jobs from a previous database to the current one:
 *    - Connects to both source and destination databases
 *    - Copies jobs and all related data (logs, assignments, applications, app_files, app_details)
 *    - Properly maintains all relationships between entities
 *    - Optionally assigns new IDs to the copied jobs
 *    - Intelligently handles duplicate records with configurable linking behavior
 *    - Provides detailed statistics on created vs linked records
 *    - Implements robust error handling with retries and recovery
 *    - Shows progress with ETA and detailed statistics during migration
 * 
 * Key Features:
 *    - Transaction-level retry logic with exponential backoff for error resilience
 *    - Detailed statistics tracking for created vs linked records
 *    - Comprehensive error categorization and reporting
 *    - Progress tracking with estimated time remaining
 *    - Configurable behavior for duplicate records and error handling
 * 
 * Usage:
 *   # To migrate job IDs within the same database:
 *   DEBUG=agm:shift-job-ids node server/workers/migrateJobIds.js
 * 
 *   # To copy jobs from a previous database:
 *   DEBUG=agm:shift-job-ids SOURCE_DB_URI="mongodb://user:pass@host:port/dbname" JOB_IDS="123,456,789" node server/workers/migrateJobIds.js copy
 * 
 * Environment Variables:
 *   - DRY_RUN=true          # Only logs what would happen without making changes
 *   - SOURCE_DB_URI         # MongoDB URI for source database (required for copy operation)
 *   - JOB_IDS               # Comma-separated list of job IDs to copy (alternative to command-line argument)
 *   - USE_NEW_IDS=true      # Whether to assign new IDs to copied jobs (default: false)
 *   - KEEP_RELATED_IDS=true # Whether to keep original IDs for applications, app files, and app details (default: false)
 *   - LINK_TO_EXISTING=false # Whether to link related data to existing records when duplicates are found (default: true)
 *   - COPY_APP_DETAILS=false # Whether to copy application details (default: true)
 *   - MAX_RETRIES=3         # Maximum number of retries for network errors and transactions (default: 3)
 *   - RETRY_DELAY=1000      # Base delay in ms between retries, will be multiplied by 2^retryCount (default: 1000)
 *   - SHOW_PROGRESS=true    # Whether to show a progress indicator during migration (default: true)
 *   - USE_CHUNKED_PROCESSING=true # Use chunked processing for all jobs (default: true)
 *   - USE_SEPARATE_TRANSACTIONS=false # Use separate transactions for all entity types (default: false)
 */

const debug = require('debug')('agm:reconcile-jobs'),
  env = require('../helpers/env.js'),
  isProd = env.PRODUCTION,
  { DBConnection } = require('../helpers/db/connect.js'),
  mongoose = require('mongoose'),
  mongoUtil = require('../helpers/mongo.js'),
  mongoEnhanced = require('../helpers/mongo_enhanced.js'),
  utils = require('../helpers/utils.js'),
  Job = require('../model/job.js'),
  JobLog = require('../model/job_log.js'),
  JobAssign = require('../model/job_assign.js'),
  Application = require('../model/application.js'),
  AppFile = require('../model/application_file.js'),
  AppDetail = require('../model/application_detail.js'),
  Invoice = require('../model/invoice.js');

// New starting ID
const NEW_START_ID = isProd ? 91105 + 1 : 30;

// Specific job IDs to migrate
const JOB_IDS_TO_MIGRATE = isProd ?
  [
    87628, 87629, 87631, 87637, 87640, 87645, 87646,
    87630, 87632, 87633, 87634, 87635, 87638, 87639, 87641, 87642, 87643, 87644, 87647
  ]
  : [20];

// Set to true to only log what would happen without making changes
const DRY_RUN = false;

// Use enhanced transaction options from mongoEnhanced for consistency
const DEFAULT_TRANSACTION_OPTIONS = {
  ...mongoEnhanced.DEFAULT_TRANSACTION_OPTIONS,
  maxCommitTimeMS: 60000, // Increased to 60 seconds for better reliability
};

// Constants for retry mechanism
const MAX_RETRIES = parseInt(process.env.MAX_RETRIES || '7', 10); // Increased from 5 to 7
const RETRY_DELAY_MS = parseInt(process.env.RETRY_DELAY || '1000', 10);
const USE_CHUNKED_PROCESSING = process.env.USE_CHUNKED_PROCESSING !== 'false';
const USE_SEPARATE_TRANSACTIONS = process.env.USE_SEPARATE_TRANSACTIONS === 'true';
const COPY_APP_DETAILS = process.env.COPY_APP_DETAILS !== 'false'; // Default true, set to false to skip

// Chunk sizes for processing - smaller chunks reduce transaction timeout risk
const CHUNK_SIZE = parseInt(process.env.CHUNK_SIZE || '5', 10); // Reduced from default 10 to 5

// Use enhanced retry logic from mongoEnhanced
const withTransactionRetry = mongoEnhanced.withTransactionRetry;

// Enhanced transaction wrapper using mongoEnhanced for better reliability
async function withTransaction(operation, sessionOptions = DEFAULT_TRANSACTION_OPTIONS) {
  return mongoEnhanced.enhancedRunInTransaction(operation, sessionOptions);
}

// Use enhanced safe operation from mongoEnhanced
const safeMongoOperation = mongoEnhanced.safeMongoOperation;

/**
 * Set up global process error handling
 */
process
  .on('uncaughtException', function (err) {
    debug(err);
    process.exit(1);
  })
  .on('unhandledRejection', (reason, p) => {
    debug(reason, 'Unhandled Rejection at Promise', p);
  });

/**
 * Shift specific job IDs to new IDs starting from NEW_START_ID
 * Uses MongoDB transactions to ensure atomicity
 * Updates all references in related collections
 * Processes one job at a time to ensure data consistency
 * Creates new jobs with new IDs and deletes the old ones
 * 
 * @returns {Promise<void>}
 */
async function shiftSpecificJobIds() {
  // First get the jobs outside of transaction to determine what we need to migrate
  const jobs = await Job.find({ _id: { $in: JOB_IDS_TO_MIGRATE } }).sort({ _id: 1 });
  debug(`Found ${jobs.length} specified jobs to migrate`);

  if (!jobs.length) {
    debug('No jobs to migrate');
    return;
  }

  // Create an ID mapping (oldId -> newId)
  const idMapping = {};
  jobs.forEach((job, index) => {
    idMapping[job._id] = NEW_START_ID + index;
  });

  debug('ID mapping created:');
  debug(Object.entries(idMapping).map(([oldId, newId]) => ({ oldId, newId })));

  if (DRY_RUN) {
    debug('DRY RUN MODE - No changes will be made');
    return;
  }

  // Process one job at a time for better data consistency
  for (const job of jobs) {
    const oldJobId = job._id;
    const newJobId = idMapping[oldJobId];

    debug(`Processing job ${oldJobId} -> ${newJobId}`);

    // Perform the migration using enhanced transaction wrapper
    await withTransaction(async (session) => {
      // 1. Update references in related collections using safe operations
      // 1.1 Update JobLog references
      const jobLogResult = await safeMongoOperation(
        async (s) => JobLog.updateMany(
          { job: oldJobId },
          { $set: { job: newJobId } },
          { session: s }
        ),
        session
      );
      debug(`Updated ${jobLogResult.modifiedCount} job logs for job ${oldJobId}`);

      // 1.2 Update JobAssign references
      const jobAssignResult = await safeMongoOperation(
        async (s) => JobAssign.updateMany(
          { job: oldJobId },
          { $set: { job: newJobId } },
          { session: s }
        ),
        session
      );
      debug(`Updated ${jobAssignResult.modifiedCount} job assignments for job ${oldJobId}`);

      // 1.3 Update Application references 
      const appResult = await safeMongoOperation(
        async (s) => Application.updateMany(
          { jobId: oldJobId },
          { $set: { jobId: newJobId } },
          { session: s }
        ),
        session
      );
      debug(`Updated ${appResult.modifiedCount} applications for job ${oldJobId}`);

      // 1.4 Get all affected application IDs to process app files and app details
      const affectedApplications = await safeMongoOperation(
        async (s) => Application.find(
          { jobId: newJobId },
          { _id: 1 },
          { session: s }
        ),
        session
      );

      const applicationIds = affectedApplications.map(app => app._id);

      if (applicationIds.length) {
        debug(`Found ${applicationIds.length} applications affected by job ID change`);
      }

      // 1.5 Update Invoice references that contain this job
      const invoicesToUpdate = await safeMongoOperation(
        async (s) => Invoice.find({
          'jobs.job': oldJobId
        }, { _id: 1 }, { session: s }),
        session
      );

      if (invoicesToUpdate.length) {
        debug(`Found ${invoicesToUpdate.length} invoices referencing job ${oldJobId}`);

        // For each invoice, update the job references in the jobs array
        for (const invoice of invoicesToUpdate) {
          // We need to fetch, modify, and save each invoice
          const fullInvoice = await safeMongoOperation(
            async (s) => Invoice.findById(invoice._id, {}, { session: s }),
            session
          );

          let updated = false;
          if (fullInvoice.jobs && fullInvoice.jobs.length) {
            fullInvoice.jobs.forEach(jobItem => {
              if (jobItem.job && jobItem.job == oldJobId) {
                jobItem.job = newJobId;
                updated = true;
              }
            });

            if (updated) {
              await safeMongoOperation(
                async (s) => fullInvoice.save({ session: s }),
                session
              );
              debug(`Updated invoice ${fullInvoice._id} with new job ID`);
            }
          }
        }
      }

      // 2. Update the job document with the new ID
      // Note: MongoDB doesn't allow direct updates to the _id field
      // We need to use a two-step process: 1) Create new document with new ID, 2) Delete old document

      // Get the current state of the job (in case it was modified since we queried it)
      const currentJob = await safeMongoOperation(
        async (s) => Job.findById(oldJobId, {}, { session: s }),
        session
      );

      if (!currentJob) {
        debug(`Warning: Could not find job ${oldJobId} to update, it may have been deleted or modified`);
      } else {
        // Create a new document based on the current state but with the new ID
        const jobData = currentJob.toObject();
        delete jobData._id; // Remove the old ID

        // Insert a new document with the new ID using upsert to avoid duplicate key errors
        await safeMongoOperation(
          async (s) => Job.collection.replaceOne({ _id: newJobId }, { ...jobData, _id: newJobId }, { session: s, upsert: true }),
          session
        );
        debug(`Created new job with ID ${newJobId}`);

        // Delete the old document with the old ID
        const deleteResult = await safeMongoOperation(
          async (s) => Job.deleteOne({ _id: oldJobId }, { session: s }),
          session
        );

        if (deleteResult.deletedCount === 1) {
          debug(`Successfully migrated job ID from ${oldJobId} to ${newJobId}`);
        } else {
          debug(`Warning: Failed to delete old job with ID ${oldJobId}`);
          throw new Error(`Failed to migrate job with ID ${oldJobId}`);
        }
      }

      debug(`Transaction for job ${oldJobId} -> ${newJobId} completed successfully`);
    });

    debug(`Successfully migrated job ${oldJobId} -> ${newJobId}`);
  }

  debug('All jobs have been successfully migrated');
}

/**
 * Copy jobs from a previous database to the current database
 * Connects to both databases and copies all related data
 * 
 * @param {Object} options - Copy options
 * @param {string} options.sourceDbUri - Source database URI
 * @param {Array<number>} options.jobIds - Job IDs to copy
 * @param {boolean} options.useNewIds - Whether to assign new IDs to copied jobs
 * @param {boolean} options.keepRelatedIds - Whether to keep original IDs for related entities
 * @param {boolean} options.linkToExisting - Whether to link to existing records
 * @param {number} options.newStartId - Starting ID for new jobs
 * @param {boolean} options.dryRun - Whether to only log without making changes
 * @returns {Promise<Object>} Statistics about the copy operation
 */
async function copyJobsFromPreviousDb({
  sourceDbUri,
  jobIds,
  useNewIds = false,
  keepRelatedIds = false,
  linkToExisting = true,
  newStartId = NEW_START_ID,
  dryRun = false
}) {
  debug(`Copying jobs from previous database: ${jobIds.join(', ')}`);
  debug(`Options: useNewIds=${useNewIds}, keepRelatedIds=${keepRelatedIds}, linkToExisting=${linkToExisting}`);

  if (dryRun) {
    debug('DRY RUN MODE - No changes will be made');
  }

  // Initialize statistics
  const stats = {
    jobs: 0,
    jobLogs: 0,
    jobAssigns: 0,
    applications: 0,
    appFiles: 0,
    appDetails: 0,
    linkedLogs: 0,
    linkedAssigns: 0,
    linkedApplications: 0,
    linkedFiles: 0,
    linkedDetails: 0,
    failureReasons: {}
  };

  // Results to return
  const result = {
    processed: 0,
    succeeded: 0,
    skipped: 0,
    failed: 0,
    skippedIds: [],
    failedIds: [],
    stats,
    linked: {
      logs: 0,
      assigns: 0,
      applications: 0,
      files: 0,
      details: 0
    }
  };

  // Connect to source database
  let sourceConnection;
  try {
    sourceConnection = await mongoose.createConnection(sourceDbUri, {
      useNewUrlParser: true,
      useUnifiedTopology: true
    });
    // Wait for connection to be open before using models
    await new Promise((resolve, reject) => {
      sourceConnection.once('open', resolve);
      sourceConnection.once('error', reject);
    });
    debug('Connected to source database');
  } catch (error) {
    debug(`Failed to connect to source database: ${error.message}`);
    throw error;
  }

  // Create models for the source database
  const SourceJob = sourceConnection.model('Job', Job.schema);
  const SourceJobLog = sourceConnection.model('Job_Log', JobLog.schema);
  const SourceJobAssign = sourceConnection.model('Job_Assign', JobAssign.schema);
  const SourceApplication = sourceConnection.model('Application', Application.schema);
  const SourceAppFile = sourceConnection.model('AppFile', AppFile.schema);
  const SourceAppDetail = sourceConnection.model('Application_Detail', AppDetail.schema);

  try {
    // Process each job
    for (const jobId of jobIds) {
      debug(`Processing job ID ${jobId}...`);
      result.processed++;

      // Skip if job already exists in destination database
      const existingJob = await Job.findById(jobId);
      if (existingJob) {
        debug(`Job ${jobId} already exists in destination database,...`);
        // debug(`Job ${jobId} already exists in destination database, skipping`);
        // result.skipped++;
        // result.skippedIds.push(jobId);
        // continue;
      }

      try {
        // Fetch job from source database
        const sourceJob = await SourceJob.findById(jobId);
        if (!sourceJob) {
          debug(`Job ${jobId} not found in source database`);
          result.failed++;
          result.failedIds.push(jobId);
          stats.failureReasons[jobId] = {
            category: 'NotFound',
            message: 'Job not found in source database'
          };
          continue;
        }

        // Determine new job ID
        const newJobId = useNewIds ? newStartId + result.succeeded : jobId;

        // Fetch related data
        const jobLogs = await SourceJobLog.find({ job: jobId });
        const jobAssigns = await SourceJobAssign.find({ job: jobId });
        const applications = await SourceApplication.find({ jobId });

        debug(`Found ${jobLogs.length} logs, ${jobAssigns.length} assignments, ${applications.length} applications`);

        // Create mappings for IDs
        const applicationIdMapping = {};
        const appFileIdMapping = {};

        // Create lists for related data
        const appFiles = [];
        const appDetails = [];

        // Fetch application files and details
        for (const app of applications) {
          // Generate new ID for application if needed
          const newAppId = keepRelatedIds ? app._id : new mongoose.Types.ObjectId();
          applicationIdMapping[app._id] = newAppId;

          // Get application files
          const files = await SourceAppFile.find({ appId: app._id });
          for (const file of files) {
            const newFileId = keepRelatedIds ? file._id : new mongoose.Types.ObjectId();
            appFileIdMapping[file._id] = newFileId;
            appFiles.push(file);
          }

          // Get application details (if enabled)
          if (COPY_APP_DETAILS) {
            // Get application details by both appId and fileId relationships
            const fileIds = await SourceAppFile.find({ appId: app._id }).distinct('_id');
            const detailsByAppId = await SourceAppDetail.find({ appId: app._id });
            const detailsByFileId = fileIds.length > 0 ? await SourceAppDetail.find({ fileId: { $in: fileIds } }) : [];

            // Combine and deduplicate application details
            const allDetails = [...detailsByAppId, ...detailsByFileId];
            const uniqueDetails = allDetails.filter((detail, index, self) =>
              index === self.findIndex(d => d._id.toString() === detail._id.toString())
            );
            appDetails.push(...uniqueDetails);
          }
        }

        debug(`Found ${appFiles.length} app files${COPY_APP_DETAILS ? `, ${appDetails.length} app details` : ' (app details skipped)'}`);

        if (dryRun) {
          debug(`DRY RUN: Would copy job ${jobId} with ${jobLogs.length} logs, ${jobAssigns.length} assignments, ${applications.length} applications, ${appFiles.length} files${COPY_APP_DETAILS ? `, ${appDetails.length} details` : ' (app details skipped)'}`);
          result.succeeded++;
          continue;
        }

        // Use different processing approaches based on configuration
        let success = false;

        if (USE_SEPARATE_TRANSACTIONS) {
          // Use completely separate transactions
          debug(`Using separate transactions for job ${jobId}`);
          success = await processJobWithSeparateTransactions(
            sourceJob, jobId, newJobId, jobLogs, jobAssigns, applications,
            appFiles, appDetails, applicationIdMapping, appFileIdMapping, keepRelatedIds, stats
          );
        } else if (USE_CHUNKED_PROCESSING) {
          // Use chunked processing for regular jobs
          debug(`Using chunked processing for job ${jobId}`);
          success = await withTransactionRetry(async () => {
            const session = await mongoose.startSession(DEFAULT_TRANSACTION_OPTIONS);
            try {
              session.startTransaction();

              // 1. Create the job using upsert to avoid duplicate key errors
              const jobData = sourceJob.toObject();
              const newJob = { ...jobData, _id: newJobId };
              const jobUpsertResult = await Job.collection.replaceOne(
                { _id: newJobId },
                newJob,
                { session, upsert: true }
              );

              if (jobUpsertResult.upsertedCount > 0) {
                debug(`Created job with ID ${newJobId} in destination database via upsert`);
                stats.jobs++;
              } else if (jobUpsertResult.matchedCount > 0) {
                debug(`Updated existing job with ID ${newJobId} in destination database`);
                stats.jobs++;
              } else {
                debug(`Warning: Job upsert for ${newJobId} did not create or match any documents`);
              }

              // 2. Process all related data in chunks
              await processRegularJobInChunks(
                sourceJob, jobId, newJobId, jobLogs, jobAssigns, applications,
                appFiles, appDetails, applicationIdMapping, appFileIdMapping, keepRelatedIds, session, linkToExisting, stats
              );

              await session.commitTransaction();
              return true;
            } catch (error) {
              if (session.inTransaction()) {
                try {
                  await session.abortTransaction();
                } catch (abortError) {
                  debug(`Error aborting transaction: ${abortError.message}`);
                }
              }
              throw error;
            } finally {
              if (typeof session.endSession === 'function') {
                try {
                  session.endSession();
                } catch (endError) {
                  debug(`Error ending session: ${endError.message}`);
                }
              }
            }
          });
        } else {
          // Use a single transaction for the entire job
          debug(`Using single transaction for job ${jobId}`);
          success = await withTransactionRetry(async () => {
            const session = await mongoose.startSession(DEFAULT_TRANSACTION_OPTIONS);
            try {
              session.startTransaction();

              // 1. Create the job using upsert to avoid duplicate key errors
              const jobData = sourceJob.toObject();
              const newJob = { ...jobData, _id: newJobId };
              const jobUpsertResult = await Job.collection.replaceOne(
                { _id: newJobId },
                newJob,
                { session, upsert: true }
              );

              if (jobUpsertResult.upsertedCount > 0) {
                debug(`Created job with ID ${newJobId} in destination database via upsert`);
                stats.jobs++;
              } else if (jobUpsertResult.matchedCount > 0) {
                debug(`Updated existing job with ID ${newJobId} in destination database`);
                stats.jobs++;
              } else {
                debug(`Warning: Job upsert for ${newJobId} did not create or match any documents`);
              }

              // 2. Create job logs
              for (const sourceJobLog of jobLogs) {
                const logData = sourceJobLog.toObject();
                const logId = keepRelatedIds ? sourceJobLog._id : new mongoose.Types.ObjectId();
                const newJobLog = { ...logData, job: newJobId, _id: logId };
                await safeInsertWithRecovery(JobLog.collection, newJobLog, { job: newJobId }, session, 'linkedLogs', false, linkToExisting, stats);
              }
              stats.jobLogs += jobLogs.length;

              // 3. Create job assignments
              for (const sourceJobAssign of jobAssigns) {
                const assignData = sourceJobAssign.toObject();
                const assignId = keepRelatedIds ? sourceJobAssign._id : new mongoose.Types.ObjectId();
                const newJobAssign = { ...assignData, job: newJobId, _id: assignId };
                await safeInsertWithRecovery(JobAssign.collection, newJobAssign, { job: newJobId }, session, 'linkedAssigns', false, linkToExisting, stats);
              }
              stats.jobAssigns += jobAssigns.length;

              // 4. Create applications and related data
              for (const sourceApp of applications) {
                const appData = sourceApp.toObject();
                const newAppId = applicationIdMapping[sourceApp._id];
                const newApp = { ...appData, jobId: newJobId, _id: newAppId };
                await safeInsertWithRecovery(Application.collection, newApp, { jobId: newJobId }, session, 'linkedApplications', false, linkToExisting, stats);
                stats.applications++;

                // 5. Create application files
                const appFilesForThisApp = appFiles.filter(file => file.appId.toString() === sourceApp._id.toString());
                for (const sourceAppFile of appFilesForThisApp) {
                  const fileData = sourceAppFile.toObject();
                  const newFileId = appFileIdMapping[sourceAppFile._id];
                  const newAppFile = {
                    ...fileData,
                    appId: newAppId,
                    _id: newFileId
                  };
                  await safeInsertWithRecovery(AppFile.collection, newAppFile, { appId: newAppId }, session, 'linkedFiles', false, linkToExisting, stats);
                  stats.appFiles++;
                }

                // 6. Create application details (if enabled)
                if (COPY_APP_DETAILS) {
                  // Filter application details by both appId and fileId relationships
                  const sourceAppFileIds = appFiles
                    .filter(file => file.appId.toString() === sourceApp._id.toString())
                    .map(file => file._id.toString());

                  const appDetailsForThisApp = appDetails.filter(detail =>
                    detail.appId?.toString() === sourceApp._id.toString() ||
                    (detail.fileId && sourceAppFileIds.includes(detail.fileId.toString()))
                  );

                  for (const sourceAppDetail of appDetailsForThisApp) {
                    const detailData = sourceAppDetail.toObject();
                    const detailId = keepRelatedIds ? sourceAppDetail._id : new mongoose.Types.ObjectId();

                    // Determine the fileId to use
                    let newFileId = null;
                    if (sourceAppDetail.fileId && appFileIdMapping[sourceAppDetail.fileId]) {
                      newFileId = appFileIdMapping[sourceAppDetail.fileId];
                    }

                    const newAppDetail = {
                      ...detailData,
                      fileId: newFileId, // Primary relationship using fileId
                      appId: sourceAppDetail.appId ? newAppId : undefined, // Preserve appId if it exists
                      _id: detailId
                    };

                    // Remove undefined fields
                    if (newAppDetail.appId === undefined) delete newAppDetail.appId;

                    await safeInsertWithRecovery(AppDetail.collection, newAppDetail, { fileId: newFileId }, session, 'linkedDetails', false, linkToExisting, stats);
                    stats.appDetails++;
                  }
                }
              }

              await session.commitTransaction();
              return true;
            } catch (error) {
              if (session.inTransaction()) {
                try {
                  await session.abortTransaction();
                } catch (abortError) {
                  debug(`Error aborting transaction: ${abortError.message}`);
                }
              }
              throw error;
            } finally {
              if (typeof session.endSession === 'function') {
                try {
                  session.endSession();
                } catch (endError) {
                  debug(`Error ending session: ${endError.message}`);
                }
              }
            }
          });
        }

        if (success) {
          debug(`Successfully copied job ${jobId} to ${newJobId}`);
          result.succeeded++;
        } else {
          debug(`Failed to copy job ${jobId}`);
          result.failed++;
          result.failedIds.push(jobId);
          stats.failureReasons[jobId] = {
            category: 'ProcessingError',
            message: 'Failed to process job'
          };
        }
      } catch (error) {
        debug(`Error processing job ${jobId}: ${error.message}`);
        result.failed++;
        result.failedIds.push(jobId);
        stats.failureReasons[jobId] = {
          category: error.name || 'UnknownError',
          message: error.message
        };
      }
    }

    // Update linked counts in result
    result.linked = {
      logs: stats.linkedLogs,
      assigns: stats.linkedAssigns,
      applications: stats.linkedApplications,
      files: stats.linkedFiles,
      details: stats.linkedDetails
    };

    return result;
  } finally {
    // Close source database connection
    if (sourceConnection) {
      await sourceConnection.close();
      debug('Closed connection to source database');
    }
  }
}

/**
 * Process a regular job in chunks to reduce transaction size and duration
 * 
 * @param {Object} sourceJob - The source job
 * @param {Number} sourceJobId - The original job ID from source database
 * @param {Number} newJobId - The new job ID in destination database
 * @param {Array} jobLogs - Job logs to copy
 * @param {Array} jobAssigns - Job assignments to copy
 * @param {Array} applications - Applications to copy
 * @param {Array} appFiles - Application files to copy
 * @param {Array} appDetails - Application details to copy
 * @param {Object} applicationIdMapping - Mapping from old to new application IDs
 * @param {Object} appFileIdMapping - Mapping from old to new application file IDs
 * @param {Boolean} keepRelatedIds - Whether to keep original IDs for related entities
 * @param {Object} session - MongoDB session
 * @param {Boolean} linkToExisting - Whether to link to existing records
 * @param {Object} stats - Statistics object to update
 * @returns {Promise<void>}
 */
async function processRegularJobInChunks(
  sourceJob,
  sourceJobId,
  newJobId,
  jobLogs,
  jobAssigns,
  applications,
  appFiles,
  appDetails,
  applicationIdMapping,
  appFileIdMapping,
  keepRelatedIds,
  session,
  linkToExisting,
  stats
) {
  // Constants for batch sizes - using configurable chunk sizes
  const LOG_BATCH_SIZE = parseInt(process.env.LOG_BATCH_SIZE || '30', 10); // Reduced from 50 to 30
  const ASSIGN_BATCH_SIZE = parseInt(process.env.ASSIGN_BATCH_SIZE || '15', 10); // Reduced from 20 to 15  
  const APP_BATCH_SIZE = CHUNK_SIZE; // Use the main chunk size (default 5)

  // 1. Process job logs in batches
  if (jobLogs.length > 0) {
    debug(`Processing ${jobLogs.length} job logs in batches of ${LOG_BATCH_SIZE}`);
    for (let i = 0; i < jobLogs.length; i += LOG_BATCH_SIZE) {
      const batch = jobLogs.slice(i, i + LOG_BATCH_SIZE);
      debug(`Processing batch of ${batch.length} job logs (${i + 1}-${Math.min(i + LOG_BATCH_SIZE, jobLogs.length)} of ${jobLogs.length})`);

      const logCreationPromises = batch.map(async (sourceJobLog) => {
        const logData = sourceJobLog.toObject();
        const logId = keepRelatedIds ? sourceJobLog._id : new mongoose.Types.ObjectId();
        const newJobLog = { ...logData, job: newJobId, _id: logId };
        return safeInsertWithRecovery(JobLog.collection, newJobLog, { job: newJobId }, session, 'linkedLogs', false, linkToExisting, stats);
      });

      await Promise.all(logCreationPromises);
      stats.jobLogs += batch.length;

      // Add a small delay between batches to reduce contention
      if (i + LOG_BATCH_SIZE < jobLogs.length) {
        await new Promise(resolve => setTimeout(resolve, 50));
      }
    }
  }

  // 2. Process job assignments in batches
  if (jobAssigns.length > 0) {
    debug(`Processing ${jobAssigns.length} job assignments in batches of ${ASSIGN_BATCH_SIZE}`);
    for (let i = 0; i < jobAssigns.length; i += ASSIGN_BATCH_SIZE) {
      const batch = jobAssigns.slice(i, i + ASSIGN_BATCH_SIZE);
      debug(`Processing batch of ${batch.length} job assignments (${i + 1}-${Math.min(i + ASSIGN_BATCH_SIZE, jobAssigns.length)} of ${jobAssigns.length})`);

      const assignCreationPromises = batch.map(async (sourceJobAssign) => {
        const assignData = sourceJobAssign.toObject();
        const assignId = keepRelatedIds ? sourceJobAssign._id : new mongoose.Types.ObjectId();
        const newJobAssign = { ...assignData, job: newJobId, _id: assignId };
        return safeInsertWithRecovery(JobAssign.collection, newJobAssign, { job: newJobId }, session, 'linkedAssigns', false, linkToExisting, stats);
      });

      await Promise.all(assignCreationPromises);
      stats.jobAssigns += batch.length;

      // Add a small delay between batches to reduce contention
      if (i + ASSIGN_BATCH_SIZE < jobAssigns.length) {
        await new Promise(resolve => setTimeout(resolve, 50));
      }
    }
  }

  // 3. Process applications and related data in batches
  if (applications.length > 0) {
    debug(`Processing ${applications.length} applications in batches of ${APP_BATCH_SIZE}`);

    for (let i = 0; i < applications.length; i += APP_BATCH_SIZE) {
      const batch = applications.slice(i, i + APP_BATCH_SIZE);
      debug(`Processing batch of ${batch.length} applications (${i + 1}-${Math.min(i + APP_BATCH_SIZE, applications.length)} of ${applications.length})`);

      for (const sourceApp of batch) {
        // Create application
        const appData = sourceApp.toObject();
        const newAppId = applicationIdMapping[sourceApp._id];
        const newApp = { ...appData, jobId: newJobId, _id: newAppId };
        await safeInsertWithRecovery(Application.collection, newApp, { jobId: newJobId }, session, 'linkedApplications', false, linkToExisting, stats);
        stats.applications++;

        // Find related app files and details
        const relatedAppFiles = appFiles.filter(file => file.appId.toString() === sourceApp._id.toString());

        // Filter application details by both appId and fileId relationships
        let relatedAppDetails = [];
        if (COPY_APP_DETAILS) {
          const sourceAppFileIds = relatedAppFiles.map(file => file._id.toString());
          relatedAppDetails = appDetails.filter(detail =>
            detail.appId?.toString() === sourceApp._id.toString() ||
            (detail.fileId && sourceAppFileIds.includes(detail.fileId.toString()))
          );
        }

        // Process app files sequentially to avoid transaction conflicts
        if (relatedAppFiles.length > 0) {
          debug(`Processing ${relatedAppFiles.length} app files sequentially for application ${newAppId}`);
          for (let fileIndex = 0; fileIndex < relatedAppFiles.length; fileIndex++) {
            const sourceAppFile = relatedAppFiles[fileIndex];
            const fileData = sourceAppFile.toObject();
            const newFileId = appFileIdMapping[sourceAppFile._id];
            const newAppFile = {
              ...fileData,
              appId: newAppId,
              _id: newFileId
            };

            debug(`Processing app file ${fileIndex + 1}/${relatedAppFiles.length} (ID: ${newFileId})`);
            const updateFields = { appId: newAppId };
            await safeInsertWithRecovery(
              AppFile.collection,
              newAppFile,
              updateFields,
              session,
              'linkedFiles',
              false, // Standard retry logic for app files
              linkToExisting,
              stats
            );
            stats.appFiles++;

            // Add a small delay between app file operations to reduce contention
            if (fileIndex < relatedAppFiles.length - 1) {
              await new Promise(resolve => setTimeout(resolve, 25 + Math.random() * 25));
            }
          }
          debug(`Completed processing ${relatedAppFiles.length} app files for application ${newAppId}`);
        }

        // Process app details using bulk insert (if enabled)
        if (COPY_APP_DETAILS && relatedAppDetails.length > 0) {
          debug(`Processing ${relatedAppDetails.length} app details using bulk insert for application ${newAppId}`);

          // Prepare all app details for bulk insert
          const appDetailsToInsert = relatedAppDetails.map((sourceAppDetail) => {
            const detailData = sourceAppDetail.toObject();
            const detailId = keepRelatedIds ? sourceAppDetail._id : new mongoose.Types.ObjectId();

            // Determine the fileId to use
            let newFileId = null;
            if (sourceAppDetail.fileId && appFileIdMapping[sourceAppDetail.fileId]) {
              newFileId = appFileIdMapping[sourceAppDetail.fileId];
            }

            const newAppDetail = {
              ...detailData,
              fileId: newFileId, // Primary relationship using fileId
              appId: sourceAppDetail.appId ? newAppId : undefined, // Preserve appId if it exists
              _id: detailId
            };

            // Remove undefined fields
            if (newAppDetail.appId === undefined) delete newAppDetail.appId;

            return newAppDetail;
          });

          // Process in chunks of 1000 documents
          const chunks = utils.chunkArray(appDetailsToInsert, 1000);
          debug(`Processing ${appDetailsToInsert.length} app details in ${chunks.length} chunks of max 1000 documents each`);

          // Process chunks sequentially using native async/await
          for (let chunkIndex = 0; chunkIndex < chunks.length; chunkIndex++) {
            const chunk = chunks[chunkIndex];
            try {
              await AppDetail.insertMany(chunk, { rawResult: true, ordered: true, session });
              debug(`Successfully inserted chunk ${chunkIndex + 1}/${chunks.length} with ${chunk.length} app details`);
            } catch (err) {
              debug(`Error in bulk insert for app details chunk ${chunkIndex + 1}/${chunks.length}: ${err.message}`);
              throw err;
            }
          }

          stats.appDetails += relatedAppDetails.length;
          debug(`Completed bulk insert of ${relatedAppDetails.length} app details for application ${newAppId}`);
        }
      }

      // Add a small delay between application batches to reduce contention
      if (i + APP_BATCH_SIZE < applications.length) {
        await new Promise(resolve => setTimeout(resolve, 50 + Math.random() * 100));
      }
    }
  }

  debug(`All data for job ${newJobId} processed in smaller chunks`);
}

/**
 * Safe insert with recovery for various errors using upsert operations
 * Uses replaceOne with upsert to handle both new documents and duplicates automatically
 * without causing transaction aborts
 * 
 * @param {Object} collection - MongoDB collection to insert into
 * @param {Object} document - Document to insert
 * @param {Object} updateFields - Fields to update for finding existing documents (legacy parameter, not used with upsert approach)
 * @param {Object} session - MongoDB session
 * @param {string} statCounter - Name of the linked counter in stats
 * @param {boolean} linkToExisting - Whether to link to existing records
 * @param {Object} stats - Statistics object to update
 * @returns {Promise<boolean>} - Whether the operation succeeded
 */
async function safeInsertWithRecovery(
  collection,
  document,
  updateFields,
  session,
  statCounter = null,
  linkToExisting = true,
  stats = null
) {
  const maxRetries = MAX_RETRIES;
  let retryCount = 0;
  const collectionName = collection.collectionName;

  while (retryCount <= maxRetries) {
    try {
      // Use replaceOne with upsert directly to handle both new inserts and duplicates
      const filter = { _id: document._id };
      const updateDoc = { ...document };

      const upsertResult = await collection.replaceOne(
        filter,
        updateDoc,
        {
          session,
          upsert: true
        }
      );

      if (upsertResult.upsertedCount > 0) {
        debug(`Created new ${collectionName} document via upsert`);
        return true;
      } else if (upsertResult.matchedCount > 0) {
        debug(`Linked to existing ${collectionName} document via upsert (matched: ${upsertResult.matchedCount}, modified: ${upsertResult.modifiedCount})`);

        // Increment the linked counter since we found an existing document
        if (statCounter && stats && linkToExisting) {
          stats[statCounter]++;
        }
        return true;
      } else {
        debug(`Upsert for ${collectionName} did not create or match any documents, retrying`);
        // Fall through to retry logic
      }
    } catch (error) {
      retryCount++;

      // Check if we should retry based on error type
      const shouldRetry = retryCount <= maxRetries && (
        (error.errorLabels && error.errorLabels.includes("TransientTransactionError")) ||
        (error.message && (
          error.message.includes("has been aborted") ||
          error.message.includes("transaction aborted") ||
          error.message.includes("Transaction has been aborted") ||
          error.message.includes("Cannot use a session that has ended")
        ))
      );

      if (shouldRetry) {
        const baseDelay = RETRY_DELAY_MS * Math.pow(2, retryCount - 1);
        const jitter = Math.random() * 0.5 * baseDelay; // Up to 50% jitter
        const delay = Math.floor(baseDelay + jitter);

        debug(`Retrying ${collectionName} operation (attempt ${retryCount}/${maxRetries}) after ${delay}ms delay due to error: ${error.message}`);

        await new Promise(resolve => setTimeout(resolve, delay));
      } else {
        // If we've exceeded retries, throw the error
        throw error;
      }
    }
  }

  throw new Error(`Failed to insert ${collectionName} document after ${maxRetries} retries`);
}

/**
 * Process a job using separate transactions for each entity type.
 * This is the most conservative approach for jobs that require maximum transaction isolation.
 * 
 * @param {Object} sourceJob - The source job to copy
 * @param {Number} oldJobId - The original job ID
 * @param {Number} newJobId - The new job ID to use
 * @param {Array} jobLogs - Job logs to copy
 * @param {Array} jobAssigns - Job assignments to copy
 * @param {Array} applications - Applications to copy
 * @param {Array} appFiles - Application files to copy
 * @param {Array} appDetails - Application details to copy
 * @param {Object} applicationIdMapping - Mapping from old to new application IDs
 * @param {Object} appFileIdMapping - Mapping from old to new application file IDs
 * @param {Boolean} keepRelatedIds - Whether to keep original IDs for related entities
 * @param {Object} stats - Statistics object to update
 * @returns {Promise<Boolean>} - Whether the job was successfully processed
 */
async function processJobWithSeparateTransactions(
  sourceJob,
  oldJobId,
  newJobId,
  jobLogs,
  jobAssigns,
  applications,
  appFiles,
  appDetails,
  applicationIdMapping,
  appFileIdMapping,
  keepRelatedIds,
  stats
) {
  debug(`Processing job ${oldJobId} -> ${newJobId} using completely separate transactions`);

  try {
    // Transaction 1: Create the job
    await withTransactionRetry(async () => {
      const session = await mongoose.startSession(DEFAULT_TRANSACTION_OPTIONS);

      try {
        session.startTransaction();
        const jobData = sourceJob.toObject();
        const newJob = { ...jobData, _id: newJobId };
        const jobUpsertResult = await Job.collection.replaceOne(
          { _id: newJobId },
          newJob,
          { session, upsert: true }
        );

        if (jobUpsertResult.upsertedCount > 0) {
          debug(`Created job with ID ${newJobId} in destination database via upsert (separate transaction)`);
          stats.jobs++;
        } else if (jobUpsertResult.matchedCount > 0) {
          debug(`Updated existing job with ID ${newJobId} in destination database (separate transaction)`);
          stats.jobs++;
        } else {
          debug(`Warning: Job upsert for ${newJobId} did not create or match any documents (separate transaction)`);
        }

        await session.commitTransaction();
      } catch (error) {
        if (session.inTransaction()) {
          try {
            await session.abortTransaction();
          } catch (abortError) {
            debug(`Error aborting transaction: ${abortError.message}`);
          }
        }
        throw error;
      } finally {
        if (typeof session.endSession === 'function') {
          try {
            session.endSession();
          } catch (endError) {
            debug(`Error ending session: ${endError.message}`);
          }
        }
      }
    });

    // Add delay between transactions
    await new Promise(resolve => setTimeout(resolve, 500 + Math.random() * 300));

    // Transaction 2: Process job logs in small batches
    if (jobLogs.length > 0) {
      const BATCH_SIZE = parseInt(process.env.LOG_BATCH_SIZE || '15', 10); // Reduced from 20 to 15
      for (let i = 0; i < jobLogs.length; i += BATCH_SIZE) {
        const batch = jobLogs.slice(i, i + BATCH_SIZE);
        debug(`Processing batch of ${batch.length} job logs (${i + 1}-${Math.min(i + BATCH_SIZE, jobLogs.length)} of ${jobLogs.length})`);

        await withTransactionRetry(async () => {
          const session = await mongoose.startSession(DEFAULT_TRANSACTION_OPTIONS);

          try {
            session.startTransaction();

            const promises = batch.map(async (sourceJobLog) => {
              const logData = sourceJobLog.toObject();
              const logId = keepRelatedIds ? sourceJobLog._id : new mongoose.Types.ObjectId();
              const newJobLog = { ...logData, job: newJobId, _id: logId };
              return safeInsertWithRecovery(JobLog.collection, newJobLog, { job: newJobId }, session, 'linkedLogs', true, true, stats);
            });

            await Promise.all(promises);
            await session.commitTransaction();
          } catch (error) {
            if (session.inTransaction()) {
              try {
                await session.abortTransaction();
              } catch (abortError) {
                debug(`Error aborting transaction: ${abortError.message}`);
              }
            }
            throw error;
          } finally {
            if (typeof session.endSession === 'function') {
              try {
                session.endSession();
              } catch (endError) {
                debug(`Error ending session: ${endError.message}`);
              }
            }
          }
        });

        // Add delay between batches
        await new Promise(resolve => setTimeout(resolve, 200 + Math.random() * 200));
      }
      stats.jobLogs += jobLogs.length;
      debug(`Completed processing all ${jobLogs.length} job logs`);
    }

    // Transaction 3: Process job assignments
    if (jobAssigns.length > 0) {
      await withTransactionRetry(async () => {
        const session = await mongoose.startSession(DEFAULT_TRANSACTION_OPTIONS);

        try {
          session.startTransaction();

          const promises = jobAssigns.map(async (sourceJobAssign) => {
            const assignData = sourceJobAssign.toObject();
            const assignId = keepRelatedIds ? sourceJobAssign._id : new mongoose.Types.ObjectId();
            const newJobAssign = { ...assignData, job: newJobId, _id: assignId };
            return safeInsertWithRecovery(JobAssign.collection, newJobAssign, { job: newJobId }, session, 'linkedAssigns', true, true, stats);
          });

          await Promise.all(promises);
          stats.jobAssigns += jobAssigns.length;

          await session.commitTransaction();
        } catch (error) {
          if (session.inTransaction()) {
            try {
              await session.abortTransaction();
            } catch (abortError) {
              debug(`Error aborting transaction: ${abortError.message}`);
            }
          }
          throw error;
        } finally {
          if (typeof session.endSession === 'function') {
            try {
              session.endSession();
            } catch (endError) {
              debug(`Error ending session: ${endError.message}`);
            }
          }
        }
      });

      // Add delay between transactions
      await new Promise(resolve => setTimeout(resolve, 300 + Math.random() * 200));
    }

    // Transaction 4: Process applications and related data one by one
    for (let i = 0; i < applications.length; i++) {
      const sourceApp = applications[i];
      const appData = sourceApp.toObject();
      const newAppId = applicationIdMapping[sourceApp._id];

      // Create application in its own transaction
      await withTransactionRetry(async () => {
        const session = await mongoose.startSession(DEFAULT_TRANSACTION_OPTIONS);

        try {
          session.startTransaction();

          const newApp = { ...appData, jobId: newJobId, _id: newAppId };
          await safeInsertWithRecovery(
            Application.collection,
            newApp,
            { jobId: newJobId },
            session,
            'linkedApplications',
            true,
            true,
            stats
          );
          stats.applications++;

          await session.commitTransaction();
        } catch (error) {
          if (session.inTransaction()) {
            try {
              await session.abortTransaction();
            } catch (abortError) {
              debug(`Error aborting transaction: ${abortError.message}`);
            }
          }
          throw error;
        } finally {
          if (typeof session.endSession === 'function') {
            try {
              session.endSession();
            } catch (endError) {
              debug(`Error ending session: ${endError.message}`);
            }
          }
        }
      });

      // Small delay between application transactions
      await new Promise(resolve => setTimeout(resolve, 100));

      // Find related app files
      const relatedAppFiles = appFiles.filter(file => file.appId.toString() === sourceApp._id.toString());

      // Process app files in a separate transaction
      if (relatedAppFiles.length > 0) {
        await withTransactionRetry(async () => {
          const session = await mongoose.startSession(DEFAULT_TRANSACTION_OPTIONS);

          try {
            session.startTransaction();

            const promises = relatedAppFiles.map(async (sourceAppFile) => {
              const fileData = sourceAppFile.toObject();
              const newFileId = appFileIdMapping[sourceAppFile._id];
              const newAppFile = {
                ...fileData,
                appId: newAppId,
                _id: newFileId
              };

              return safeInsertWithRecovery(
                AppFile.collection,
                newAppFile,
                { appId: newAppId },
                session,
                'linkedFiles',
                true,
                true,
                stats
              );
            });

            await Promise.all(promises);
            stats.appFiles += relatedAppFiles.length;

            await session.commitTransaction();
          } catch (error) {
            if (session.inTransaction()) {
              try {
                await session.abortTransaction();
              } catch (abortError) {
                debug(`Error aborting transaction: ${abortError.message}`);
              }
            }
            throw error;
          } finally {
            if (typeof session.endSession === 'function') {
              try {
                session.endSession();
              } catch (endError) {
                debug(`Error ending session: ${endError.message}`);
              }
            }
          }
        });

        // Small delay after file processing
        await new Promise(resolve => setTimeout(resolve, 100));
      }

      // Find related app details and process them (if enabled)
      if (COPY_APP_DETAILS) {
        // Filter application details by both appId and fileId relationships
        const sourceAppFileIds = relatedAppFiles.map(file => file._id.toString());
        const relatedAppDetails = appDetails.filter(detail =>
          detail.appId?.toString() === sourceApp._id.toString() ||
          (detail.fileId && sourceAppFileIds.includes(detail.fileId.toString()))
        );

        // Process app details using bulk insert in a separate transaction
        if (relatedAppDetails.length > 0) {
          await withTransactionRetry(async () => {
            const session = await mongoose.startSession(DEFAULT_TRANSACTION_OPTIONS);

            try {
              session.startTransaction();

              // Prepare all app details for bulk insert
              const appDetailsToInsert = relatedAppDetails.map((sourceAppDetail) => {
                const detailData = sourceAppDetail.toObject();
                const detailId = keepRelatedIds ? sourceAppDetail._id : new mongoose.Types.ObjectId();

                // Determine the fileId to use
                let newFileId = null;
                if (sourceAppDetail.fileId && appFileIdMapping[sourceAppDetail.fileId]) {
                  newFileId = appFileIdMapping[sourceAppDetail.fileId];
                }

                const newAppDetail = {
                  ...detailData,
                  fileId: newFileId, // Primary relationship using fileId
                  appId: sourceAppDetail.appId ? newAppId : undefined, // Preserve appId if it exists
                  _id: detailId
                };

                // Remove undefined fields
                if (newAppDetail.appId === undefined) delete newAppDetail.appId;

                return newAppDetail;
              });

              // Process in chunks of 1000 documents
              const chunks = utils.chunkArray(appDetailsToInsert, 1000);
              debug(`Processing ${appDetailsToInsert.length} app details in ${chunks.length} chunks using bulk insert (separate transaction)`);

              // Process chunks sequentially using native async/await
              for (let chunkIndex = 0; chunkIndex < chunks.length; chunkIndex++) {
                const chunk = chunks[chunkIndex];
                try {
                  await AppDetail.insertMany(chunk, { rawResult: true, ordered: true, session });
                  debug(`Successfully inserted chunk ${chunkIndex + 1}/${chunks.length} with ${chunk.length} app details (separate transaction)`);
                } catch (err) {
                  debug(`Error in bulk insert for app details chunk ${chunkIndex + 1}/${chunks.length} (separate transaction): ${err.message}`);
                  throw err;
                }
              }

              stats.appDetails += relatedAppDetails.length;
              debug(`Completed bulk insert of ${relatedAppDetails.length} app details (separate transaction)`);

              await session.commitTransaction();
            } catch (error) {
              if (session.inTransaction()) {
                try {
                  await session.abortTransaction();
                } catch (abortError) {
                  debug(`Error aborting transaction: ${abortError.message}`);
                }
              }
              throw error;
            } finally {
              if (typeof session.endSession === 'function') {
                try {
                  session.endSession();
                } catch (endError) {
                  debug(`Error ending session: ${endError.message}`);
                }
              }
            }
          });
        }
      }

      debug(`Processed application ${i + 1}/${applications.length}`);
    }

    debug(`Successfully processed job ${oldJobId} -> ${newJobId} using separate transactions`);
    return true;
  } catch (error) {
    debug(`Error processing job ${oldJobId} with separate transactions: ${error.message}`);
    throw error;
  }
}

/**
 * Main execution function
 * Parses command-line arguments and executes the appropriate function
 */
(async function main() {
  const args = process.argv.slice(2);
  const command = args[0]?.toLowerCase();
  const dbConn = new DBConnection('Migrate Job IDs Script');

  try {
    await dbConn.initialize({ setupExitHandlers: false });
    debug('Database connected');

    if (command === 'copy') {
      const sourceDbUri = process.env.SOURCE_DB_URI;
      const jobIdsArg = process.env.JOB_IDS || args[1] || JOB_IDS_TO_MIGRATE.join(',');
      const useNewIds = process.env.USE_NEW_IDS === 'true' || false;
      const keepRelatedIds = process.env.KEEP_RELATED_IDS === 'true' || false;
      const linkToExisting = process.env.LINK_TO_EXISTING !== 'false';

      if (!sourceDbUri) {
        debug('Error: SOURCE_DB_URI environment variable is required for copy operation');
        process.exit(1);
      }

      if (!jobIdsArg) {
        debug('Error: JOB_IDS environment variable or command-line argument required for copy operation');
        process.exit(1);
      }

      const jobIds = jobIdsArg.split(',').map(id => parseInt(id.trim())).filter(id => !isNaN(id));

      if (!jobIds.length) {
        debug('Error: No valid job IDs provided');
        process.exit(1);
      }

      debug('Starting copy operation from external database');

      if (mongoose.connection.readyState !== 1) {
        debug('Error: Destination database is not connected (readyState:', mongoose.connection.readyState, ')');
        debug('Waiting for connection to be established...');

        let attempts = 0;
        while (mongoose.connection.readyState !== 1 && attempts < 10) {
          await new Promise(resolve => setTimeout(resolve, 1000));
          attempts++;
          debug(`Connection attempt ${attempts}, readyState: ${mongoose.connection.readyState}`);
        }

        if (mongoose.connection.readyState !== 1) {
          debug('Error: Failed to establish connection to destination database');
          process.exit(1);
        }

        debug('Destination database connection established successfully');
      }

      const result = await copyJobsFromPreviousDb({
        sourceDbUri,
        jobIds,
        useNewIds,
        keepRelatedIds,
        linkToExisting,
        newStartId: NEW_START_ID,
        dryRun: DRY_RUN
      });

      debug('Copy operation result:', result);
      if (result.skipped > 0) {
        debug(`Skipped ${result.skipped} jobs that already exist in destination database:`, result.skippedIds.join(', '));
      }
      if (result.failed > 0) {
        debug(`Failed to copy ${result.failed} jobs:`);
        result.failedIds.forEach(jobId => {
          const reason = result.stats.failureReasons[jobId];
          debug(`  Job ${jobId}: [${reason.category || 'Unknown'}] ${reason.message}`);
        });
      }

      const linked = result.linked;
      if (linked && (linked.logs > 0 || linked.assigns > 0 || linked.applications > 0 || linked.files > 0 || linked.details > 0)) {
        debug('Linked existing records in destination database:');
        if (linked.logs > 0) debug(`  - ${linked.logs} job logs`);
        if (linked.assigns > 0) debug(`  - ${linked.assigns} job assignments`);
        if (linked.applications > 0) debug(`  - ${linked.applications} applications`);
        if (linked.files > 0) debug(`  - ${linked.files} application files`);
        if (linked.details > 0) debug(`  - ${linked.details} application details`);
      }
    } else {
      await shiftSpecificJobIds();
      debug('Job ID shifting completed successfully');
    }
  } catch (error) {
    debug('Operation failed:', error);
  } finally {
    await dbConn.close();
    process.exit();
  }
})();

module.exports = {
  shiftSpecificJobIds,
  safeMongoOperation,
  withTransactionRetry,
  copyJobsFromPreviousDb,
  processJobWithSeparateTransactions
};