agmission/server/scripts/backfill_application_datetimes.js

#!/usr/bin/env node
/* DEPRECATED — use scripts/migrate_applications.js instead.
 * This script is kept for reference only.
 */
'use strict';

/**
 * Backfill Application.utcOffset, Application.startDateTimeUTC, and Application.endDateTimeUTC.
 *
 * Usage:
 *   node scripts/backfill_application_datetimes.js
 *   node scripts/backfill_application_datetimes.js --env ../environment_prod.env
 *   node scripts/backfill_application_datetimes.js --dry-run
 *   node scripts/backfill_application_datetimes.js --missing-limit 200
 *   node scripts/backfill_application_datetimes.js --force
 *     Re-process ALL apps that have startDateTime, even those already having UTC fields.
 *     Use this after a formula fix to recompute previously stored values.
 */

const path = require('path');

const args = process.argv.slice(2);
let envFile = './environment.env';
let dryRun = false;
let missingLimit = 100;
let force = false;

for (let i = 0; i < args.length; i++) {
  if (args[i] === '--env' && args[i + 1]) {
    envFile = args[i + 1];
    i++;
  } else if (args[i] === '--dry-run' || args[i] === '--preview') {
    dryRun = true;
  } else if (args[i] === '--missing-limit' && args[i + 1]) {
    const parsed = parseInt(args[i + 1], 10);
    if (!Number.isNaN(parsed) && parsed > 0) missingLimit = parsed;
    i++;
  } else if (args[i] === '--force') {
    force = true;
  }
}

const envPath = path.resolve(process.cwd(), envFile);
console.log(`Loading environment from: ${envPath}`);
require('dotenv').config({ path: envPath });

const debug = require('debug')('agm:backfill-application-datetimes');
const { DBConnection } = require('../helpers/db/connect.js');
const { App, Job, AppFile, AppDetail } = require('../model/index.js');
const appDateTime = require('../helpers/application_datetime');

const BATCH_SIZE = 100;
const PROGRESS_EVERY = 100;

async function getReferenceDetail(appId) {
  const files = await AppFile.find({ appId, markedDelete: { $ne: true } }, { _id: 1 }).lean();
  const fileIds = files.map(file => file._id);

  if (fileIds.length) {
    return AppDetail.findOne({
      fileId: { $in: fileIds },
      lat: { $type: 'number' },
      lon: { $type: 'number' }
    }).sort({ gpsTime: 1, _id: 1 }).lean();
  }

  return AppDetail.findOne({
    appId,
    lat: { $type: 'number' },
    lon: { $type: 'number' }
  }).sort({ gpsTime: 1, _id: 1 }).lean();
}

async function reportMissingLegacyDateApps(limit = 100) {
  const missingLegacyQuery = {
    $or: [
      { startDateTime: { $exists: false } },
      { startDateTime: null },
      { endDateTime: { $exists: false } },
      { endDateTime: null }
    ]
  };

  const totalMissingLegacy = await App.countDocuments(missingLegacyQuery);
  if (!totalMissingLegacy) return;

  const apps = await App.find(
    missingLegacyQuery,
    '_id jobId status startDateTime endDateTime createdDate updateDate errorMsg'
  ).sort({ _id: 1 }).limit(limit).lean();

  // Job._id is a numeric auto-increment field (mongoose-sequence, inc_field: '_id').
  // Application.jobId stores that same numeric _id directly.
  const jobIds = [...new Set(apps.map(a => a.jobId).filter(v => v !== null && v !== undefined && Number.isFinite(Number(v))).map(v => Number(v)))];

  const jobs = jobIds.length
    ? await Job.find({ _id: { $in: jobIds } }, '_id name status').lean()
    : [];
  const jobMapById = new Map(jobs.map(j => [Number(j._id), j]));

  console.log('');
  console.log(`[Backfill] Apps missing legacy start/end datetime (cannot be backfilled by this script): total=${totalMissingLegacy}, showing=${apps.length}`);

  for (const app of apps) {
    const files = await AppFile.find({ appId: app._id, markedDelete: { $ne: true } }, { _id: 1 }).lean();
    const fileIds = files.map(f => f._id);
    const detailCount = fileIds.length
      ? await AppDetail.countDocuments({ fileId: { $in: fileIds } })
      : await AppDetail.countDocuments({ appId: app._id });

    const job = (app.jobId !== null && app.jobId !== undefined) ? (jobMapById.get(Number(app.jobId)) || null) : null;
    const likelyNoDataFiles = files.length === 0 || detailCount === 0;

    console.log(
      `[Backfill] appId=${app._id}`
      + ` jobId=${app.jobId || 'null'}`
      + ` jobNo=${job ? job._id : 'n/a'}`
      + ` jobStatus=${job && job.status !== undefined ? job.status : 'n/a'}`
      + ` appStatus=${app.status !== undefined ? app.status : 'n/a'}`
      + ` startDateTime=${app.startDateTime || 'null'}`
      + ` endDateTime=${app.endDateTime || 'null'}`
      + ` appFiles=${files.length}`
      + ` appDetails=${detailCount}`
      + ` likelyNoDataFiles=${likelyNoDataFiles ? 'yes' : 'no'}`
    );
  }

  if (totalMissingLegacy > apps.length) {
    console.log(`[Backfill] ... ${totalMissingLegacy - apps.length} more apps omitted. Use --missing-limit <n> to show more.`);
  }
}

async function migrate() {
  // --force: recompute all apps regardless of whether UTC fields already exist.
  // Use this after a formula fix to correct previously stored values.
  const query = force
    ? { startDateTime: { $exists: true, $ne: null } }
    : {
      startDateTime: { $exists: true, $ne: null },
      $or: [
        { utcOffset: { $exists: false } },
        { utcOffset: 0 },               // May have been written incorrectly (worker bug: coords not found)
        { startDateTimeUTC: { $exists: false } },
        { endDateTimeUTC: { $exists: false } },
        { $expr: { $gt: ['$startDateTimeUTC', '$endDateTimeUTC'] } } // Remaining inverted dates (bad data)
      ]
    };

  if (force) debug('--force mode: reprocessing all apps with startDateTime');

  const total = await App.countDocuments(query);
  debug(`Apps to process: ${total}`);

  // Emit quick diagnostics so operators can understand why query matches 0 apps.
  const diag = {
    totalApps: await App.countDocuments({}),
    missingStartDateTime: await App.countDocuments({ startDateTime: { $exists: false } }),
    missingEndDateTime: await App.countDocuments({ endDateTime: { $exists: false } }),
    nullStartDateTime: await App.countDocuments({ startDateTime: null }),
    nullEndDateTime: await App.countDocuments({ endDateTime: null }),
    missingStartDateTimeUTC: await App.countDocuments({ startDateTimeUTC: { $exists: false } }),
    missingEndDateTimeUTC: await App.countDocuments({ endDateTimeUTC: { $exists: false } }),
    missingUtcOffset: await App.countDocuments({ utcOffset: { $exists: false } }),
    zeroUtcOffset: await App.countDocuments({ utcOffset: 0 }),
    invertedUtcDates: await App.countDocuments({ $expr: { $gt: ['$startDateTimeUTC', '$endDateTimeUTC'] } }),
    selectableByScript: total
  };
  debug(`Diagnostics: ${JSON.stringify(diag)}`);

  if (!total) {
    debug('No applications matched selection criteria. This script only backfills apps that already have legacy startDateTime.');
    if (diag.missingStartDateTime > 0 || diag.missingEndDateTime > 0 || diag.nullStartDateTime > 0 || diag.nullEndDateTime > 0) {
      await reportMissingLegacyDateApps(missingLimit);
    }
    return;
  }

  const cursor = App.find(query, '_id startDateTime endDateTime').sort({ _id: 1 }).lean().cursor();

  let processed = 0;
  let updated = 0;
  let skipped = 0;
  let errors = 0;
  let bulk = [];

  for await (const app of cursor) {
    try {
      const referenceDetail = await getReferenceDetail(app._id);
      if (!referenceDetail) {
        skipped++;
        debug(`Skip ${app._id}: no reference detail with coordinates found`);
      } else {
        const dateFields = appDateTime.buildApplicationDateFields({
          startDateTime: app.startDateTime,
          endDateTime: app.endDateTime,
          latitude: referenceDetail.lat,
          longitude: referenceDetail.lon
        });

        bulk.push({
          updateOne: {
            filter: { _id: app._id },
            update: {
              $set: {
                utcOffset: dateFields.utcOffset,
                startDateTimeUTC: dateFields.startDateTimeUTC,
                endDateTimeUTC: dateFields.endDateTimeUTC
              }
            }
          }
        });
        updated++;
      }

      if (bulk.length >= BATCH_SIZE) {
        if (!dryRun) await App.bulkWrite(bulk, { ordered: false });
        bulk = [];
      }
    } catch (err) {
      errors++;
      debug(`Error on App ${app._id}: ${err.message}`);
    }

    processed++;
    if (processed % PROGRESS_EVERY === 0) {
      debug(`Progress: ${processed}/${total} (updated=${updated}, skipped=${skipped}, errors=${errors})`);
    }
  }

  if (bulk.length && !dryRun) {
    await App.bulkWrite(bulk, { ordered: false });
  }

  debug(`Done. processed=${processed}, updated=${updated}, skipped=${skipped}, errors=${errors}${dryRun ? ' (DRY RUN — no writes)' : ''}`);
}

const workerDB = new DBConnection('Backfill application datetimes');
workerDB.initialize({
  setupExitHandlers: false,
  onReady: async () => {
    try {
      await migrate();
      process.exit(0);
    } catch (err) {
      debug('Backfill failed:', err);
      process.exit(1);
    }
  }
});