170 lines
6.8 KiB
JavaScript
170 lines
6.8 KiB
JavaScript
'use strict';
|
||
|
||
/**
|
||
* Migration: Fix double-counted rptOp.coverage on Job documents
|
||
*
|
||
* Background:
|
||
* The /reportOps endpoint previously computed coverage as:
|
||
* sum(App.totalSprayed) + sum(App.totalSprLength) × swathWidth × 1e-4
|
||
* The length-based term was intended to cover SatLoc/non-AgNav apps that have no
|
||
* totalSprayed area. Once totalSprLength was backfilled for ALL app types (including
|
||
* AgNav), the addition double-counted AgNav coverage because
|
||
* totalSprLength × swathWidth ≈ totalSprayed for AgNav data.
|
||
* Jobs whose pre-app reports were generated and saved while this logic was active
|
||
* have an inflated rptOp.coverage stored in DB.
|
||
*
|
||
* Fix:
|
||
* For each affected job, sum App.totalSprayed across all its applications and compare
|
||
* against the stored rptOp.coverage. If the stored value exceeds the fresh sum by more
|
||
* than 50%, the job is considered inflated and rptOp.coverage is reset to the fresh sum.
|
||
*
|
||
* The 50% threshold is chosen because the double-count roughly doubles the coverage
|
||
* (AgNav: totalSprLength × swathWidth ≈ totalSprayed), so a genuine inflation shows
|
||
* a delta near 100%. The 50% guard safely excludes small floating-point drift and
|
||
* legitimate manual overrides while catching all real double-counted values.
|
||
*
|
||
* Jobs where the stored value is at or below the fresh sum are left untouched —
|
||
* this preserves any intentional manual reductions entered by the user.
|
||
*
|
||
* Field updated:
|
||
* Job.rptOp.coverage — reset to sum(App.totalSprayed) in ha when inflation > 50%
|
||
*
|
||
* Usage:
|
||
* node scripts/fix_rptop_coverage_double_count.js
|
||
* node scripts/fix_rptop_coverage_double_count.js --dry-run
|
||
* node scripts/fix_rptop_coverage_double_count.js --tier-days=30
|
||
* node scripts/fix_rptop_coverage_double_count.js --from-date=2025-01-01
|
||
* node scripts/fix_rptop_coverage_double_count.js --env ./environment_prod.env
|
||
*
|
||
* Options:
|
||
* --env <path> Environment file (default: ./environment.env)
|
||
* --dry-run Report only; make no DB writes
|
||
* --tier-days=N Only process jobs created in the last N days
|
||
* --from-date=YYYY-MM-DD Only process jobs created on/after this date (UTC)
|
||
* --batch-size=N Jobs per bulkWrite flush (default: 50)
|
||
*/
|
||
|
||
// ─── Environment bootstrap (must run before any require that reads process.env) ─
|
||
const path = require('path');
|
||
const _args = process.argv.slice(2);
|
||
let envFile = './environment.env';
|
||
for (let i = 0; i < _args.length; i++) {
|
||
if (_args[i] === '--env' && _args[i + 1]) { envFile = _args[i + 1]; i++; }
|
||
else if (_args[i].startsWith('--env=')) { envFile = _args[i].split('=')[1]; }
|
||
}
|
||
require('dotenv').config({ path: path.resolve(process.cwd(), envFile) });
|
||
|
||
const debug = require('debug')('agm:fix-rptop-coverage');
|
||
const { DBConnection } = require('../helpers/db/connect.js');
|
||
const { Job, App } = require('../model/index.js');
|
||
|
||
// ─── Argument parsing ─────────────────────────────────────────────────────────
|
||
const DRY_RUN = _args.includes('--dry-run');
|
||
const BATCH_SIZE = parseInt((_args.find(a => a.startsWith('--batch-size=')) || '').split('=')[1], 10) || 50;
|
||
const TIER_DAYS = parseInt((_args.find(a => a.startsWith('--tier-days=')) || '').split('=')[1], 10) || 0;
|
||
const FROM_DATE = (_args.find(a => a.startsWith('--from-date=')) || '').split('=')[1] || '';
|
||
|
||
const INFLATION_THRESHOLD = 0.5; // only fix if stored exceeds fresh sum by > 50%
|
||
|
||
// ─── Main migration ───────────────────────────────────────────────────────────
|
||
async function migrate() {
|
||
debug(`Starting${DRY_RUN ? ' (DRY RUN)' : ''}...`);
|
||
|
||
const jobFilter = { 'rptOp.coverage': { $gt: 0 } };
|
||
if (TIER_DAYS) jobFilter.createdAt = { $gte: new Date(Date.now() - TIER_DAYS * 86400 * 1000) };
|
||
if (FROM_DATE) jobFilter.createdAt = { $gte: new Date(FROM_DATE + 'T00:00:00Z') };
|
||
|
||
debug('Job filter: %o', jobFilter);
|
||
|
||
const stats = { examined: 0, updated: 0, skipped: 0, noApps: 0, errors: 0, startedAt: Date.now() };
|
||
let bulk = [];
|
||
|
||
async function flushBulk() {
|
||
if (!bulk.length) return;
|
||
if (!DRY_RUN) await Job.bulkWrite(bulk, { ordered: false });
|
||
bulk = [];
|
||
}
|
||
|
||
const cursor = Job
|
||
.find(jobFilter)
|
||
.select('_id rptOp')
|
||
.sort({ _id: -1 })
|
||
.lean()
|
||
.cursor();
|
||
|
||
for await (const job of cursor) {
|
||
stats.examined++;
|
||
|
||
try {
|
||
const [agg] = await App.aggregate([
|
||
{ $match: { jobId: job._id, markedDelete: { $ne: true } } },
|
||
{ $group: { _id: null, totalSprayed: { $sum: '$totalSprayed' } } },
|
||
]);
|
||
|
||
if (!agg || !agg.totalSprayed) {
|
||
stats.noApps++;
|
||
continue;
|
||
}
|
||
|
||
const stored = job.rptOp.coverage;
|
||
const fresh = agg.totalSprayed;
|
||
const delta = stored - fresh;
|
||
|
||
// Only fix inflated values: stored must exceed fresh sum by > 50%
|
||
if (delta / fresh <= INFLATION_THRESHOLD) {
|
||
stats.skipped++;
|
||
continue;
|
||
}
|
||
|
||
debug(`Job ${job._id}: stored=${stored.toFixed(4)} ha fresh=${fresh.toFixed(4)} ha over by ${delta.toFixed(4)} ha (${(delta / fresh * 100).toFixed(1)}%)`);
|
||
|
||
bulk.push({
|
||
updateOne: {
|
||
filter: { _id: job._id },
|
||
update: { $set: { 'rptOp.coverage': fresh } },
|
||
},
|
||
});
|
||
stats.updated++;
|
||
|
||
if (bulk.length >= BATCH_SIZE) await flushBulk();
|
||
|
||
} catch (err) {
|
||
debug(`Error on job ${job._id}: ${err.message}`);
|
||
stats.errors++;
|
||
}
|
||
|
||
if (stats.examined % 100 === 0) {
|
||
const elapsed = ((Date.now() - stats.startedAt) / 1000).toFixed(1);
|
||
debug(`Progress: examined=${stats.examined} updated=${stats.updated} skipped=${stats.skipped} errors=${stats.errors} elapsed=${elapsed}s`);
|
||
}
|
||
}
|
||
|
||
await flushBulk();
|
||
|
||
const elapsed = ((Date.now() - stats.startedAt) / 1000).toFixed(1);
|
||
debug('─'.repeat(60));
|
||
debug('Migration complete.');
|
||
debug(` Examined : ${stats.examined}`);
|
||
debug(` Updated : ${stats.updated}${DRY_RUN ? ' (dry-run — no writes)' : ''}`);
|
||
debug(` Skipped : ${stats.skipped} (stored coverage already correct)`);
|
||
debug(` No apps : ${stats.noApps} (job has no App records)`);
|
||
debug(` Errors : ${stats.errors}`);
|
||
debug(` Duration : ${elapsed}s`);
|
||
debug('─'.repeat(60));
|
||
}
|
||
|
||
// ─── Entry point ──────────────────────────────────────────────────────────────
|
||
const workerDB = new DBConnection('fix-rptop-coverage');
|
||
workerDB.initialize({
|
||
setupExitHandlers: false,
|
||
onReady: async () => {
|
||
try {
|
||
await migrate();
|
||
process.exit(0);
|
||
} catch (err) {
|
||
debug('Migration failed:', err);
|
||
process.exit(1);
|
||
}
|
||
}
|
||
});
|