277 lines
9.7 KiB
JavaScript
277 lines
9.7 KiB
JavaScript
#!/usr/bin/env node
|
|
'use strict';
|
|
|
|
/**
|
|
* Areas Geometry Audit Script
|
|
*
|
|
* Scans the `areas` collection for documents that would cause the geojson-rbush
|
|
* spatial index to fail during duplicate checking (checkDupAreas), or that would
|
|
* be silently filtered out by the defensive fixes added to that function.
|
|
*
|
|
* Issue classes detected:
|
|
* 1. null_geometry — document has no `geometry` field
|
|
* 2. null_coordinates — geometry exists but `coordinates` is null/missing/empty
|
|
* 3. wrong_geo_type — geometry.type is not "Polygon"
|
|
* 4. bad_nesting — coordinates nesting depth != 3 (not [[[lng,lat],...]])
|
|
* 5. nan_coordinates — coordinates contain null, undefined, or NaN values
|
|
* 6. rbush_fail — tree.insert() throws even though structure looked valid
|
|
* (mirrors the exact production code path)
|
|
*
|
|
* Documents matching any issue would be silently dropped from the rbush tree,
|
|
* meaning areas belonging to those clients may not have full dup-checking coverage.
|
|
*
|
|
* Usage (run from the server/ root):
|
|
* node scripts/audit_areas_geometry.js
|
|
* node scripts/audit_areas_geometry.js --env ./environment_prod.env
|
|
* node scripts/audit_areas_geometry.js --client=<ObjectId>
|
|
* node scripts/audit_areas_geometry.js --output=./audit_results.json
|
|
*
|
|
* Dry-run is the ONLY mode — this script never writes to the database.
|
|
* Use the output JSON to plan follow-up remediation.
|
|
*
|
|
* Options:
|
|
* --env <file> Path to environment file (default: ./environment.env)
|
|
* --env=<file> Alternate form
|
|
* --client=<ObjectId> Limit scan to a single client's areas
|
|
* --output=<file> Write full findings as JSON to this path
|
|
* --batch=<n> Cursor batch size (default: 500)
|
|
*/
|
|
|
|
const fs = require('fs');
|
|
const path = require('path');
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Arg parsing
|
|
// ---------------------------------------------------------------------------
|
|
const args = process.argv.slice(2);
|
|
let envFile = './environment.env';
|
|
let clientFilter = null;
|
|
let outputFile = null;
|
|
let batchSize = 500;
|
|
|
|
for (let i = 0; i < args.length; i++) {
|
|
if (args[i] === '--env' && args[i + 1] && !args[i + 1].startsWith('--')) {
|
|
envFile = args[i + 1]; i++;
|
|
} else if (args[i].startsWith('--env=')) {
|
|
envFile = args[i].split('=').slice(1).join('=');
|
|
} else if (args[i].startsWith('--client=')) {
|
|
clientFilter = args[i].split('=')[1];
|
|
} else if (args[i].startsWith('--output=')) {
|
|
outputFile = args[i].split('=')[1];
|
|
} else if (args[i].startsWith('--batch=')) {
|
|
batchSize = parseInt(args[i].split('=')[1], 10) || 500;
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Load environment
|
|
// ---------------------------------------------------------------------------
|
|
const envPath = path.resolve(process.cwd(), envFile);
|
|
if (!fs.existsSync(envPath)) {
|
|
console.error(`[audit_areas_geometry] Environment file not found: ${envPath}`);
|
|
process.exit(1);
|
|
}
|
|
require('dotenv').config({ path: envPath });
|
|
console.log(`[audit_areas_geometry] Environment loaded from: ${envPath}`);
|
|
if (clientFilter) console.log(`[audit_areas_geometry] Filtering to client: ${clientFilter}`);
|
|
console.log(`[audit_areas_geometry] DRY-RUN — no writes will occur\n`);
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// DB + model bootstrap
|
|
// ---------------------------------------------------------------------------
|
|
const mongoose = require('mongoose');
|
|
const { DBConnection } = require('../helpers/db/connect');
|
|
const GeojsonRbush = require('@mickeyjohn/geojson-rbush').default;
|
|
|
|
const AreaSchema = new mongoose.Schema({
|
|
properties: mongoose.Schema.Types.Mixed,
|
|
geometry: mongoose.Schema.Types.Mixed,
|
|
client: mongoose.Schema.Types.ObjectId,
|
|
}, { strict: false });
|
|
const Area = mongoose.models.Area || mongoose.model('Area', AreaSchema, 'areas');
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Validation helpers
|
|
// ---------------------------------------------------------------------------
|
|
const ISSUE = {
|
|
NULL_GEOMETRY: 'null_geometry',
|
|
NULL_COORDINATES: 'null_coordinates',
|
|
WRONG_GEO_TYPE: 'wrong_geo_type',
|
|
BAD_NESTING: 'bad_nesting',
|
|
NAN_COORDINATES: 'nan_coordinates',
|
|
RBUSH_FAIL: 'rbush_fail',
|
|
};
|
|
|
|
// Shared test tree — insert then clear to mirror the exact production code path
|
|
// without accumulating state between documents.
|
|
const _testTree = GeojsonRbush();
|
|
|
|
function hasNaNValues(coords) {
|
|
if (!Array.isArray(coords)) return true;
|
|
for (const item of coords) {
|
|
if (Array.isArray(item)) {
|
|
if (hasNaNValues(item)) return true;
|
|
} else {
|
|
if (item === null || item === undefined || isNaN(item)) return true;
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
function coordinateDepth(coords) {
|
|
if (!Array.isArray(coords)) return 0;
|
|
if (!Array.isArray(coords[0])) return 1;
|
|
if (!Array.isArray(coords[0][0])) return 2;
|
|
return 3;
|
|
}
|
|
|
|
function auditDocument(doc) {
|
|
const issues = [];
|
|
const geo = doc.geometry;
|
|
|
|
if (!geo || typeof geo !== 'object') {
|
|
issues.push(ISSUE.NULL_GEOMETRY);
|
|
return issues; // no point checking further
|
|
}
|
|
|
|
if (!geo.coordinates || !Array.isArray(geo.coordinates) || geo.coordinates.length === 0) {
|
|
issues.push(ISSUE.NULL_COORDINATES);
|
|
return issues;
|
|
}
|
|
|
|
if (geo.type !== 'Polygon') {
|
|
issues.push(ISSUE.WRONG_GEO_TYPE);
|
|
// still check coordinates
|
|
}
|
|
|
|
if (coordinateDepth(geo.coordinates) !== 3) {
|
|
issues.push(ISSUE.BAD_NESTING);
|
|
}
|
|
|
|
if (hasNaNValues(geo.coordinates)) {
|
|
issues.push(ISSUE.NAN_COORDINATES);
|
|
}
|
|
|
|
// If no structural issues found yet, mirror the exact production code path:
|
|
// try inserting into a real rbush tree and clear it immediately after.
|
|
if (issues.length === 0) {
|
|
const feature = { type: 'Feature', geometry: geo, properties: {} };
|
|
try {
|
|
_testTree.insert(feature);
|
|
_testTree.clear();
|
|
} catch (e) {
|
|
issues.push(ISSUE.RBUSH_FAIL);
|
|
}
|
|
}
|
|
|
|
return issues;
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Main audit
|
|
// ---------------------------------------------------------------------------
|
|
async function run() {
|
|
const db = new DBConnection('audit_areas_geometry');
|
|
await db.connect({ setupExitHandlers: false, setupEventListeners: false, exitOnError: false });
|
|
|
|
const query = clientFilter ? { client: new mongoose.Types.ObjectId(clientFilter) } : {};
|
|
const cursor = Area.find(query, { _id: 1, client: 1, geometry: 1 }).lean().cursor({ batchSize });
|
|
|
|
const findings = []; // { _id, client, issues[] }
|
|
const countByIssue = {};
|
|
Object.values(ISSUE).forEach(k => { countByIssue[k] = 0; });
|
|
|
|
let scanned = 0;
|
|
let invalidCount = 0;
|
|
|
|
process.stdout.write('[audit_areas_geometry] Scanning');
|
|
|
|
for await (const doc of cursor) {
|
|
scanned++;
|
|
if (scanned % 5000 === 0) process.stdout.write('.');
|
|
|
|
const issues = auditDocument(doc);
|
|
if (issues.length === 0) continue;
|
|
|
|
invalidCount++;
|
|
issues.forEach(k => { countByIssue[k]++; });
|
|
findings.push({
|
|
_id: doc._id.toString(),
|
|
client: doc.client ? doc.client.toString() : null,
|
|
issues,
|
|
});
|
|
}
|
|
|
|
console.log(` done.\n`);
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Report
|
|
// ---------------------------------------------------------------------------
|
|
console.log('='.repeat(60));
|
|
console.log('AUDIT SUMMARY');
|
|
console.log('='.repeat(60));
|
|
console.log(` Total scanned : ${scanned.toLocaleString()}`);
|
|
console.log(` Invalid docs : ${invalidCount.toLocaleString()} (${scanned ? ((invalidCount / scanned) * 100).toFixed(2) : 0}%)`);
|
|
console.log('');
|
|
console.log(' Breakdown by issue type:');
|
|
Object.entries(countByIssue).forEach(([key, n]) => {
|
|
if (n > 0) console.log(` ${key.padEnd(20)} ${n.toLocaleString()}`);
|
|
});
|
|
console.log('='.repeat(60));
|
|
|
|
if (invalidCount > 0) {
|
|
// Group by client for a quick "which clients are affected" view
|
|
const byClient = {};
|
|
findings.forEach(f => {
|
|
const k = f.client || '(no client)';
|
|
if (!byClient[k]) byClient[k] = { count: 0, issues: new Set() };
|
|
byClient[k].count++;
|
|
f.issues.forEach(i => byClient[k].issues.add(i));
|
|
});
|
|
|
|
console.log('\nAFFECTED CLIENTS:');
|
|
Object.entries(byClient)
|
|
.sort((a, b) => b[1].count - a[1].count)
|
|
.forEach(([clientId, info]) => {
|
|
console.log(` ${clientId} → ${info.count} doc(s) [${[...info.issues].join(', ')}]`);
|
|
});
|
|
|
|
// Print first 20 IDs inline for quick reference
|
|
console.log('\nFIRST 20 INVALID DOCUMENT IDs:');
|
|
findings.slice(0, 20).forEach(f => {
|
|
console.log(` ${f._id} client=${f.client} issues=${f.issues.join(',')}`);
|
|
});
|
|
if (findings.length > 20) {
|
|
console.log(` ... and ${findings.length - 20} more (use --output to capture all)`);
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Optional JSON output
|
|
// ---------------------------------------------------------------------------
|
|
if (outputFile) {
|
|
const outPath = path.resolve(process.cwd(), outputFile);
|
|
const payload = {
|
|
auditDate: new Date().toISOString(),
|
|
envFile,
|
|
clientFilter: clientFilter || null,
|
|
totalScanned: scanned,
|
|
totalInvalid: invalidCount,
|
|
countByIssue,
|
|
findings,
|
|
};
|
|
fs.writeFileSync(outPath, JSON.stringify(payload, null, 2));
|
|
console.log(`\n[audit_areas_geometry] Full findings written to: ${outPath}`);
|
|
} else if (invalidCount > 0) {
|
|
console.log('\nTip: re-run with --output=./audit_results.json to capture all IDs.');
|
|
}
|
|
|
|
await db.close();
|
|
process.exit(0);
|
|
}
|
|
|
|
run().catch(err => {
|
|
console.error('[audit_areas_geometry] Fatal error:', err);
|
|
process.exit(1);
|
|
});
|