Devin Major
df31b2080d
All checks were successful
Server Tests / Mocha – Unit & Utility Tests (push) Successful in 42s
+ Added public data export API enhancements, tests, and customer documentation + Extended /api/v1 data export endpoints with richer session, records, area, and async export output + Added confirmed/fallback report values, client metadata, mapped area, over-spray, volume/apprate (string) units, and weather blocks + Normalized flowController to "No FC" and align record field names with playback output + Converted record wind speed output to knots, add Fligh Mater only record/export fields behind fm=true, and persist fm on export jobs + Added export status/area constants, HTTP 202 support, route-level API docs, and per-account export rate limiting support + Added comprehensive endpoint, format, and verification test coverage plus test-suite README + Added customer-facing data export design, integration, rate-limit, and documentation index guides + Updated README/DLQ docs and related documentation links to current HTTPS dashboard paths
481 lines
18 KiB
JavaScript
481 lines
18 KiB
JavaScript
'use strict';
|
||
|
||
/**
|
||
* Public Data Export API controller — /api/v1/ routes.
|
||
* All functions are authenticated via checkApiKey (X-API-Key header).
|
||
* req.uid is set identically to checkUser, so all ownership scoping is automatic.
|
||
*
|
||
* Endpoints implemented here:
|
||
* GET /api/v1/jobs/:jobId/sessions → session summary list
|
||
* GET /api/v1/jobs/:jobId/sessions/:fileId/records → raw GPS trace (paginated)
|
||
* GET /api/v1/jobs/:jobId/areas → GeoJSON spray-area polygons
|
||
*/
|
||
|
||
const ObjectId = require('mongodb').ObjectId;
|
||
const moment = require('moment');
|
||
const { Job, App, AppFile, AppDetail, JobAssign, Vehicle, Pilot } = require('../model');
|
||
const { paginateWithCursor, validateCursorParams } = require('../helpers/cursor_pagination');
|
||
const { AppParamError, AppAuthError } = require('../helpers/app_error');
|
||
const { Errors, HttpStatus, ExportAreaTypes } = require('../helpers/constants');
|
||
const utils = require('../helpers/utils');
|
||
|
||
// ─── helpers ─────────────────────────────────────────────────────────────────
|
||
|
||
/** Parse a positive-float interval value from a query/body param. Returns null if absent or invalid. */
|
||
function parseInterval(raw) {
|
||
if (raw == null || raw === '') return null;
|
||
const v = parseFloat(raw);
|
||
return isFinite(v) && v > 0 ? v : null;
|
||
}
|
||
|
||
/** Decode satsInView from raw satsIn field (>99 means corrected) */
|
||
function decodeSatsIn(raw) {
|
||
if (!utils.isNumber(raw)) return null;
|
||
return raw > 99 ? raw - 100 : raw;
|
||
}
|
||
|
||
/** Decode correctionId and waasId from tslu and calcodeFreq */
|
||
function decodeCorrectionFields(tslu, calcodeFreq) {
|
||
const correctionId = utils.isNumber(tslu) ? (tslu > 100 ? tslu - 100 : tslu) : null;
|
||
let waasId = null;
|
||
if (utils.isNumber(calcodeFreq) && calcodeFreq >= 20001 && calcodeFreq <= 29999) {
|
||
waasId = calcodeFreq - 20000;
|
||
}
|
||
return { correctionId, waasId };
|
||
}
|
||
|
||
/**
|
||
* Compute appRateApplied from raw fields.
|
||
* Formula: lminApp / (grSpeed_m_s × swath_m) × 10000
|
||
* Returns null on zero-division to avoid Infinity.
|
||
*/
|
||
function computeAppRateApplied(lminApp, grSpeed, swath) {
|
||
if (!utils.isNumber(lminApp) || !utils.isNumber(grSpeed) || !utils.isNumber(swath)) return null;
|
||
if (grSpeed === 0 || swath === 0) return null;
|
||
return lminApp / (grSpeed * swath) * 10000;
|
||
}
|
||
|
||
/**
|
||
* Convert AppDetail.gpsTime to an ISO UTC timestamp.
|
||
* Supports both epoch-seconds and legacy seconds-of-day values.
|
||
*/
|
||
function toRecordTimeUtc(gpsTime, appStartDateTime) {
|
||
if (!utils.isNumber(gpsTime)) return null;
|
||
|
||
// Epoch seconds (>= year 2000-01-01 UTC) can be converted directly.
|
||
if (gpsTime >= 946684800) {
|
||
return moment.unix(gpsTime).utc().toISOString();
|
||
}
|
||
|
||
// Legacy format: seconds-of-day, anchor to app start date when available.
|
||
const base = moment.utc(appStartDateTime, [moment.ISO_8601, 'YYYYMMDDTHHmmss'], true);
|
||
if (base.isValid()) {
|
||
const dayOffset = Math.floor(gpsTime / 86400);
|
||
const secOfDay = ((gpsTime % 86400) + 86400) % 86400;
|
||
return base.clone().startOf('day').add(dayOffset, 'days').add(secOfDay, 'seconds').toISOString();
|
||
}
|
||
|
||
// Fallback for malformed app start datetime.
|
||
return moment.unix(gpsTime).utc().toISOString();
|
||
}
|
||
|
||
/**
|
||
* Map a raw AppDetail document to the public API record shape.
|
||
* sessionMeta contains session-constant fields from AppFile.meta injected once per page.
|
||
*/
|
||
/**
|
||
* Normalise flow controller name to match playback display:
|
||
* null/empty/case-insensitive 'none' values → 'No FC'.
|
||
*/
|
||
function normaliseFlowController(fcName) {
|
||
return (fcName && !/none/i.test(fcName)) ? fcName : 'No FC';
|
||
}
|
||
|
||
function getLaserAlt(detail) {
|
||
return detail?.laserAlt ?? detail?.raserAlt ?? null;
|
||
}
|
||
|
||
function mapDetailRecord(d, sessionMeta, appStartDateTime, includeFm = false) {
|
||
const { correctionId, waasId } = decodeCorrectionFields(d.tslu, d.calcodeFreq);
|
||
const appRateApplied = computeAppRateApplied(d.lminApp, d.grSpeed, d.swath);
|
||
const pulsesPerLiter = sessionMeta?.pulsesPerLit ?? null;
|
||
const rec = {
|
||
// GPS Data
|
||
timeUtc: toRecordTimeUtc(d.gpsTime, appStartDateTime),
|
||
gpsTime: d.gpsTime,
|
||
lat: d.lat,
|
||
lon: d.lon,
|
||
utmX: d.utmX,
|
||
utmY: d.utmY,
|
||
alt: d.alt,
|
||
grSpeed: d.grSpeed,
|
||
heading: d.head,
|
||
xTrack: d.xTrack,
|
||
lockedLine: d.llnum,
|
||
hdop: d.stdHdop,
|
||
satsInView: decodeSatsIn(d.satsIn),
|
||
correctionId,
|
||
waasId,
|
||
sprayStat: d.sprayStat,
|
||
// Application Info
|
||
flowRateApplied: d.lminApp,
|
||
flowRateRequired: d.lminReq,
|
||
appRateRequired: d.lhaReq,
|
||
appRateApplied,
|
||
swathWidth: d.swath,
|
||
boomPressure_psi: d.psi,
|
||
// Session-constant fields from AppFile.meta (repeated per record for flat-file consumers)
|
||
flowController: normaliseFlowController(sessionMeta?.fcName),
|
||
sprayOnLag_s: sessionMeta?.sprOnLag ?? null,
|
||
sprayOffLag_s: sessionMeta?.sprOffLag ?? null,
|
||
pulsesPerLiter,
|
||
rpm: d.rpm,
|
||
// MET — wind speed in knots to match playback display; AppDetail stores m/s internally
|
||
windSpeed_kt: utils.isNumber(d.windSpd) ? +(d.windSpd * 1.94384).toFixed(4) : null,
|
||
windDir_deg: d.windDir,
|
||
temp_c: d.temp,
|
||
humidity_pct: d.humid
|
||
};
|
||
if (includeFm) {
|
||
// Flight Master / AgDisp fields — only included when fm=true is requested.
|
||
// raserAlt is a typo in the AppDetail schema; exposed here as laserAlt_m.
|
||
rec.sprayHeight_m = d.sprayHeight ?? null;
|
||
rec.driftX_m = d.driftX ?? null;
|
||
rec.driftY_m = d.driftY ?? null;
|
||
rec.depositX_m = d.depositX ?? null;
|
||
rec.depositY_m = d.depositY ?? null;
|
||
rec.radarAlt_m = d.radarAlt ?? null;
|
||
rec.laserAlt_m = getLaserAlt(d);
|
||
}
|
||
return rec;
|
||
}
|
||
|
||
/**
|
||
* Build the confirmed-values block for a session, with fallback to raw aggregates.
|
||
* @param {Object} job - lean Job document (needs rptOp, useCustWI, weatherInfo, sprayAreas)
|
||
* @param {Object[]} apps - lean App[] for this job
|
||
*/
|
||
function buildConfirmedValues(job, apps, firstMetaAppRate = null) {
|
||
const rptOp = job.rptOp;
|
||
const reportConfirmed = !!(rptOp && rptOp.coverage != null);
|
||
|
||
// Area size: confirmed or sum of sprayArea polygons
|
||
const areaSize_ha = reportConfirmed
|
||
? rptOp.areaSize
|
||
: (Array.isArray(job.sprayAreas) && job.sprayAreas.length
|
||
? job.sprayAreas.reduce((s, a) => s + (a.properties?.area || 0), 0)
|
||
: null);
|
||
|
||
// Coverage: confirmed or sum of App.totalSprayed
|
||
const coverage_ha = reportConfirmed
|
||
? rptOp.coverage
|
||
: apps.reduce((s, a) => s + (a.totalSprayed || 0), 0);
|
||
|
||
// AppRate: confirmed or fallback to first AppFile.meta.appRate per requirements.
|
||
const appRate = reportConfirmed ? rptOp.appRate : firstMetaAppRate;
|
||
|
||
const sprayVolume = (utils.isNumber(coverage_ha) && utils.isNumber(appRate))
|
||
? coverage_ha * appRate
|
||
: null;
|
||
|
||
const useActualVolume = reportConfirmed ? !!(rptOp.useActualVol) : false;
|
||
const actualVolume = (reportConfirmed && useActualVolume) ? (rptOp.actualVol ?? null) : null;
|
||
const effectiveVolume = useActualVolume ? actualVolume : sprayVolume;
|
||
|
||
// Rate and volume units (derived from job setting)
|
||
const appRateUnitCode = utils.isNumber(job.appRateUnit) ? job.appRateUnit : null;
|
||
const appRateUnit = appRateUnitCode != null ? utils.rateUnitString(appRateUnitCode, true) : null;
|
||
const volumeUnit = appRateUnitCode != null ? utils.rateUnitString(appRateUnitCode, true, 1) : null;
|
||
|
||
const useCustomWeather = !!job.useCustWI;
|
||
const weather = (useCustomWeather && job.weatherInfo)
|
||
? {
|
||
windSpeed_kt: job.weatherInfo.windSpd ?? null,
|
||
windDir: job.weatherInfo.windDir ?? null,
|
||
temp_c: job.weatherInfo.temp ?? null,
|
||
humidity_pct: job.weatherInfo.humid ?? null
|
||
}
|
||
: null;
|
||
|
||
const overSprayedPct = (utils.isNumber(coverage_ha) && utils.isNumber(areaSize_ha) && areaSize_ha !== 0)
|
||
? ((coverage_ha - areaSize_ha) / areaSize_ha) * 100
|
||
: null;
|
||
|
||
return {
|
||
reportConfirmed,
|
||
areaSize_ha,
|
||
coverage_ha,
|
||
overSprayedPct,
|
||
appRate,
|
||
appRateUnit,
|
||
appRateConfirmed: reportConfirmed ? appRate : null,
|
||
sprayVolume,
|
||
volumeUnit,
|
||
useActualVolume,
|
||
actualVolume,
|
||
effectiveVolume,
|
||
useCustomWeather,
|
||
weather
|
||
};
|
||
}
|
||
|
||
/** Verify the job belongs to the authenticated owner (req.uid via byPuid). */
|
||
async function ownerJob(jobId, ownerId) {
|
||
const job = await Job.findOne({ _id: jobId, markedDelete: { $ne: true } })
|
||
.populate('operator', '_id name')
|
||
.populate('vehicle', '_id name tailNumber')
|
||
.populate('client', '_id name')
|
||
.lean();
|
||
if (!job) AppParamError.throw(Errors.JOB_NOT_FOUND);
|
||
if (!job.byPuid || job.byPuid.toString() !== ownerId.toString()) AppAuthError.throw();
|
||
return job;
|
||
}
|
||
|
||
// ─── Session Summary ─────────────────────────────────────────────────────────
|
||
|
||
/**
|
||
* GET /api/v1/jobs/:jobId/sessions
|
||
*
|
||
* Returns one summary record per uploaded application session (App + AppFile).
|
||
* Includes reportConfirmed block with fallback to raw aggregates.
|
||
*
|
||
* FE / integration note:
|
||
* - Poll this endpoint after the file-upload job status becomes "done".
|
||
* - Re-fetch when reportConfirmed changes from false to true (applicator confirms report).
|
||
*/
|
||
async function getSessions(req, res) {
|
||
const jobId = parseInt(req.params.jobId, 10);
|
||
if (!isFinite(jobId)) AppParamError.throw('invalid jobId');
|
||
|
||
const job = await ownerJob(jobId, req.uid);
|
||
|
||
// Get all non-deleted Apps for this job
|
||
const apps = await App.find({ jobId, markedDelete: { $ne: true } })
|
||
.sort({ createdDate: 1 })
|
||
.lean();
|
||
|
||
if (!apps.length) {
|
||
return res.json({ data: [], jobId, reportConfirmed: false });
|
||
}
|
||
|
||
const appIds = apps.map(a => a._id);
|
||
|
||
// Get all AppFiles grouped by appId
|
||
const appFiles = await AppFile.find({ appId: { $in: appIds }, markedDelete: { $ne: true } })
|
||
.sort({ agn: 1 })
|
||
.lean();
|
||
|
||
const filesByApp = {};
|
||
for (const f of appFiles) {
|
||
const key = f.appId.toString();
|
||
if (!filesByApp[key]) filesByApp[key] = [];
|
||
filesByApp[key].push(f);
|
||
}
|
||
|
||
const firstAppFile = appFiles.length ? appFiles[0] : null;
|
||
const firstMetaAppRate = firstAppFile?.meta?.appRate ?? null;
|
||
|
||
// Latest JobAssign for pilot traceability
|
||
const assign = await JobAssign.findOne({ jobId, status: { $gte: 0 } })
|
||
.sort({ createdAt: -1 })
|
||
.lean();
|
||
|
||
const confirmedBlock = buildConfirmedValues(job, apps, firstMetaAppRate);
|
||
const mappedArea_ha = Array.isArray(job.sprayAreas)
|
||
? job.sprayAreas.reduce((s, a) => s + (a?.properties?.area || 0), 0)
|
||
: null;
|
||
|
||
const sessions = apps.map(app => {
|
||
const files = filesByApp[app._id.toString()] || [];
|
||
const firstFile = files[0]; // primary file for metadata
|
||
const meta = firstFile?.meta || {};
|
||
|
||
return {
|
||
sessionId: app._id,
|
||
fileName: app.fileName,
|
||
startDateTime: app.startDateTime,
|
||
endDateTime: app.endDateTime,
|
||
// Timing
|
||
totalFlightTime_s: app.totalFlightTime ?? null,
|
||
totalSprayTime_s: app.totalSprayTime ?? null,
|
||
totalTurnTime_s: app.totalTurnTime ?? null,
|
||
// Application
|
||
totalSprayed_ha: app.totalSprayed ?? null,
|
||
totalSprayMat: app.totalSprayMat ?? null,
|
||
totalSprayMatUnit: utils.isNumber(app.totalSprayMatUnit) ? utils.rateUnitString(app.totalSprayMatUnit, true, 1) : null,
|
||
avgSpraySpeed_ms: app.avgSpraySpeed ?? null,
|
||
// File metadata (from first AppFile)
|
||
sprayZoneName: meta.areaOrZone ?? null,
|
||
sprayZoneArea_ha: meta.sprCoverage?.[1] ?? null,
|
||
appRate: meta.appRate ?? null,
|
||
appRateUnit: confirmedBlock.appRateUnit,
|
||
matType: meta.matType ?? null, // 'wet' | 'dry'
|
||
flowController: normaliseFlowController(meta.fcName),
|
||
sprayOnLag_s: meta.sprOnLag ?? null,
|
||
sprayOffLag_s: meta.sprOffLag ?? null,
|
||
pulsesPerLiter: meta.pulsesPerLit ?? null,
|
||
// Per-session files list (for consumers that need fileId to fetch records)
|
||
files: files.map(f => ({ fileId: f._id, name: f.name })),
|
||
// Pilot traceability
|
||
sessionPilotName: meta.operator ?? null, // name as recorded in the data file
|
||
pilotId: job.operator?._id ?? null,
|
||
pilotName: job.operator?.name ?? null, // assigned pilot on the job record
|
||
aircraftName: job.vehicle?.name ?? null,
|
||
aircraftTailNumber: job.vehicle?.tailNumber ?? null,
|
||
assignedDate: assign?.createdAt ?? null,
|
||
// Confirmed summary fields repeated per session for consumer convenience.
|
||
reportConfirmed: confirmedBlock.reportConfirmed,
|
||
areaSize_ha: confirmedBlock.areaSize_ha,
|
||
coverage_ha: confirmedBlock.coverage_ha,
|
||
appRateConfirmed: confirmedBlock.appRateConfirmed,
|
||
sprayVolume: confirmedBlock.sprayVolume,
|
||
volumeUnit: confirmedBlock.volumeUnit,
|
||
useActualVolume: confirmedBlock.useActualVolume,
|
||
actualVolume: confirmedBlock.actualVolume,
|
||
effectiveVolume: confirmedBlock.effectiveVolume
|
||
};
|
||
});
|
||
|
||
res.json({
|
||
jobId,
|
||
clientId: job.client?._id ?? null,
|
||
clientName: job.client?.name ?? null,
|
||
mappedArea_ha,
|
||
...confirmedBlock,
|
||
data: sessions
|
||
});
|
||
}
|
||
|
||
// ─── Raw GPS Trace Records ────────────────────────────────────────────────────
|
||
|
||
/**
|
||
* GET /api/v1/jobs/:jobId/sessions/:fileId/records
|
||
* Query: startingAfter, endingBefore, limit (default 500, max 2000), interval (seconds float)
|
||
*
|
||
* Returns cursor-paginated AppDetail records for one AppFile.
|
||
* sprayStat=3 (internal segment marker) is excluded.
|
||
* interval=N returns one record per N-second GPS time window (thinning for large exports).
|
||
*
|
||
* FE / integration note:
|
||
* - Use startingAfter cursor from previous page's last_id to paginate forward.
|
||
* - For Power BI incremental refresh: use interval=1 or interval=5 for overview.
|
||
* - For ArcGIS import: use the /export endpoint instead (full async download).
|
||
*/
|
||
async function getSessionRecords(req, res) {
|
||
const jobId = parseInt(req.params.jobId, 10);
|
||
const fileId = req.params.fileId;
|
||
|
||
if (!isFinite(jobId)) AppParamError.throw('invalid jobId');
|
||
if (!ObjectId.isValid(fileId)) AppParamError.throw('invalid fileId');
|
||
|
||
// Verify job ownership (also confirms job exists)
|
||
await ownerJob(jobId, req.uid);
|
||
|
||
// Verify the AppFile belongs to this job
|
||
const appFile = await AppFile.findOne({ _id: ObjectId(fileId), markedDelete: { $ne: true } }).lean();
|
||
if (!appFile) AppParamError.throw(Errors.NOT_FOUND);
|
||
|
||
// Verify the App (session) belongs to this job
|
||
const app = await App.findOne({ _id: appFile.appId, jobId }).lean();
|
||
if (!app) AppAuthError.throw();
|
||
|
||
const params = { ...req.query };
|
||
// Customer requirements use `after`; cursor helper expects `startingAfter`.
|
||
if (!params.startingAfter && params.after) params.startingAfter = params.after;
|
||
// Apply 2000-record hard cap for raw trace endpoint
|
||
if (!params.limit) params.limit = 500;
|
||
const requestedLimit = parseInt(params.limit);
|
||
if (requestedLimit > 2000) params.limit = 2000;
|
||
|
||
const validation = validateCursorParams(params);
|
||
if (!validation.valid) return res.status(HttpStatus.BAD_REQUEST).json({ error: validation.error });
|
||
|
||
const interval = parseInterval(params.interval);
|
||
const includeFm = params.fm === 'true'; // opt-in: ?fm=true adds Flight Master / AgDisp fields
|
||
const sessionMeta = appFile.meta || {};
|
||
|
||
// Base filter: exclude internal segment markers (sprayStat=3)
|
||
const baseFilter = { fileId: ObjectId(fileId), sprayStat: { $ne: 3 } };
|
||
|
||
const result = await paginateWithCursor(AppDetail, params, baseFilter, { cursorField: '_id' });
|
||
|
||
// Apply interval thinning if requested
|
||
let records = result.data || [];
|
||
if (interval && records.length) {
|
||
const thinned = [];
|
||
let windowStart = null;
|
||
for (const r of records) {
|
||
if (windowStart === null || (r.gpsTime - windowStart) >= interval) {
|
||
thinned.push(r);
|
||
windowStart = r.gpsTime;
|
||
}
|
||
}
|
||
records = thinned;
|
||
}
|
||
|
||
res.json({
|
||
...result,
|
||
data: records.map(d => mapDetailRecord(d, sessionMeta, app.startDateTime, includeFm))
|
||
});
|
||
}
|
||
|
||
// ─── Spray-Area GeoJSON Polygons ─────────────────────────────────────────────
|
||
|
||
/**
|
||
* GET /api/v1/jobs/:jobId/areas
|
||
*
|
||
* Returns the planned spray-area polygons as a GeoJSON FeatureCollection.
|
||
* Each Feature includes area metadata (name, planned appRate, area_ha) in properties.
|
||
*
|
||
* FE / integration note:
|
||
* - Import directly as an ArcGIS layer once the endpoint is confirmed.
|
||
* - This endpoint is gated on AMAGGI confirming the GeoJSON boundary requirement.
|
||
*/
|
||
async function getAreas(req, res) {
|
||
const jobId = parseInt(req.params.jobId, 10);
|
||
if (!isFinite(jobId)) AppParamError.throw('invalid jobId');
|
||
|
||
const job = await ownerJob(jobId, req.uid);
|
||
|
||
const appRateUnitCode = utils.isNumber(job.appRateUnit) ? job.appRateUnit : null;
|
||
const appRateUnit = appRateUnitCode != null ? utils.rateUnitString(appRateUnitCode, true) : null;
|
||
|
||
// area_ha fallback: confirmed report total → ttSprArea (total sprayable area)
|
||
const areaReportConfirmed = !!(job.rptOp && job.rptOp.coverage != null);
|
||
const fallbackAreaHa = (areaReportConfirmed
|
||
? (job.rptOp?.areaSize ?? job.ttSprArea)
|
||
: job.ttSprArea) ?? null;
|
||
|
||
const sprayFeatures = (job.sprayAreas || []).map(area => ({
|
||
type: 'Feature',
|
||
properties: {
|
||
name: area.properties?.name ?? null,
|
||
appRate: utils.isNumber(area.properties?.appRate) ? area.properties.appRate : (job.appRate ?? null),
|
||
appRateUnit,
|
||
appRateUnitCode,
|
||
area_ha: area.properties?.area ?? fallbackAreaHa,
|
||
type: ExportAreaTypes.AREA
|
||
},
|
||
geometry: area.geometry
|
||
}));
|
||
|
||
const xclFeatures = (job.excludedAreas || []).map(area => ({
|
||
type: 'Feature',
|
||
properties: {
|
||
name: area.properties?.name ?? null,
|
||
type: ExportAreaTypes.EXCLUDED
|
||
},
|
||
geometry: area.geometry
|
||
}));
|
||
|
||
const features = sprayFeatures.concat(xclFeatures);
|
||
|
||
res.json({
|
||
type: 'FeatureCollection',
|
||
jobId,
|
||
features
|
||
});
|
||
}
|
||
|
||
module.exports = { getSessions, getSessionRecords, getAreas };
|