/** * Comprehensive test for Data Export API โ€” all 6 public endpoints * Tests that API output matches database values exactly (no wrong/assumed data) */ const path = require('path'); const crypto = require('crypto'); const args = process.argv.slice(2); let envFile = './environment.env'; for (let i = 0; i < args.length; i++) { if (args[i] === '--env' && args[i + 1]) { envFile = args[i + 1]; i++; } } require('dotenv').config({ path: path.resolve(process.cwd(), envFile) }); const { expect } = require('chai'); const axios = require('axios'); const bcrypt = require('bcryptjs'); const https = require('https'); const { ObjectId } = require('mongodb'); const moment = require('moment'); const { Job, App, AppFile, AppDetail, User, Pilot, Vehicle } = require('../model'); const ApiKey = require('../model/api_key'); const ExportJob = require('../model/export_job'); const { ApiKeyServices, ExportUnits, RateUnits } = require('../helpers/constants'); const dbConnect = require('../helpers/db/connect'); const BASE_URL = `https://localhost:${process.env.AGM_PORT || process.env.PORT || 4100}`; const httpClient = axios.create({ baseURL: BASE_URL, httpsAgent: new https.Agent({ rejectUnauthorized: false }) }); describe('Data Export API - All Endpoints Verification', function() { this.timeout(120000); let testUserId, testJobId, testAppId, testFileId, testIntervalFileId, testApiKey, testKeyId; let testPilotId, testVehicleId, testClientId; before(async function() { console.log('\n๐Ÿ”ง Connecting to database...'); await dbConnect(); console.log('โœ… Database connected\n'); // Sweep orphan records left by previously aborted/crashed test runs try { const orphanUsers = await User.find({ username: /^admin_\d+$/ }).select('_id').lean(); if (orphanUsers.length) { const ids = orphanUsers.map(u => u._id); await ApiKey.deleteMany({ owner: { $in: ids } }); await User.deleteMany({ _id: { $in: ids } }); } } catch (err) { console.warn('Orphan sweep warning:', err.message); } }); after(async function() { console.log('\n๐Ÿงน Cleaning up test data...'); try { if (testIntervalFileId) await AppDetail.deleteMany({ fileId: testIntervalFileId }); if (testFileId) await AppDetail.deleteMany({ fileId: testFileId }); if (testIntervalFileId) await AppFile.deleteOne({ _id: testIntervalFileId }); if (testFileId) await AppFile.deleteOne({ _id: testFileId }); if (testAppId) await App.deleteOne({ _id: testAppId }); if (testJobId) await ExportJob.deleteMany({ jobId: testJobId }); if (testJobId) await Job.deleteOne({ _id: testJobId }); if (testKeyId) await ApiKey.deleteOne({ _id: testKeyId }); if (testUserId) await User.deleteOne({ _id: testUserId }); if (testClientId) await User.deleteOne({ _id: testClientId }); if (testPilotId) await Pilot.deleteOne({ _id: testPilotId }); if (testVehicleId) await Vehicle.deleteOne({ _id: testVehicleId }); console.log('โœ… Test data cleaned up\n'); } catch (err) { console.error('Cleanup error:', err.message); } }); it('Setup: Create admin user', async function() { const user = new User({ username: `admin_${Date.now()}`, email: `admin_${Date.now()}@test.com`, passwordHash: 'hash', status: 'active', role: 'admin', kind: 'REGULAR' }); await user.save(); testUserId = user._id; console.log(` ๐Ÿ“ Admin: ${testUserId}`); expect(testUserId).to.exist; }); it('Setup: Create client user', async function() { const user = new User({ username: `client_${Date.now()}`, email: `client_${Date.now()}@test.com`, passwordHash: 'hash', status: '3', role: 'client', kind: 'REGULAR' }); await user.save(); testClientId = user._id; console.log(` ๐Ÿ“ Client: ${testClientId}`); }); it('Setup: Create pilot', async function() { const pilot = new Pilot({ name: `Pilot_${Date.now()}`, licenseNum: 'TST001', active: true }); await pilot.save(); testPilotId = pilot._id; console.log(` ๐Ÿ“ Pilot: ${testPilotId}`); }); it('Setup: Create vehicle', async function() { const vehicle = new Vehicle({ name: `Aircraft_${Date.now()}`, tailNumber: `N${Math.floor(Math.random() * 100000)}`, active: true }); await vehicle.save(); testVehicleId = vehicle._id; console.log(` ๐Ÿ“ Vehicle: ${testVehicleId}`); }); it('Setup: Create job', async function() { const job = new Job({ _id: Math.floor(Math.random() * 900000) + 100000, name: `Job_${Date.now()}`, orderNumber: String(Math.floor(Math.random() * 10000)), byPuid: testUserId, client: testClientId, operator: testPilotId, vehicle: testVehicleId, status: 0, swathWidth: 12.5, measureUnit: false, appRate: 50, appRateUnit: RateUnits.LIT_PER_HA, ttSprArea: 10, sprayAreas: [{ properties: { name: 'Area1', appRate: 50, area: 99 }, geometry: { type: 'Polygon', coordinates: [[[-50, -30], [-50, -20], [-40, -20], [-40, -30], [-50, -30]]] } }], excludedAreas: [{ properties: { name: 'XCL1', area: 1.5 }, geometry: { type: 'Polygon', coordinates: [[[-49.8, -29.8], [-49.8, -29.6], [-49.6, -29.6], [-49.6, -29.8], [-49.8, -29.8]]] } }] }); await job.save(); testJobId = job._id; console.log(` ๐Ÿ“ Job: ${testJobId}`); }); it('Setup: Create app (session)', async function() { const app = new App({ jobId: testJobId, fileName: `session_${Date.now()}.log`, fileSize: 2048, status: 3, totalFlightTime: 3600, totalSprayTime: 2400, totalTurnTime: 1200, totalSprayed: 5.0, totalSprayMat: 250, totalSprayMatUnit: RateUnits.LIT_PER_HA, avgSpraySpeed: 40, markedDelete: false }); await app.save(); testAppId = app._id; console.log(` ๐Ÿ“ App: ${testAppId}`); }); it('Setup: Create app file', async function() { const appFile = new AppFile({ appId: testAppId, name: `file_${Date.now()}.log`, agn: 1, meta: { areaOrZone: 'Main Area', sprCoverage: [100, 5.0], appRate: 50, appRateUnitStr: 'L/ha', fcName: 'Controller1', sprOnLag: 0.5, sprOffLag: 0.3, pulsesPerLit: 10, operator: 'Test Pilot', matType: 'wet' } }); await appFile.save(); testFileId = appFile._id; console.log(` ๐Ÿ“ AppFile: ${testFileId}`); }); it('Setup: Create GPS records', async function() { const baseTime = moment().unix(); const records = []; for (let i = 0; i < 10; i++) { const sprayOnWithMissingRates = i === 1; records.push({ fileId: testFileId, gpsTime: baseTime + (i * 10), lat: 40.71 + (i * 0.0001), lon: -74.00 + (i * 0.0001), utmX: 583960 + (i * 10), utmY: 4506721 + (i * 10), alt: 100 + (i * 2), grSpeed: 35 + (i * 0.5), head: 45, xTrack: 0.5, llnum: 1, stdHdop: 0.8, satsIn: 12, tslu: 0, calcodeFreq: 0, sprayStat: i === 0 ? 3 : (i % 2 === 0 ? 0 : 1), // i=1 simulates legacy/no-FC data where spray is ON but per-point rates are missing. lminApp: sprayOnWithMissingRates ? 0 : (i % 2 === 0 ? 0 : 45), lminReq: sprayOnWithMissingRates ? 0 : 45, lhaReq: sprayOnWithMissingRates ? 0 : 50, swath: 12, psi: 2.5, rpm: 1800, windSpd: 2.5, windDir: 180, temp: 22, humid: 65 }); } await AppDetail.insertMany(records); console.log(` ๐Ÿ“ Created 10 GPS records`); }); it('Setup: Create API key', async function() { const plainApiKey = crypto.randomBytes(32).toString('hex'); const prefix = plainApiKey.substring(0, 8); const keyHash = await bcrypt.hash(plainApiKey, 10); const apiKey = new ApiKey({ owner: testUserId, label: `key_${Date.now()}`, prefix, keyHash, service: ApiKeyServices.DATA_EXPORT, active: true }); await apiKey.save(); testKeyId = apiKey._id; testApiKey = plainApiKey; console.log(` ๐Ÿ“ API Key: ${testKeyId}`); }); it('Setup: Create interval-pagination file and records', async function() { const appFile = new AppFile({ appId: testAppId, name: `file_interval_${Date.now()}.log`, agn: 2, meta: { areaOrZone: 'Interval Test Area', sprCoverage: [100, 5.0], appRate: 50, appRateUnitStr: 'L/ha', fcName: 'Controller1' } }); await appFile.save(); testIntervalFileId = appFile._id; const baseTime = moment().unix(); const records = []; for (let i = 0; i < 20; i++) { records.push({ fileId: testIntervalFileId, gpsTime: baseTime + i, lat: 41 + (i * 0.00001), lon: -73 + (i * 0.00001), utmX: 580000 + i, utmY: 4500000 + i, alt: 120, grSpeed: 40, head: 90, xTrack: 0.1, llnum: 1, stdHdop: 1, satsIn: 12, tslu: 0, calcodeFreq: 0, sprayStat: 1, lminApp: 45, lminReq: 45, lhaReq: 50, swath: 12, psi: 2.5, rpm: 1800, windSpd: 2.5, windDir: 180, temp: 22, humid: 65 }); } await AppDetail.insertMany(records); console.log(` ๐Ÿ“ Interval file: ${testIntervalFileId} (20 records @1s)`); }); // โ”€โ”€โ”€ Endpoint Tests โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ it('Endpoint: GET /api/v1/jobs/:jobId/sessions', async function() { const res = await httpClient.get(`/api/v1/jobs/${testJobId}/sessions`, { headers: { 'X-API-Key': testApiKey } }); expect(res.status).to.equal(200); expect(res.data.data).to.be.an('array'); expect(res.data.data.length).to.be.greaterThan(0); const session = res.data.data[0]; // Requirement traceability: fallback path when rptOp.coverage is not set. expect(res.data.reportConfirmed).to.equal(false); expect(res.data.areaSize_ha).to.equal(10); expect(res.data.coverage_ha).to.be.closeTo(5.0, 0.1); expect(res.data.overSprayed_pct).to.be.closeTo(-50, 0.01); expect(res.data.assignedAircraftId).to.equal(null); expect(res.data.assignedAircraftName).to.equal(null); expect(res.data.assignedAircraftTailNumber).to.equal(null); expect(res.data.mappedArea_ha).to.equal(10); expect(res.data.appRate).to.equal(50); expect(res.data.appRateUnit).to.equal('lit/ha'); // sprayVolume is planned estimate: coverage_ha ร— appRate = 5 ร— 50 expect(res.data.sprayVolume).to.be.closeTo(250, 0.1); expect(res.data.volumeUnit).to.equal('lit'); expect(res.data.useConfirmedVolume).to.equal(false); expect(res.data.actualSprayVolume).to.be.closeTo(250, 0.1); expect(res.data.confirmedActualVolume).to.equal(null); expect(res.data.effectiveVolume).to.be.closeTo(250, 0.1); expect(res.data.useCustomWeather).to.equal(false); expect(res.data.weather).to.equal(null); expect(session.totalFlightTime_s).to.equal(3600); expect(session.totalSprayTime_s).to.equal(2400); expect(session.totalTurnTime_s).to.equal(1200); expect(session.totalSprayed_ha).to.be.closeTo(5.0, 0.1); expect(session.totalSprayMat).to.be.closeTo(250, 1); expect(session.totalSprayMatUnit).to.equal('lit'); expect(session.avgSpraySpeed_ms).to.be.closeTo(40, 1); console.log(` โœ… Sessions: values match database`); }); it('Endpoint: GET /api/v1/jobs/:jobId/sessions (confirmed values)', async function() { await Job.updateOne( { _id: testJobId }, { $set: { rptOp: { areaSize: 11, coverage: 6.5, appRate: 55, useActualVol: true, actualVol: 340 }, useCustWI: true, weatherInfo: { windSpd: 12, windDir: 225, temp: 24, humid: 58 } } } ); const res = await httpClient.get(`/api/v1/jobs/${testJobId}/sessions`, { headers: { 'X-API-Key': testApiKey } }); expect(res.status).to.equal(200); expect(res.data.reportConfirmed).to.equal(true); expect(res.data.areaSize_ha).to.equal(11); expect(res.data.coverage_ha).to.equal(6.5); expect(res.data.assignedAircraftId).to.equal(null); expect(res.data.assignedAircraftName).to.equal(null); expect(res.data.assignedAircraftTailNumber).to.equal(null); expect(res.data.appRate).to.equal(55); // sprayVolume is planned estimate: coverage_ha ร— appRate = 6.5 ร— 55 expect(res.data.sprayVolume).to.be.closeTo(357.5, 0.01); expect(res.data.useConfirmedVolume).to.equal(true); expect(res.data.actualSprayVolume).to.be.closeTo(250, 0.01); expect(res.data.confirmedActualVolume).to.equal(340); expect(res.data.effectiveVolume).to.equal(340); expect(res.data.useCustomWeather).to.equal(true); expect(res.data.weather).to.deep.equal({ windSpeed_kt: 12, windDir: '225', temp_c: 24, humidity_pct: 58 }); const session = res.data.data[0]; expect(session).to.not.have.property('reportConfirmed'); expect(session).to.not.have.property('appRateConfirmed'); expect(session).to.not.have.property('useConfirmedVolume'); expect(session).to.not.have.property('actualSprayVolume'); expect(session).to.not.have.property('confirmedActualVolume'); expect(session).to.not.have.property('effectiveVolume'); console.log(' โœ… Sessions confirmed block: rptOp + weather values returned'); }); it('Endpoint: GET /api/v1/jobs/:jobId/sessions (US volume units via job.measureUnit)', async function() { await Job.updateOne({ _id: testJobId }, { $set: { measureUnit: true } }); const res = await httpClient.get(`/api/v1/jobs/${testJobId}/sessions`, { headers: { 'X-API-Key': testApiKey } }); expect(res.status).to.equal(200); expect(res.data.volumeUnit).to.equal('gal'); // sprayVolume is planned estimate: 6.5 ha ร— 55 L/ha = 357.5 L, converted to gal. expect(res.data.sprayVolume).to.be.closeTo(94.442, 0.01); // actualSprayVolume is calculated from applications: 250 L converted to gal. expect(res.data.actualSprayVolume).to.be.closeTo(66.043, 0.01); // confirmedActualVolume uses rptOp.actualVol (340 L in metric base), converted to gal. expect(res.data.confirmedActualVolume).to.be.closeTo(89.818, 0.01); expect(res.data.effectiveVolume).to.be.closeTo(89.818, 0.01); console.log(' โœ… Sessions US units: volume fields converted to gal from job.measureUnit'); // Restore to metric for downstream tests await Job.updateOne({ _id: testJobId }, { $set: { measureUnit: false } }); }); it('Endpoint: GET /api/v1/jobs/:jobId/sessions/:fileId/records', async function() { const res = await httpClient.get( `/api/v1/jobs/${testJobId}/sessions/${testFileId}/records`, { headers: { 'X-API-Key': testApiKey }, params: { limit: 100 } } ); expect(res.status).to.equal(200); expect(res.data.data).to.be.an('array'); expect(res.data.data.length).to.equal(10, 'Should have all 10 records (including spray-state markers)'); // Verify sprayStat markers are included in public records const hasSprayStat3 = res.data.data.some(r => r.sprayStat === 3); expect(hasSprayStat3).to.be.true; const first = res.data.data[0]; expect(first).to.have.property('windDir_deg'); expect(first.windDir_deg).to.equal(180); expect(first).to.not.have.property('windDir'); const fallbackRecord = res.data.data.find(r => r.sprayStat === 1 && r.flowRateApplied > 0 && r.flowRateRequired > 0 && r.appRateRequired > 0); expect(fallbackRecord, 'Expected spray-on record with fallback-applied rates').to.exist; expect(fallbackRecord.appRateApplied).to.be.greaterThan(0); console.log(` โœ… Records: ${res.data.data.length} records (markers included)`); }); it('Endpoint: GET /records paging with interval>0 stays consistent across pages', async function() { const page1 = await httpClient.get( `/api/v1/jobs/${testJobId}/sessions/${testIntervalFileId}/records`, { headers: { 'X-API-Key': testApiKey }, params: { limit: 2, interval: 5 } } ); expect(page1.status).to.equal(200); console.log(' โ„น๏ธ interval page1 gpsTime:', page1.data.data.map(r => r.gpsTime)); expect(page1.data.data).to.have.length(2); expect(page1.data.hasMore).to.equal(true); expect(page1.data.startingAfter).to.exist; const firstGps = page1.data.data[0].gpsTime; const secondGps = page1.data.data[1].gpsTime; expect(secondGps - firstGps).to.be.at.least(5); const page2 = await httpClient.get( `/api/v1/jobs/${testJobId}/sessions/${testIntervalFileId}/records`, { headers: { 'X-API-Key': testApiKey }, params: { limit: 2, interval: 5, startingAfter: page1.data.startingAfter } } ); expect(page2.status).to.equal(200); console.log(' โ„น๏ธ interval page2 gpsTime:', page2.data.data.map(r => r.gpsTime)); expect(page2.data.data).to.have.length(2); const page2FirstGps = page2.data.data[0].gpsTime; const page2SecondGps = page2.data.data[1].gpsTime; expect(page2FirstGps - secondGps).to.be.at.least(5); expect(page2SecondGps - page2FirstGps).to.be.at.least(5); const page1Ids = new Set(page1.data.data.map(r => `${r.gpsTime}`)); const overlap = page2.data.data.some(r => page1Ids.has(`${r.gpsTime}`)); expect(overlap).to.equal(false); console.log(' โœ… Interval pagination: consistent thinning across page boundaries'); }); it('Endpoint: GET /records with interval=0 returns unthinned page', async function() { const res = await httpClient.get( `/api/v1/jobs/${testJobId}/sessions/${testIntervalFileId}/records`, { headers: { 'X-API-Key': testApiKey }, params: { limit: 25, interval: 0 } } ); expect(res.status).to.equal(200); expect(res.data.data).to.have.length(20); console.log(' โœ… interval=0 behaves as no thinning'); }); it('Endpoint: GET /api/v1/jobs/:jobId/areas', async function() { await Job.updateOne( { _id: testJobId }, { $set: { sprayAreas: [{ properties: { name: 'Area1', area: 99.1267, appRate: 50.6789 }, geometry: { type: 'Polygon', coordinates: [[ [-50.123456789, -30.123456789], [-50.123456781, -30.023456781], [-50.023456781, -30.023456781], [-50.023456789, -30.123456789], [-50.123456789, -30.123456789] ]] } }], excludedAreas: [{ properties: { name: 'XCL1', area: 1.5 }, geometry: { type: 'Polygon', coordinates: [[[-49.876543219, -29.876543219], [-49.876543211, -29.676543211], [-49.676543211, -29.676543211], [-49.676543219, -29.876543219], [-49.876543219, -29.876543219]]] } }] } } ); const res = await httpClient.get(`/api/v1/jobs/${testJobId}/areas`, { headers: { 'X-API-Key': testApiKey } }); expect(res.status).to.equal(200); expect(res.data.type).to.equal('FeatureCollection'); expect(res.data.features).to.be.an('array'); expect(res.data.features.length).to.equal(2); const sprayFeature = res.data.features.find(f => f.properties.type === 'area'); const xclFeature = res.data.features.find(f => f.properties.type === 'xcl'); expect(sprayFeature).to.exist; expect(xclFeature).to.exist; expect(sprayFeature.type).to.equal('Feature'); expect(sprayFeature.geometry.type).to.equal('Polygon'); expect(sprayFeature.properties.name).to.equal('Area1'); expect(sprayFeature.properties.appRate).to.equal(50.68); expect(sprayFeature.properties.appRateUnit).to.equal('lit/ha'); expect(sprayFeature.properties.area_ha).to.equal(99.13); expect(sprayFeature.geometry.coordinates[0][0][0]).to.equal(-50.1234568); expect(sprayFeature.geometry.coordinates[0][0][1]).to.equal(-30.1234568); expect(xclFeature.type).to.equal('Feature'); expect(xclFeature.geometry.type).to.equal('Polygon'); expect(xclFeature.properties.name).to.equal('XCL1'); expect(xclFeature.properties.type).to.equal('xcl'); expect(xclFeature.properties.appRate).to.not.exist; expect(xclFeature.properties.appRateUnit).to.not.exist; expect(xclFeature.geometry.coordinates[0][0][0]).to.equal(-49.8765432); expect(xclFeature.geometry.coordinates[0][0][1]).to.equal(-29.8765432); console.log(` โœ… Areas: GeoJSON valid`); }); it('Endpoint: POST /api/v1/jobs/:jobId/export (CSV)', async function() { const res = await httpClient.post( `/api/v1/jobs/${testJobId}/export`, { format: 'csv', interval: null, units: ExportUnits.METRIC }, { headers: { 'X-API-Key': testApiKey } } ); expect(res.status).to.equal(202); expect(res.data.exportId).to.exist; expect(res.data.status).to.equal('pending'); expect(res.data.format).to.equal('csv'); expect(res.data.units).to.equal(ExportUnits.METRIC); this.exportId = res.data.exportId; console.log(` โœ… Export created: ${res.data.exportId}`); }); it('Endpoint: GET /api/v1/exports/:exportId (status)', async function() { const exportId = this.exportId; if (!exportId) this.skip(); let status = 'pending'; for (let i = 0; i < 30 && ['pending', 'processing'].includes(status); i++) { const res = await httpClient.get( `/api/v1/exports/${exportId}`, { headers: { 'X-API-Key': testApiKey } } ); expect(res.data.status).to.be.oneOf(['pending', 'processing', 'ready', 'error']); status = res.data.status; if (['pending', 'processing'].includes(status)) await new Promise(r => setTimeout(r, 1000)); } expect(status).to.be.oneOf(['ready', 'error']); console.log(` โœ… Export status: ${status}`); }); it('Endpoint: GET /api/v1/exports/:exportId/download', async function() { const exportId = this.exportId; if (!exportId) this.skip(); const res = await httpClient.get( `/api/v1/exports/${exportId}/download`, { headers: { 'X-API-Key': testApiKey }, responseType: 'text' } ); expect(res.status).to.equal(200); expect(res.data).to.be.a('string'); expect(res.data.length).to.be.greaterThan(0); const lines = res.data.trim().split('\n'); const headers = lines[0].split(','); // Verify CSV has expected columns (metric units) expect(headers).to.include('gpsTime'); expect(headers).to.include('lat'); expect(headers).to.include('lon'); expect(headers).to.include('alt_m', 'Should use metric unit'); expect(headers).to.include('groundSpeed_ms', 'Should use metric unit'); expect(headers).to.include('windDir_deg'); // Verify all rows are present in CSV (all seeded AppDetail rows for the job) const dataLines = lines.slice(1).filter(l => l.trim()); expect(dataLines.length).to.equal(30, 'CSV should have 30 data rows from seeded session files'); console.log(` โœ… CSV: ${dataLines.length} data rows, ${headers.length} columns`); }); it('Auth: Invalid key rejected', async function() { try { await httpClient.get(`/api/v1/jobs/${testJobId}/sessions`, { headers: { 'X-API-Key': 'invalid_key_12345678901234567890' } }); expect.fail('Should reject invalid key'); } catch (err) { expect(err.response.status).to.equal(401); console.log(` โœ… Invalid key rejected (401)`); } }); });