411 lines
13 KiB
JavaScript
411 lines
13 KiB
JavaScript
/**
|
|
* Export Format Validation Test — CSV and JSON integrity
|
|
* Verifies that exported formats match requirements and values are accurate
|
|
*/
|
|
|
|
const path = require('path');
|
|
const crypto = require('crypto');
|
|
|
|
const args = process.argv.slice(2);
|
|
let envFile = './environment.env';
|
|
for (let i = 0; i < args.length; i++) {
|
|
if (args[i] === '--env' && args[i + 1]) {
|
|
envFile = args[i + 1];
|
|
i++;
|
|
}
|
|
}
|
|
|
|
require('dotenv').config({ path: path.resolve(process.cwd(), envFile) });
|
|
|
|
const { expect } = require('chai');
|
|
const axios = require('axios');
|
|
const bcrypt = require('bcryptjs');
|
|
const https = require('https');
|
|
const { ObjectId } = require('mongodb');
|
|
const moment = require('moment');
|
|
|
|
const { Job, App, AppFile, AppDetail, User, Pilot, Vehicle } = require('../model');
|
|
const ApiKey = require('../model/api_key');
|
|
const ExportJob = require('../model/export_job');
|
|
const { ApiKeyServices, ExportUnits } = require('../helpers/constants');
|
|
const dbConnect = require('../helpers/db/connect');
|
|
|
|
const BASE_URL = `https://localhost:${process.env.AGM_PORT || process.env.PORT || 4100}`;
|
|
const httpClient = axios.create({
|
|
baseURL: BASE_URL,
|
|
httpsAgent: new https.Agent({ rejectUnauthorized: false })
|
|
});
|
|
|
|
describe('Data Export API - Format Validation', function() {
|
|
this.timeout(120000);
|
|
|
|
let testUserId, testJobId, testAppId, testFileId, testApiKey, testKeyId;
|
|
let testPilotId, testVehicleId, testClientId;
|
|
|
|
before(async function() {
|
|
console.log('\n🔧 Connecting to database...');
|
|
await dbConnect();
|
|
console.log('✅ Database connected\n');
|
|
// Sweep orphan records left by previously aborted/crashed test runs
|
|
try {
|
|
const orphanUsers = await User.find({ username: /^fmt_user_\d+$/ }).select('_id').lean();
|
|
if (orphanUsers.length) {
|
|
const ids = orphanUsers.map(u => u._id);
|
|
await ApiKey.deleteMany({ owner: { $in: ids } });
|
|
await User.deleteMany({ _id: { $in: ids } });
|
|
}
|
|
} catch (err) {
|
|
console.warn('Orphan sweep warning:', err.message);
|
|
}
|
|
|
|
// Create users
|
|
const user = new User({
|
|
username: `fmt_user_${Date.now()}`,
|
|
email: `fmt_${Date.now()}@test.com`,
|
|
passwordHash: 'hash',
|
|
status: 'active',
|
|
role: 'admin',
|
|
kind: 'REGULAR'
|
|
});
|
|
await user.save();
|
|
testUserId = user._id;
|
|
|
|
const clientUser = new User({
|
|
username: `fmt_client_${Date.now()}`,
|
|
email: `fmt_client_${Date.now()}@test.com`,
|
|
passwordHash: 'hash',
|
|
status: '3',
|
|
role: 'client',
|
|
kind: 'REGULAR'
|
|
});
|
|
await clientUser.save();
|
|
testClientId = clientUser._id;
|
|
|
|
// Create pilot and vehicle
|
|
const pilot = new Pilot({
|
|
name: `Pilot_${Date.now()}`,
|
|
licenseNum: 'FMT001',
|
|
active: true
|
|
});
|
|
await pilot.save();
|
|
testPilotId = pilot._id;
|
|
|
|
const vehicle = new Vehicle({
|
|
name: `Aircraft_${Date.now()}`,
|
|
tailNumber: `N${Math.floor(Math.random() * 100000)}`,
|
|
active: true
|
|
});
|
|
await vehicle.save();
|
|
testVehicleId = vehicle._id;
|
|
|
|
// Create Job
|
|
const job = new Job({
|
|
_id: Math.floor(Math.random() * 900000) + 100000,
|
|
name: `FmtJob_${Date.now()}`,
|
|
orderNumber: String(Math.floor(Math.random() * 10000)),
|
|
byPuid: testUserId,
|
|
client: testClientId,
|
|
operator: testPilotId,
|
|
vehicle: testVehicleId,
|
|
status: 0,
|
|
swathWidth: 12.5,
|
|
measureUnit: false,
|
|
sprayAreas: [{
|
|
properties: { name: 'TestArea', appRate: 50, area: 10 },
|
|
geometry: { type: 'Polygon', coordinates: [[[-50, -30], [-50, -20], [-40, -20], [-40, -30], [-50, -30]]] }
|
|
}]
|
|
});
|
|
await job.save();
|
|
testJobId = job._id;
|
|
|
|
// Create App
|
|
const app = new App({
|
|
jobId: testJobId,
|
|
fileName: `fmt_session_${Date.now()}.log`,
|
|
fileSize: 2048,
|
|
status: 3,
|
|
totalFlightTime: 3600,
|
|
totalSprayTime: 2400,
|
|
totalTurnTime: 1200,
|
|
totalSprayed: 5.0,
|
|
totalSprayMat: 250,
|
|
totalSprayMatUnit: 1,
|
|
avgSpraySpeed: 40,
|
|
markedDelete: false
|
|
});
|
|
await app.save();
|
|
testAppId = app._id;
|
|
|
|
// Create AppFile
|
|
const appFile = new AppFile({
|
|
appId: testAppId,
|
|
name: `fmt_file_${Date.now()}.log`,
|
|
agn: 1,
|
|
meta: {
|
|
areaOrZone: 'Test Area',
|
|
sprCoverage: [100, 5.0],
|
|
appRate: 50,
|
|
appRateUnitStr: 'L/ha',
|
|
fcName: 'Controller1',
|
|
sprOnLag: 0.5,
|
|
sprOffLag: 0.3,
|
|
pulsesPerLit: 10,
|
|
operator: 'Test Pilot',
|
|
matType: 'wet'
|
|
}
|
|
});
|
|
await appFile.save();
|
|
testFileId = appFile._id;
|
|
|
|
// Create GPS records with mixed sprayStat
|
|
const baseTime = moment().unix();
|
|
const records = [];
|
|
for (let i = 0; i < 15; i++) {
|
|
records.push({
|
|
fileId: testFileId,
|
|
gpsTime: baseTime + (i * 10),
|
|
lat: 40.71 + (i * 0.0001),
|
|
lon: -74.00 + (i * 0.0001),
|
|
utmX: 583960 + (i * 10),
|
|
utmY: 4506721 + (i * 10),
|
|
alt: 100 + (i * 2),
|
|
grSpeed: 35 + (i * 0.5),
|
|
head: 45,
|
|
xTrack: 0.5,
|
|
llnum: 1,
|
|
stdHdop: 0.8,
|
|
satsIn: 12,
|
|
tslu: 0,
|
|
calcodeFreq: 0,
|
|
sprayStat: i === 0 ? 3 : (i % 2 === 0 ? 0 : 1),
|
|
lminApp: i % 2 === 0 ? 0 : 45,
|
|
lminReq: 45,
|
|
lhaReq: 50,
|
|
swath: 12,
|
|
psi: 2.5,
|
|
rpm: 1800,
|
|
windSpd: 2.5,
|
|
windDir: 180,
|
|
temp: 22,
|
|
humid: 65
|
|
});
|
|
}
|
|
await AppDetail.insertMany(records);
|
|
|
|
// Create API key
|
|
const plainApiKey = crypto.randomBytes(32).toString('hex');
|
|
const prefix = plainApiKey.substring(0, 8);
|
|
const keyHash = await bcrypt.hash(plainApiKey, 10);
|
|
|
|
const apiKey = new ApiKey({
|
|
owner: testUserId,
|
|
label: `key_${Date.now()}`,
|
|
prefix,
|
|
keyHash,
|
|
service: ApiKeyServices.DATA_EXPORT,
|
|
active: true
|
|
});
|
|
await apiKey.save();
|
|
testKeyId = apiKey._id;
|
|
testApiKey = plainApiKey;
|
|
|
|
console.log('✅ Test data ready\n');
|
|
});
|
|
|
|
after(async function() {
|
|
console.log('\n🧹 Cleaning up...');
|
|
try {
|
|
if (testFileId) await AppDetail.deleteMany({ fileId: testFileId });
|
|
if (testFileId) await AppFile.deleteOne({ _id: testFileId });
|
|
if (testAppId) await App.deleteOne({ _id: testAppId });
|
|
if (testJobId) await ExportJob.deleteMany({ jobId: testJobId });
|
|
if (testJobId) await Job.deleteOne({ _id: testJobId });
|
|
if (testKeyId) await ApiKey.deleteOne({ _id: testKeyId });
|
|
if (testUserId) await User.deleteOne({ _id: testUserId });
|
|
if (testClientId) await User.deleteOne({ _id: testClientId });
|
|
if (testPilotId) await Pilot.deleteOne({ _id: testPilotId });
|
|
if (testVehicleId) await Vehicle.deleteOne({ _id: testVehicleId });
|
|
console.log('✅ Cleaned up\n');
|
|
} catch (err) {
|
|
console.error('Cleanup error:', err.message);
|
|
}
|
|
});
|
|
|
|
it('Format: CSV with metric units', async function() {
|
|
const res = await httpClient.post(
|
|
`/api/v1/jobs/${testJobId}/export`,
|
|
{ format: 'csv', interval: null, units: ExportUnits.METRIC },
|
|
{ headers: { 'X-API-Key': testApiKey } }
|
|
);
|
|
|
|
expect(res.status).to.equal(202);
|
|
this.metricExportId = res.data.exportId;
|
|
|
|
// Poll for ready
|
|
let status = 'pending';
|
|
for (let i = 0; i < 30 && ['pending', 'processing'].includes(status); i++) {
|
|
const statusRes = await httpClient.get(
|
|
`/api/v1/exports/${this.metricExportId}`,
|
|
{ headers: { 'X-API-Key': testApiKey } }
|
|
);
|
|
status = statusRes.data.status;
|
|
if (['pending', 'processing'].includes(status)) await new Promise(r => setTimeout(r, 1000));
|
|
}
|
|
|
|
const downloadRes = await httpClient.get(
|
|
`/api/v1/exports/${this.metricExportId}/download`,
|
|
{ headers: { 'X-API-Key': testApiKey }, responseType: 'text' }
|
|
);
|
|
|
|
const lines = downloadRes.data.trim().split('\n');
|
|
const headers = lines[0].split(',');
|
|
|
|
// Verify metric headers
|
|
expect(headers).to.include('alt_m', 'Expected metric altitude header');
|
|
expect(headers).to.include('groundSpeed_ms', 'Expected metric speed header');
|
|
expect(headers).to.not.include('alt_ft', 'Should not have US unit headers');
|
|
expect(headers).to.not.include('groundSpeed_mph', 'Should not have US unit headers');
|
|
|
|
// Verify data rows (all records included)
|
|
const dataLines = lines.slice(1).filter(l => l.trim());
|
|
expect(dataLines.length).to.equal(15, 'Should have 15 data rows (markers included)');
|
|
|
|
console.log(` ✅ CSV metric: ${headers.length} columns, ${dataLines.length} data rows`);
|
|
});
|
|
|
|
it('Format: CSV with US units', async function() {
|
|
const res = await httpClient.post(
|
|
`/api/v1/jobs/${testJobId}/export`,
|
|
{ format: 'csv', interval: null, units: ExportUnits.US },
|
|
{ headers: { 'X-API-Key': testApiKey } }
|
|
);
|
|
|
|
expect(res.status).to.equal(202);
|
|
this.usExportId = res.data.exportId;
|
|
|
|
// Poll for ready
|
|
let status = 'pending';
|
|
for (let i = 0; i < 30 && ['pending', 'processing'].includes(status); i++) {
|
|
const statusRes = await httpClient.get(
|
|
`/api/v1/exports/${this.usExportId}`,
|
|
{ headers: { 'X-API-Key': testApiKey } }
|
|
);
|
|
status = statusRes.data.status;
|
|
if (['pending', 'processing'].includes(status)) await new Promise(r => setTimeout(r, 1000));
|
|
}
|
|
|
|
const downloadRes = await httpClient.get(
|
|
`/api/v1/exports/${this.usExportId}/download`,
|
|
{ headers: { 'X-API-Key': testApiKey }, responseType: 'text' }
|
|
);
|
|
|
|
const lines = downloadRes.data.trim().split('\n');
|
|
const headers = lines[0].split(',');
|
|
|
|
// Verify US headers
|
|
expect(headers).to.include('alt_ft', 'Expected US altitude header');
|
|
expect(headers).to.include('groundSpeed_mph', 'Expected US speed header');
|
|
expect(headers).to.not.include('alt_m', 'Should not have metric unit headers');
|
|
expect(headers).to.not.include('groundSpeed_ms', 'Should not have metric unit headers');
|
|
|
|
console.log(` ✅ CSV US units: ${headers.length} columns, metric headers replaced with US`);
|
|
});
|
|
|
|
it('Format: CSV includes spray-state markers', async function() {
|
|
const res = await httpClient.post(
|
|
`/api/v1/jobs/${testJobId}/export`,
|
|
{ format: 'csv', interval: null, units: ExportUnits.METRIC },
|
|
{ headers: { 'X-API-Key': testApiKey } }
|
|
);
|
|
|
|
this.csvExportId = res.data.exportId;
|
|
|
|
// Poll for ready
|
|
let status = 'pending';
|
|
for (let i = 0; i < 30 && ['pending', 'processing'].includes(status); i++) {
|
|
const statusRes = await httpClient.get(
|
|
`/api/v1/exports/${this.csvExportId}`,
|
|
{ headers: { 'X-API-Key': testApiKey } }
|
|
);
|
|
status = statusRes.data.status;
|
|
if (['pending', 'processing'].includes(status)) await new Promise(r => setTimeout(r, 1000));
|
|
}
|
|
|
|
const downloadRes = await httpClient.get(
|
|
`/api/v1/exports/${this.csvExportId}/download`,
|
|
{ headers: { 'X-API-Key': testApiKey }, responseType: 'text' }
|
|
);
|
|
|
|
const csv = downloadRes.data;
|
|
const lines = csv.trim().split('\n');
|
|
const headers = lines[0].split(',');
|
|
const sprayStatIndex = headers.indexOf('sprayStat');
|
|
|
|
expect(sprayStatIndex).to.be.greaterThan(-1, 'CSV should have sprayStat column');
|
|
|
|
// Check data rows for sprayStat marker presence
|
|
const dataLines = lines.slice(1).filter(l => l.trim());
|
|
let hasSprayStat3 = false;
|
|
for (const line of dataLines) {
|
|
const values = line.split(',');
|
|
const sprayStatValue = values[sprayStatIndex];
|
|
if (sprayStatValue === '3') hasSprayStat3 = true;
|
|
}
|
|
expect(hasSprayStat3).to.equal(true, 'CSV should include sprayStat=3 marker rows when present in source data');
|
|
|
|
console.log(` ✅ spray-state markers included: ${dataLines.length} rows verified`);
|
|
});
|
|
|
|
it('Format: JSON is valid array of records', async function() {
|
|
const res = await httpClient.post(
|
|
`/api/v1/jobs/${testJobId}/export`,
|
|
{ format: 'json', interval: null, units: ExportUnits.METRIC },
|
|
{ headers: { 'X-API-Key': testApiKey } }
|
|
);
|
|
|
|
expect(res.status).to.equal(202);
|
|
this.jsonExportId = res.data.exportId;
|
|
|
|
// Poll for ready
|
|
let status = 'pending';
|
|
for (let i = 0; i < 30 && ['pending', 'processing'].includes(status); i++) {
|
|
const statusRes = await httpClient.get(
|
|
`/api/v1/exports/${this.jsonExportId}`,
|
|
{ headers: { 'X-API-Key': testApiKey } }
|
|
);
|
|
status = statusRes.data.status;
|
|
if (['pending', 'processing'].includes(status)) await new Promise(r => setTimeout(r, 1000));
|
|
}
|
|
|
|
const downloadRes = await httpClient.get(
|
|
`/api/v1/exports/${this.jsonExportId}/download`,
|
|
{ headers: { 'X-API-Key': testApiKey }, responseType: 'text' }
|
|
);
|
|
|
|
let records;
|
|
try {
|
|
records = JSON.parse(downloadRes.data);
|
|
} catch (e) {
|
|
expect.fail('JSON should be valid array of records');
|
|
}
|
|
|
|
expect(records).to.be.an('array');
|
|
expect(records.length).to.equal(15, 'Should have 15 records (markers included)');
|
|
|
|
// Verify each record has expected fields
|
|
for (const record of records) {
|
|
expect(record).to.be.an('object');
|
|
expect(record).to.have.property('jobId');
|
|
expect(record).to.have.property('lat');
|
|
expect(record).to.have.property('lon');
|
|
expect(record).to.have.property('sprayStat');
|
|
expect(record).to.have.property('appRateApplied');
|
|
expect(record).to.have.property('appRateRequired');
|
|
expect(record).to.have.property('flowRateApplied');
|
|
expect(record).to.have.property('flowRateRequired');
|
|
}
|
|
|
|
console.log(` ✅ JSON valid: ${records.length} records, all have required fields`);
|
|
});
|
|
});
|