agmission/Development/server/tests/test_data_export_formats.js

/**
 * Export Format Validation Test — CSV and GeoJSON integrity
 * Verifies that exported formats match requirements and values are accurate
 */

const path = require('path');
const crypto = require('crypto');

const args = process.argv.slice(2);
let envFile = './environment.env';
for (let i = 0; i < args.length; i++) {
  if (args[i] === '--env' && args[i + 1]) {
    envFile = args[i + 1];
    i++;
  }
}

require('dotenv').config({ path: path.resolve(process.cwd(), envFile) });

const { expect } = require('chai');
const axios = require('axios');
const bcrypt = require('bcryptjs');
const https = require('https');
const { ObjectId } = require('mongodb');
const moment = require('moment');

const { Job, App, AppFile, AppDetail, User, Pilot, Vehicle } = require('../model');
const ApiKey = require('../model/api_key');
const { ApiKeyServices, ExportUnits } = require('../helpers/constants');
const dbConnect = require('../helpers/db/connect');

const BASE_URL = `https://localhost:${process.env.AGM_PORT || process.env.PORT || 4100}`;
const httpClient = axios.create({
  baseURL: BASE_URL,
  httpsAgent: new https.Agent({ rejectUnauthorized: false })
});

describe('Data Export API - Format Validation', function() {
  this.timeout(120000);

  let testUserId, testJobId, testAppId, testFileId, testApiKey, testKeyId;
  let testPilotId, testVehicleId, testClientId;

  before(async function() {
    console.log('\n🔧 Connecting to database...');
    await dbConnect();
    console.log('✅ Database connected\n');

    // Create users
    const user = new User({
      username: `fmt_user_${Date.now()}`,
      email: `fmt_${Date.now()}@test.com`,
      passwordHash: 'hash',
      status: 'active',
      role: 'admin',
      kind: 'REGULAR'
    });
    await user.save();
    testUserId = user._id;

    const clientUser = new User({
      username: `fmt_client_${Date.now()}`,
      email: `fmt_client_${Date.now()}@test.com`,
      passwordHash: 'hash',
      status: '3',
      role: 'client',
      kind: 'REGULAR'
    });
    await clientUser.save();
    testClientId = clientUser._id;

    // Create pilot and vehicle
    const pilot = new Pilot({
      name: `Pilot_${Date.now()}`,
      licenseNum: 'FMT001',
      active: true
    });
    await pilot.save();
    testPilotId = pilot._id;

    const vehicle = new Vehicle({
      name: `Aircraft_${Date.now()}`,
      tailNumber: `N${Math.floor(Math.random() * 100000)}`,
      active: true
    });
    await vehicle.save();
    testVehicleId = vehicle._id;

    // Create Job
    const job = new Job({
      _id: Math.floor(Math.random() * 900000) + 100000,
      name: `FmtJob_${Date.now()}`,
      orderNumber: String(Math.floor(Math.random() * 10000)),
      byPuid: testUserId,
      client: testClientId,
      operator: testPilotId,
      vehicle: testVehicleId,
      status: 0,
      swathWidth: 12.5,
      measureUnit: false,
      sprayAreas: [{
        properties: { name: 'TestArea', appRate: 50, area: 10 },
        geometry: { type: 'Polygon', coordinates: [[[-50, -30], [-50, -20], [-40, -20], [-40, -30], [-50, -30]]] }
      }]
    });
    await job.save();
    testJobId = job._id;

    // Create App
    const app = new App({
      jobId: testJobId,
      fileName: `fmt_session_${Date.now()}.log`,
      fileSize: 2048,
      status: 3,
      totalFlightTime: 3600,
      totalSprayTime: 2400,
      totalTurnTime: 1200,
      totalSprayed: 5.0,
      totalSprayMat: 250,
      totalSprayMatUnit: 1,
      avgSpraySpeed: 40,
      markedDelete: false
    });
    await app.save();
    testAppId = app._id;

    // Create AppFile
    const appFile = new AppFile({
      appId: testAppId,
      name: `fmt_file_${Date.now()}.log`,
      agn: 1,
      meta: {
        areaOrZone: 'Test Area',
        sprCoverage: [100, 5.0],
        appRate: 50,
        appRateUnitStr: 'L/ha',
        fcName: 'Controller1',
        sprOnLag: 0.5,
        sprOffLag: 0.3,
        pulsesPerLit: 10,
        operator: 'Test Pilot',
        matType: 'wet'
      }
    });
    await appFile.save();
    testFileId = appFile._id;

    // Create GPS records with mixed sprayStat
    const baseTime = moment().unix();
    const records = [];
    for (let i = 0; i < 15; i++) {
      records.push({
        fileId: testFileId,
        gpsTime: baseTime + (i * 10),
        lat: 40.71 + (i * 0.0001),
        lon: -74.00 + (i * 0.0001),
        utmX: 583960 + (i * 10),
        utmY: 4506721 + (i * 10),
        alt: 100 + (i * 2),
        grSpeed: 35 + (i * 0.5),
        head: 45,
        xTrack: 0.5,
        llnum: 1,
        stdHdop: 0.8,
        satsIn: 12,
        tslu: 0,
        calcodeFreq: 0,
        sprayStat: i === 0 ? 3 : (i % 2 === 0 ? 0 : 1),
        lminApp: i % 2 === 0 ? 0 : 45,
        lminReq: 45,
        lhaReq: 50,
        swath: 12,
        psi: 2.5,
        rpm: 1800,
        windSpd: 2.5,
        windDir: 180,
        temp: 22,
        humid: 65
      });
    }
    await AppDetail.insertMany(records);

    // Create API key
    const plainApiKey = crypto.randomBytes(32).toString('hex');
    const prefix = plainApiKey.substring(0, 8);
    const keyHash = await bcrypt.hash(plainApiKey, 10);

    const apiKey = new ApiKey({
      owner: testUserId,
      label: `key_${Date.now()}`,
      prefix,
      keyHash,
      service: ApiKeyServices.DATA_EXPORT,
      active: true
    });
    await apiKey.save();
    testKeyId = apiKey._id;
    testApiKey = plainApiKey;

    console.log('✅ Test data ready\n');
  });

  after(async function() {
    console.log('\n🧹 Cleaning up...');
    try {
      if (testFileId) await AppFile.deleteOne({ _id: testFileId });
      if (testAppId) await App.deleteOne({ _id: testAppId });
      if (testJobId) await Job.deleteOne({ _id: testJobId });
      if (testKeyId) await ApiKey.deleteOne({ _id: testKeyId });
      if (testUserId) await User.deleteOne({ _id: testUserId });
      if (testClientId) await User.deleteOne({ _id: testClientId });
      if (testPilotId) await Pilot.deleteOne({ _id: testPilotId });
      if (testVehicleId) await Vehicle.deleteOne({ _id: testVehicleId });
      console.log('✅ Cleaned up\n');
    } catch (err) {
      console.error('Cleanup error:', err.message);
    }
  });

  it('Format: CSV with metric units', async function() {
    const res = await httpClient.post(
      `/api/v1/jobs/${testJobId}/export`,
      { format: 'csv', interval: null, units: ExportUnits.METRIC },
      { headers: { 'X-API-Key': testApiKey } }
    );

    expect(res.status).to.equal(202);
    this.metricExportId = res.data.exportId;

    // Poll for ready
    let status = 'pending';
    for (let i = 0; i < 30 && ['pending', 'processing'].includes(status); i++) {
      const statusRes = await httpClient.get(
        `/api/v1/exports/${this.metricExportId}`,
        { headers: { 'X-API-Key': testApiKey } }
      );
      status = statusRes.data.status;
      if (['pending', 'processing'].includes(status)) await new Promise(r => setTimeout(r, 1000));
    }

    const downloadRes = await httpClient.get(
      `/api/v1/exports/${this.metricExportId}/download`,
      { headers: { 'X-API-Key': testApiKey }, responseType: 'text' }
    );

    const lines = downloadRes.data.trim().split('\n');
    const headers = lines[0].split(',');

    // Verify metric headers
    expect(headers).to.include('alt_m', 'Expected metric altitude header');
    expect(headers).to.include('groundSpeed_ms', 'Expected metric speed header');
    expect(headers).to.not.include('alt_ft', 'Should not have US unit headers');
    expect(headers).to.not.include('groundSpeed_mph', 'Should not have US unit headers');

    // Verify data rows (should be 14: 15 - 1 with sprayStat=3)
    const dataLines = lines.slice(1).filter(l => l.trim());
    expect(dataLines.length).to.equal(14, 'Should have 14 data rows (15 - 1 with sprayStat=3)');

    console.log(`  ✅ CSV metric: ${headers.length} columns, ${dataLines.length} data rows`);
  });

  it('Format: CSV with US units', async function() {
    const res = await httpClient.post(
      `/api/v1/jobs/${testJobId}/export`,
      { format: 'csv', interval: null, units: ExportUnits.US },
      { headers: { 'X-API-Key': testApiKey } }
    );

    expect(res.status).to.equal(202);
    this.usExportId = res.data.exportId;

    // Poll for ready
    let status = 'pending';
    for (let i = 0; i < 30 && ['pending', 'processing'].includes(status); i++) {
      const statusRes = await httpClient.get(
        `/api/v1/exports/${this.usExportId}`,
        { headers: { 'X-API-Key': testApiKey } }
      );
      status = statusRes.data.status;
      if (['pending', 'processing'].includes(status)) await new Promise(r => setTimeout(r, 1000));
    }

    const downloadRes = await httpClient.get(
      `/api/v1/exports/${this.usExportId}/download`,
      { headers: { 'X-API-Key': testApiKey }, responseType: 'text' }
    );

    const lines = downloadRes.data.trim().split('\n');
    const headers = lines[0].split(',');

    // Verify US headers
    expect(headers).to.include('alt_ft', 'Expected US altitude header');
    expect(headers).to.include('groundSpeed_mph', 'Expected US speed header');
    expect(headers).to.not.include('alt_m', 'Should not have metric unit headers');
    expect(headers).to.not.include('groundSpeed_ms', 'Should not have metric unit headers');

    console.log(`  ✅ CSV US units: ${headers.length} columns, metric headers replaced with US`);
  });

  it('Format: CSV excludes sprayStat=3', async function() {
    const res = await httpClient.post(
      `/api/v1/jobs/${testJobId}/export`,
      { format: 'csv', interval: null, units: ExportUnits.METRIC },
      { headers: { 'X-API-Key': testApiKey } }
    );

    this.csvExportId = res.data.exportId;

    // Poll for ready
    let status = 'pending';
    for (let i = 0; i < 30 && ['pending', 'processing'].includes(status); i++) {
      const statusRes = await httpClient.get(
        `/api/v1/exports/${this.csvExportId}`,
        { headers: { 'X-API-Key': testApiKey } }
      );
      status = statusRes.data.status;
      if (['pending', 'processing'].includes(status)) await new Promise(r => setTimeout(r, 1000));
    }

    const downloadRes = await httpClient.get(
      `/api/v1/exports/${this.csvExportId}/download`,
      { headers: { 'X-API-Key': testApiKey }, responseType: 'text' }
    );

    const csv = downloadRes.data;
    const lines = csv.trim().split('\n');
    const headers = lines[0].split(',');
    const sprayStatIndex = headers.indexOf('sprayStat');

    expect(sprayStatIndex).to.be.greaterThan(-1, 'CSV should have sprayStat column');

    // Check all data rows for sprayStat value
    const dataLines = lines.slice(1).filter(l => l.trim());
    for (const line of dataLines) {
      const values = line.split(',');
      const sprayStatValue = values[sprayStatIndex];
      expect(sprayStatValue).to.not.equal('3', 'No rows should have sprayStat=3');
    }

    console.log(`  ✅ sprayStat=3 excluded: ${dataLines.length} rows verified`);
  });

  it('Format: GeoJSON is valid', async function() {
    const res = await httpClient.post(
      `/api/v1/jobs/${testJobId}/export`,
      { format: 'geojson', interval: null, units: ExportUnits.METRIC },
      { headers: { 'X-API-Key': testApiKey } }
    );

    expect(res.status).to.equal(202);
    this.geoJsonExportId = res.data.exportId;

    // Poll for ready
    let status = 'pending';
    for (let i = 0; i < 30 && ['pending', 'processing'].includes(status); i++) {
      const statusRes = await httpClient.get(
        `/api/v1/exports/${this.geoJsonExportId}`,
        { headers: { 'X-API-Key': testApiKey } }
      );
      status = statusRes.data.status;
      if (['pending', 'processing'].includes(status)) await new Promise(r => setTimeout(r, 1000));
    }

    const downloadRes = await httpClient.get(
      `/api/v1/exports/${this.geoJsonExportId}/download`,
      { headers: { 'X-API-Key': testApiKey }, responseType: 'text' }
    );

    let geojson;
    try {
      geojson = JSON.parse(downloadRes.data);
    } catch (e) {
      expect.fail('GeoJSON should be valid JSON');
    }

    expect(geojson.type).to.equal('FeatureCollection');
    expect(geojson.features).to.be.an('array');
    expect(geojson.features.length).to.equal(14, 'Should have 14 features (15 - 1 with sprayStat=3)');

    // Verify each feature
    for (const feature of geojson.features) {
      expect(feature.type).to.equal('Feature');
      expect(feature.geometry.type).to.equal('Point');
      expect(feature.geometry.coordinates).to.be.an('array').with.length(3); // [lon, lat, alt]
      expect(feature.properties).to.exist;
      expect(feature.properties.sprayStat).to.not.equal(3);
    }

    console.log(`  ✅ GeoJSON valid: ${geojson.features.length} features, all have Point geometry`);
  });
});