agnav/agmission
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
feature/data-export-api
agmission/Development/server/routes/api_pub.js
Devin Major df31b2080d
All checks were successful
Server Tests / Mocha – Unit & Utility Tests (push) Successful in 42s
Details
-(#3013) Data Export - Implement Data Export API BE (Cont.) 
+ Added public data export API enhancements, tests, and customer documentation
  + Extended /api/v1 data export endpoints with richer session, records, area, and async export output
  + Added confirmed/fallback report values, client metadata, mapped area, over-spray, volume/apprate (string) units, and weather blocks
  + Normalized flowController to "No FC" and align record field names with playback output
  + Converted record wind speed output to knots, add Fligh Mater only record/export fields behind fm=true, and persist fm on export jobs
  + Added export status/area constants, HTTP 202 support, route-level API docs, and per-account export rate limiting support
  + Added comprehensive endpoint, format, and verification test coverage plus test-suite README
  + Added customer-facing data export design, integration, rate-limit, and documentation index guides
  + Updated README/DLQ docs and related documentation links to current HTTPS dashboard paths
2026-04-24 09:05:55 -04:00

353 lines
18 KiB
JavaScript
Raw Permalink Blame History

'use strict';
/**
* Public Data Export API routes — mounted at /api/v1/
* All routes authenticated via checkApiKey (X-API-Key header).
*
* ─── Integration guide ───────────────────────────────────────────────────────
*
* Session summary:
* GET /api/v1/jobs/:jobId/sessions
* → Returns one record per uploaded file for the job.
* reportConfirmed: false when applicator has not yet confirmed values in Report Settings.
* Re-fetch when your data warehouse detects this field changed.
*
* Raw GPS trace (paginated):
* GET /api/v1/jobs/:jobId/sessions/:fileId/records
* Query: startingAfter=<cursor>, limit=<n≤2000>, interval=<seconds>
* → Use interval=1 or interval=5 for lighter Power BI queries.
* → Use the /export endpoint instead for full bulk loads.
*
* Spray-area polygons:
* GET /api/v1/jobs/:jobId/areas
* → GeoJSON FeatureCollection of planned spray-area polygons.
*
* Async bulk export:
* POST /api/v1/jobs/:jobId/export body: { format: 'csv'|'geojson', interval?: number }
* GET /api/v1/exports/:exportId poll for { status, downloadUrl }
* GET /api/v1/exports/:exportId/download stream file
*
* ─────────────────────────────────────────────────────────────────────────────
*
* FE integration notes:
* - The key management UI (create/list/revoke keys) lives at /api/keys — see routes/api_keys.js.
* - The API key is supplied as the X-API-Key request header, NOT Authorization Bearer.
* - For Power BI: use paginated records endpoint with startingAfter cursor for incremental refresh.
* - For ArcGIS / daily batch: use the export endpoint — POST once, poll, then download CSV/GeoJSON.
*/
module.exports = function (app) {
const router = require('express').Router();
const rateLimit = require('express-rate-limit');
const { checkApiKey } = require('../middlewares/app_validator');
const pubCtl = require('../controllers/api_pub');
const exportCtl = require('../controllers/api_export');
const env = require('../helpers/env');
// Apply API key auth to all /api/v1/ routes
router.use(checkApiKey);
/**
* Per-account rate limiter — applied after checkApiKey so req.uid is available.
*
* ── Configuration ──────────────────────────────────────────────────────
* Keyed on account ID (not IP) to prevent one API key from flooding the export pipeline.
*
* Environment variables (see helpers/env.js):
* EXPORT_RATE_LIMIT_MAX: 20 — Max export triggers per account per window
* EXPORT_RATE_LIMIT_WINDOW_MINS: 60 — Time window in minutes
*
* ── Behavior ────────────────────────────────────────────────────────────
* Default: 20 exports per 60 minutes = 1 export every 3 minutes
*
* When exceeded:
* HTTP 429 Too Many Requests
* Response headers: RateLimit-Limit, RateLimit-Remaining, RateLimit-Reset, Retry-After
*
* ── Deduplication ───────────────────────────────────────────────────────
* Rate limit is NOT consumed if the request is deduplicated:
* - Existing ready export (same job/format/units) → return cached
* - Existing in-progress export (within EXPORT_DEDUP_MINS) → return existing
*
* ── Documentation ───────────────────────────────────────────────────────
* See docs/DATA_EXPORT_API_RATE_LIMITING.md for:
* - Detailed examples and scenarios
* - Best practices for batch workflows
* - Handling 429 responses
* - Integration guide for customers
*
* See docs/DATA_EXPORT_CUSTOMER_INTEGRATION_GUIDE.md for:
* - Full API documentation (all 6 endpoints)
* - Use cases and code examples
* - Error handling
*/
const exportAccountLimiter = rateLimit({
windowMs: env.EXPORT_RATE_LIMIT_WINDOW_MINS * 60 * 1000,
max: env.EXPORT_RATE_LIMIT_MAX,
keyGenerator: req => String(req.uid),
skipFailedRequests: true,
standardHeaders: true,
legacyHeaders: false,
message: { error: 'Export rate limit exceeded. Please wait before requesting another export.' }
});
// ── Session summary ──────────────────────────────────────────────────────
/**
* @api {get} /api/v1/jobs/:jobId/sessions Get Session Summary
* @apiVersion 1.0.0
* @apiName GetSessions
* @apiGroup Sessions
* @apiDescription Returns aggregated spray application data (coverage, timing, pilot, aircraft)
* from one or more flight files. Each session represents one uploaded log file.
*
* @apiParam {Number} jobId Job ID
*
* @apiHeader {String} X-API-Key API key (e.g., ak_test_xxx)
*
* @apiSuccess (200) {Number} jobId Job identifier
* @apiSuccess (200) {Boolean} reportConfirmed True if applicator confirmed values in Report Settings
* @apiSuccess (200) {Number} areaSize_ha Planned spray area (hectares)
* @apiSuccess (200) {Number} coverage_ha Actual coverage (hectares)
* @apiSuccess (200) {Number} appRate Application rate (material per area)
* @apiSuccess (200) {String} appRateUnit Rate unit string (e.g., 'lit/ha', 'oz/ac')
* @apiSuccess (200) {String} volumeUnit Material unit string (e.g., 'lit', 'oz', 'kg', 'lbs')
* @apiSuccess (200) {Number} sprayVolume Total material sprayed
* @apiSuccess (200) {Number} mappedArea_ha Actual mapped spray area (may differ from areaSize_ha)
* @apiSuccess (200) {Number} overSprayedPct Over-spray percentage (mapped area vs. planned)
* @apiSuccess (200) {Object[]} data Array of session records (one per file)
* @apiSuccess (200) {String} data.sessionId Session/file ID
* @apiSuccess (200) {String} data.fileName Log file name
* @apiSuccess (200) {String} data.startDateTime ISO 8601 start time
* @apiSuccess (200) {String} data.endDateTime ISO 8601 end time
* @apiSuccess (200) {Number} data.totalFlightTime_s Total flight time (seconds)
* @apiSuccess (200) {Number} data.totalSprayTime_s Total spray time (seconds)
* @apiSuccess (200) {Number} data.totalTurnTime_s Total turn time (seconds)
* @apiSuccess (200) {Number} data.totalSprayed_ha Area sprayed (hectares)
* @apiSuccess (200) {Number} data.totalSprayMat Total material sprayed
* @apiSuccess (200) {String} data.totalSprayMatUnit Material unit (e.g., 'lit', 'gal', 'kg')
* @apiSuccess (200) {Number} data.avgSpraySpeed_ms Average spray speed (m/s)
* @apiSuccess (200) {Number} data.appRate Application rate
* @apiSuccess (200) {String} data.appRateUnit Rate unit (e.g., 'lit/ha')
* @apiSuccess (200) {String} data.pilotName Pilot name
* @apiSuccess (200) {String} data.aircraftName Aircraft type
* @apiSuccess (200) {String} data.aircraftTailNumber Aircraft tail number
* @apiSuccess (200) {Boolean} data.reportConfirmed Report confirmation status
*
* @apiError (401) {Object} error Not authorized (missing/invalid X-API-Key)
* @apiError (404) {Object} error Job not found
*
* @apiExample {curl} Example Usage:
* curl -X GET https://api.agmission.com/api/v1/jobs/12345/sessions \
* -H "X-API-Key: ak_test_..."
*
* @apiSeeAlso GET /api/v1/jobs/:jobId/sessions/:fileId/records, GET /api/v1/jobs/:jobId/areas, POST /api/v1/jobs/:jobId/export
*/
router.get('/jobs/:jobId/sessions', pubCtl.getSessions);
// ── Raw GPS trace records ────────────────────────────────────────────────
/**
* @api {get} /api/v1/jobs/:jobId/sessions/:fileId/records Get Session Records (Paginated)
* @apiVersion 1.0.0
* @apiName GetSessionRecords
* @apiGroup Sessions
* @apiDescription Returns raw GPS trace points with cursor-based pagination.
* Use `interval` parameter for GPS thinning (e.g., every 5 seconds).
* Recommended for incremental Power BI refresh and lightweight queries.
*
* @apiParam {Number} jobId Job ID
* @apiParam {String} fileId Session/file ID
* @apiParam {String} [startingAfter] Cursor for next page
* @apiParam {Number} [limit=500] Records per page (max 2000)
* @apiParam {Number} [interval] GPS thinning interval (seconds, float)
*
* @apiHeader {String} X-API-Key API key
*
* @apiSuccess (200) {Object[]} data Array of GPS records
* @apiSuccess (200) {String} data.timeUtc ISO 8601 timestamp
* @apiSuccess (200) {Number} data.lat Latitude (decimal degrees)
* @apiSuccess (200) {Number} data.lon Longitude (decimal degrees)
* @apiSuccess (200) {Number} data.alt Altitude (meters)
* @apiSuccess (200) {Number} data.grSpeed Ground speed (m/s)
* @apiSuccess (200) {Number} data.heading Heading (degrees)
* @apiSuccess (200) {Number} data.sprayStat Spray status (0=off, 1=on)
* @apiSuccess (200) {Number} data.flowRateApplied Flow rate applied (L/min)
* @apiSuccess (200) {Number} data.appRateApplied Application rate applied (L/ha)
* @apiSuccess (200) {Number} data.windSpeed_ms Wind speed (m/s)
* @apiSuccess (200) {Number} data.windDir_deg Wind direction (0-360°)
* @apiSuccess (200) {Number} data.temp_c Temperature (°C)
* @apiSuccess (200) {Number} data.humidity_pct Humidity (%)
* @apiSuccess (200) {Boolean} hasMore True if more records available
* @apiSuccess (200) {String} [nextCursor] Cursor for next page
*
* @apiError (401) {Object} error Not authorized
* @apiError (404) {Object} error Session/file not found
*
* @apiExample {curl} Fetch 500 records, every 5 seconds:
* curl "https://api.agmission.com/api/v1/jobs/12345/sessions/507f1f77.../records?limit=500&interval=5" \
* -H "X-API-Key: ak_test_..."
*
* @apiExample {curl} Fetch next page:
* curl "https://api.agmission.com/api/v1/jobs/12345/sessions/507f1f77.../records?startingAfter=507f191e810c19729de8605f" \
* -H "X-API-Key: ak_test_..."
*/
router.get('/jobs/:jobId/sessions/:fileId/records', pubCtl.getSessionRecords);
// ── Spray-area GeoJSON polygons ──────────────────────────────────────────
/**
* @api {get} /api/v1/jobs/:jobId/areas Get Spray Areas (GeoJSON)
* @apiVersion 1.0.0
* @apiName GetAreas
* @apiGroup Areas
* @apiDescription Returns GeoJSON FeatureCollection of planned spray zones and exclusion boundaries.
* Features include spray areas (`type: "area"`) and no-spray zones (`type: "xcl"`).
*
* @apiParam {Number} jobId Job ID
*
* @apiHeader {String} X-API-Key API key
*
* @apiSuccess (200) {String} type GeoJSON type ("FeatureCollection")
* @apiSuccess (200) {Number} jobId Associated job ID
* @apiSuccess (200) {Number} mappedArea_ha Total mapped area (hectares)
* @apiSuccess (200) {Object[]} features Array of GeoJSON features
* @apiSuccess (200) {String} features.type GeoJSON type ("Feature")
* @apiSuccess (200) {Object} features.properties Feature properties
* @apiSuccess (200) {String} features.properties.name Feature name
* @apiSuccess (200) {String} features.properties.type Feature type ("area" or "xcl")
* @apiSuccess (200) {Number} [features.properties.area_ha] Area in hectares (for type="area")
* @apiSuccess (200) {Number} [features.properties.appRate] Application rate (for type="area")
* @apiSuccess (200) {String} [features.properties.appRateUnit] Rate unit (for type="area", e.g., 'lit/ha')
* @apiSuccess (200) {Object} features.geometry GeoJSON geometry (Polygon)
* @apiSuccess (200) {String} features.geometry.type Geometry type ("Polygon")
* @apiSuccess (200) {Number[][][]} features.geometry.coordinates Polygon coordinates
*
* @apiError (401) {Object} error Not authorized
* @apiError (404) {Object} error Job not found
*
* @apiExample {curl} Example Usage:
* curl -X GET https://api.agmission.com/api/v1/jobs/12345/areas \
* -H "X-API-Key: ak_test_..."
*
* @apiSeeAlso GET /api/v1/jobs/:jobId/sessions
*/
router.get('/jobs/:jobId/areas', pubCtl.getAreas);
// ── Async export ─────────────────────────────────────────────────────────
/**
* @api {post} /api/v1/jobs/:jobId/export Trigger Async Export
* @apiVersion 1.0.0
* @apiName TriggerExport
* @apiGroup Exports
* @apiDescription Initiates async generation of a bulk CSV or GeoJSON export.
* Returns immediately with exportId; use GET /exports/:exportId to poll for status.
*
* Request deduplication: Identical requests within 5 minutes reuse existing export (no rate limit consumed).
* Per-account rate limit: 20 exports per 60 minutes (configurable).
*
* @apiParam {Number} jobId Job ID
*
* @apiHeader {String} X-API-Key API key
* @apiHeader {String} Content-Type application/json
*
* @apiBody {String} format Export format: "csv" or "geojson"
* @apiBody {String} [units="metric"] Unit system: "metric" (default) or "us"
* @apiBody {Number} [interval] GPS thinning interval in seconds (float, optional)
*
* @apiSuccess (202) {String} exportId Export job ID
* @apiSuccess (202) {String} status Export status ("pending")
* @apiSuccess (202) {String} format Export format
* @apiSuccess (202) {String} units Unit system
* @apiSuccess (202) {String} createdAt ISO 8601 creation timestamp
*
* @apiSuccess (200) {String} exportId Export job ID (reused from cache)
* @apiSuccess (200) {String} status Export status ("ready" or "pending")
* @apiSuccess (200) {Boolean} reused=true Indicates request was deduplicated
* @apiSuccess (200) {String} [downloadUrl] Download URL (if status="ready")
*
* @apiError (401) {Object} error Not authorized
* @apiError (404) {Object} error Job not found
* @apiError (409) {Object} error Invalid parameters
* @apiError (429) {Object} error Rate limit exceeded
*
* @apiHeader {Number} RateLimit-Limit Maximum requests per account per window
* @apiHeader {Number} RateLimit-Remaining Requests remaining in current window
* @apiHeader {Number} RateLimit-Reset Unix timestamp of window reset
* @apiHeader {Number} Retry-After Seconds to wait before retrying (on 429 only)
*
* @apiExample {curl} Trigger CSV export:
* curl -X POST https://api.agmission.com/api/v1/jobs/12345/export \
* -H "X-API-Key: ak_test_..." \
* -H "Content-Type: application/json" \
* -d '{"format":"csv","units":"metric"}'
*
* @apiSeeAlso GET /api/v1/exports/:exportId, GET /api/v1/exports/:exportId/download
*/
router.post('/jobs/:jobId/export', exportAccountLimiter, exportCtl.triggerExport);
/**
* @api {get} /api/v1/exports/:exportId Get Export Status
* @apiVersion 1.0.0
* @apiName GetExportStatus
* @apiGroup Exports
* @apiDescription Polls the status of an async export job.
* Keep polling until status is "ready", then download the file.
*
* @apiParam {String} exportId Export job ID (returned by POST /export)
*
* @apiHeader {String} X-API-Key API key
*
* @apiSuccess (200) {String} exportId Export job ID
* @apiSuccess (200) {String} status Export status: "pending", "processing", "ready", or "error"
* @apiSuccess (200) {String} format Export format ("csv" or "geojson")
* @apiSuccess (200) {String} units Unit system
* @apiSuccess (200) {String} createdAt ISO 8601 creation timestamp
* @apiSuccess (200) {String} [expiresAt] ISO 8601 expiry timestamp (file available until this time)
* @apiSuccess (200) {String} [downloadUrl] Download endpoint URL (when status="ready")
* @apiSuccess (200) {String} [error] Error message (when status="error")
*
* @apiError (401) {Object} error Not authorized
* @apiError (404) {Object} error Export not found
*
* @apiExample {curl} Poll export status:
* curl -X GET https://api.agmission.com/api/v1/exports/66f4a8c1.../status \
* -H "X-API-Key: ak_test_..."
*
* @apiSeeAlso POST /api/v1/jobs/:jobId/export, GET /api/v1/exports/:exportId/download
*/
router.get('/exports/:exportId', exportCtl.getExportStatus);
/**
* @api {get} /api/v1/exports/:exportId/download Download Export File
* @apiVersion 1.0.0
* @apiName DownloadExport
* @apiGroup Exports
* @apiDescription Streams the ready export file (CSV or GeoJSON).
* Must call GET /exports/:exportId first and wait for status="ready".
*
* Files remain available for download until expiresAt (default 24 hours after ready).
* Can be downloaded multiple times before expiry.
*
* @apiParam {String} exportId Export job ID
*
* @apiHeader {String} X-API-Key API key
*
* @apiSuccess (200) {Binary} file File stream (CSV or GeoJSON)
* @apiSuccessExample {curl} Response Headers:
* HTTP/1.1 200 OK
* Content-Type: text/csv
* Content-Disposition: attachment; filename="export_job12345_66f4a8c1.csv"
* Content-Length: 1048576
*
* @apiError (401) {Object} error Not authorized
* @apiError (404) {Object} error Export not found or expired
*
* @apiExample {curl} Download export:
* curl -X GET https://api.agmission.com/api/v1/exports/66f4a8c1.../download \
* -H "X-API-Key: ak_test_..." \
* -o export_job12345.csv
*
* @apiSeeAlso GET /api/v1/exports/:exportId
*/
router.get('/exports/:exportId/download', exportCtl.downloadExport);
app.use('/api/v1', router);
};
Reference in New Issue View Git Blame Copy Permalink