'use strict'; const debug = require('debug')('agm:file-kml'), togeojson = require('@mapbox/togeojson'), fs = require('fs-extra'), async = require('async'), unzipStream = require('unzip-stream'), // node doesn't have xml parsing or a dom. use xmldom DOMParser = require('xmldom').DOMParser, util = require('util'), domain = require('domain'), path = require('path'), turf = require('@turf/turf'), utils = require('./utils'), polyUtil = require('./poly_util'), geoUtil = require('./geo_util'), FILE = require('./file_constants'), { Errors } = require('./constants'), { AppError } = require('./app_error'); function readKmzForKmlText(filePath, cb) { let kmlContent, foundDocKML = false, returned = false; const d = domain.create(); d.on('error', err => { if (!returned) { returned = true; cb(err); } else { d.exit(); } }); d.run(function () { const zipStream = fs.createReadStream(filePath); zipStream.pipe(unzipStream.Parse()) .on('entry', entry => { if (entry.path && entry.type === 'File' && /\.kml$/i.test(entry.path)) { foundDocKML = true; let chunks = []; entry.on('error', err => { throw err }); entry.on('data', chunk => chunks.push(chunk)); entry.on('end', (e) => { if (chunks.length) { try { kmlContent = Buffer.from(Buffer.concat(chunks)).toString('utf8'); returned = true; cb(null, kmlContent); } catch (error) { throw error; } } }); } else { entry.autodrain(); } }) .on('close', function (e) { if (!foundDocKML && !returned) { // 'close' always occurs multiple times returned = true; return cb(); } }) .on('error', err => { throw err }); }); } /** * Read Kml or Kmz file for GeoJSON objects (spray, xcl, waypoint, buffer and placemark) * Supported geometry: Polygon, LineString, Point * @param filePath * @param cb error or { sprayAreas: sprayAreas, xclAreas: xclAreas, waypoints: wptItems, bufs: bufferItems, places: placeMarks } */ function readKmlKmzToGeoItems(filePath, areaTypeAndFiles, ops, cb) { let kmlContent; let sprayAreas = [], xclAreas = [], wptItems = [], bufItems = [], placeMarks = []; const fileType = areaTypeAndFiles.type; async.series([ function (callback) { if (fileType === FILE.FILE_KMZ) { kmlContent = ''; readKmzForKmlText(path.join(filePath, areaTypeAndFiles.area), (err, data) => { if (err) return callback(err); if (!utils.isBlank(data)) kmlContent = data; callback(); }); } else if (fileType === FILE.FILE_KML) { kmlContent = ''; fs.readFile(path.join(filePath, areaTypeAndFiles.area), 'utf8', (err, data) => { if (err) return callback(err); if (!utils.isBlank(data)) kmlContent = data; callback(); }); } else callback(AppError.create(Errors.NOT_KML_KMZ)); }, function (callback) { if (!kmlContent) return callback(AppError.create(Errors.BLANK_KML_KMZ)); // Parse the file itselft locally only; does not support NetworkLinks let geoJson; try { geoJson = togeojson.kml(new DOMParser().parseFromString(kmlContent, 'text/xml')); } catch (err) { debug(err, filePath); return callback(AppError.create(Errors.INVALID_AREAS_FILE)); } const features = geoJson.features || []; let res = {}, placeIndex = 0, name; for (let i = 0; i < features.length; i++) { if (!features[i] || !features[i].geometry) continue; name = utils.noLinebreaks(utils.getProp(features[i].properties, "name", "")); if (features[i].geometry.type === "GeometryCollection") { if (/SprOn|Flight Path/i.test(name)) continue; // Skip non-data items const geometries = features[i].geometry.geometries; if (utils.isEmptyArray(geometries) || sprayAreas.length === FILE.MAX_ITEM || xclAreas.length === FILE.MAX_ITEM) continue; for (let j = 0; j < geometries.length; j++) { if (geometries[j].type === "Polygon" || geometries[j].type === "LineString") { const coors = geometries[j].type === "Polygon" ? geometries[j].coordinates : [geometries[j].coordinates]; if (utils.isEmptyArray(coors)) continue; // Check for the valid coordinates format if (!geoUtil.isValidLLAr(coors[0][0], true)) return callback(AppError.create(Errors.INVALID_AREAS_FILE)); res = processPolyGon(name, coors, ops); if (sprayAreas.length < FILE.MAX_ITEM && !utils.isEmptyArray(res.areas)) sprayAreas = sprayAreas.concat(res.areas); if (xclAreas.length < FILE.MAX_ITEM && !utils.isEmptyArray(res.xcls)) xclAreas = xclAreas.concat(res.xcls); } } } if (features[i].geometry.type === "Polygon") { if (/SprOn/i.test(name) || utils.isEmptyArray(features[i].geometry.coordinates)) continue; // Skip non-area items // Check for the valid coordinates format if (!geoUtil.isValidLLAr(features[i].geometry.coordinates[0][0], true)) return callback(AppError.create(Errors.INVALID_AREAS_FILE)); res = processPolyGon(name, features[i].geometry.coordinates, ops); if (sprayAreas.length < FILE.MAX_ITEM && !utils.isEmptyArray(res.areas)) sprayAreas = sprayAreas.concat(res.areas); if (xclAreas.length < FILE.MAX_ITEM && !utils.isEmptyArray(res.xcls)) xclAreas = xclAreas.concat(res.xcls); } if (ops.onlyAreas) continue; if (features[i].geometry.type === "LineString") { if (!name || utils.isEmptyArray(features[i].geometry.coordinates)) continue; // Check for the valid coordinates format if (!geoUtil.isValidLLAr(features[i].geometry.coordinates[0], true)) return callback(AppError.create(Errors.INVALID_AREAS_FILE)); const match = name.match(/(.*(?:buffer)+.*)(?:\s*:[\s_]*(\d+(?:\.\d*)?){1}\W*_*([a-zA-Z]+))\W*$/im); if (match && match.length >= 4) { let name = match[1], width = Number(match[2]), widthMet = match[3].startsWith('f') ? turf.convertLength(width, "feet", "meters") : width; width = ops.isUS ? turf.convertLength(widthMet, "meters", "feet") : widthMet; const bufItem = polyUtil.createGeoBuffer(features[i].geometry.coordinates, name, width); if (bufItems.length < FILE.MAX_ITEM && bufItem) bufItems.push(bufItem); } } if (features[i].geometry.type === "Point") { if (utils.isEmptyArray(features[i].geometry.coordinates)) continue; // Check for the valid coordinates format if (!geoUtil.isValidLLAr(features[i].geometry.coordinates, true)) return callback(AppError.create(Errors.INVALID_AREAS_FILE)); if (/.*(wpt|waypoint)+.*/i.test(name)) { if (name && name.trim().toUpperCase() != "COR1") { const wpt = polyUtil.createGeoPoint(3, features[i].geometry.coordinates, name); if (wptItems.length < FILE.MAX_ITEM && wpt) wptItems.push(wpt); } } else { if (/agnav|^Ln/i.test(name)) continue; const place = polyUtil.createGeoPoint(4, features[i].geometry.coordinates, name ? name : `Place_${placeIndex++}`); if (placeMarks.length < FILE.MAX_ITEM && place) placeMarks.push(place); } } } callback(); }] , function (err) { if (err) return cb(err); const defaultName = utils.normalizeName(utils.bareFilename(areaTypeAndFiles.area.indexOf('_') ? areaTypeAndFiles.area.split('_')[1] : areaTypeAndFiles.area)); return cb(null, { sprayAreas: setDefaultName(sprayAreas, defaultName), xclAreas: setDefaultName(xclAreas, "XCL"), waypoints: setDefaultName(wptItems, "WPT"), bufs: setDefaultName(bufItems, "Buffer"), places: placeMarks }); }); } function processPolyGon(name, coors, ops) { if (!ops) { ops = { sprZoneColor: 'blue', appRate: 0 }; } let sprayAreas = [], xclAreas = []; const match = name.match(/^xcl.*|xcl$/i); if (match) { // Match name convention for xcl area, contains 'xcl' if (!utils.isEmptyArray(coors) && coors[0].length >= 3) { const xcls = polyUtil.createAreas(1, coors[0], name); if (!utils.isEmptyArray(xcls)) xclAreas = xclAreas.concat(xcls); } } else { const areas = polyUtil.createAreas(0, coors[0], name, ops.sprZoneColor, ops.appRate, ops.crop); if (!utils.isEmptyArray(areas)) sprayAreas = sprayAreas.concat(areas); if (coors.length > 1) { for (let k = 1; k < coors.length; k++) { const xcls = polyUtil.createAreas(1, coors[k], name); if (!utils.isEmptyArray(xcls)) xclAreas = xclAreas.concat(xcls); } } } return { areas: sprayAreas, xcls: xclAreas }; } function setDefaultName(items, startName) { if (!items || !startName) return items; let nameIndex = 0; for (let i = 0; i < items.length; i++) { if (utils.isBlank(items[i].properties.name)) { items[i].properties.name = nameIndex > 0 ? `${startName}_${nameIndex}` : startName; nameIndex++; } } return items; } module.exports = { readKmlKmzToGeoItems, readKmlKmzToGeoItemsASync: util.promisify(readKmlKmzToGeoItems) };