'use strict'; const debug = require('debug')('agm:file-shp'), shp = require('@mickeyjohn/shapefile'), fs = require('fs'), path = require('path'), util = require('util'), utils = require('./utils'), geoUtil = require('./geo_util'), polyUtil = require('./poly_util'), fileAgn = require('./file_storage'), FILE = require('./file_constants'), dbfstream = require('@mickeyjohn/dbfstream'), { Errors } = require('../helpers/constants'), { AppInputError, AppError } = require('./app_error'); require('string-format-js'); const readFileAsync = util.promisify(fs.readFile); const readDBF4ItemsAsync = util.promisify(readDBF4Items); function readDBF4Items(filePath, reqFields, cb) { let items = []; const dbf = dbfstream(filePath); dbf.on('header', header => { if (!utils.isEmptyArray(reqFields)) { let sameFounds = 0; for (let i = 0; i < reqFields.length; i++) { for (let j = 0; j < header.listOfFields.length; j++) { if (reqFields[i] === header.listOfFields[j].name) { sameFounds++; break; } } } if (sameFounds < reqFields.length) dbf.emit('error', 'missing_fields'); } }); dbf.on('data', (data) => { if (data) { delete data["@deleted"]; delete data["@numOfRecord"]; items.push(data); } }); dbf.on('end', () => { cb(null, items); }); dbf.on('error', err => { debug(filePath, err ? err.message : ''); }); } function readSHPToGeoItems(filePath, areaTypeAndFiles, ops, cb) { let jobItem = { sprayAreas: [], xclAreas: [], waypoints: [] }; let wptItems = [], areaItems = [], sitecodes = []; return readSHP4Items(path.join(filePath, areaTypeAndFiles.area), areaTypeAndFiles.dbf ? path.join(filePath, areaTypeAndFiles.dbf) : undefined, 5) .then(areas => { if (!utils.isEmptyArray(areas)) { if (!utils.isBlank(utils.getProp(areas[0].properties, "TIMEGPS"))) AppInputError.throw(Errors.EMPTY_SHP_FILE); // Skip the known report file areaItems = areas.slice(0, FILE.MAX_ITEM); } else AppInputError.throw(Errors.EMPTY_SHP_FILE); if (areaTypeAndFiles.sitecode) return readDBF4ItemsAsync(path.join(filePath, areaTypeAndFiles.sitecode), null); }) .then(scodes => { if (!utils.isEmptyArray(scodes)) sitecodes = scodes; }) .then(() => { // Read waypoints if (!ops.onlyAreas && areaTypeAndFiles.wpt) return readSHP4Items(path.join(filePath, areaTypeAndFiles.wpt), areaTypeAndFiles.wptDbf ? path.join(filePath, areaTypeAndFiles.wptDbf) : undefined, 1); }) .then(wpts => { if (!utils.isEmptyArray(wpts)) wptItems = wpts; if (areaTypeAndFiles.agn) return fileAgn.readAGN_PRJAsync(filePath, { area: areaTypeAndFiles.agn, type: FILE.FILE_AGN }, ops) .then(agn => { if (agn.meta && agn.meta.coorSys && agn.meta.hemisphere && agn.meta.zoneCM != undefined) { return geoUtil.getUTMProj(agn.meta.coorSys, agn.meta.hemisphere, agn.meta.zoneCM); } else return null; }) .catch(e => { throw e }); else return (areaTypeAndFiles.prj) ? readFileAsync(path.join(filePath, areaTypeAndFiles.prj)) : null; }) .then(proj => { const fromPrj = proj ? proj.toString() : null; const bareFilename = utils.normalizeName(utils.bareFilename(areaTypeAndFiles.area)); const numLength = areaItems.length.toString().length; let isUTM, coors; if (!utils.isEmptyArray(areaItems)) { let areaItem, knownFields, coorSets = []; for (let i = 0; i < areaItems.length; i++) { const defaultName = areaItems.length > 1 ? `${bareFilename}_%0${numLength}d`.format(i + 1) : bareFilename; areaItem = areaItems[i]; if (i === 0) knownFields = getKnownFieldNames(areaItem); coorSets.length = 0; if (areaItem.geometry.type === 'MultiPolygon') { for (let j = 0; j < areaItem.geometry.coordinates.length; j++) { coorSets.push(areaItem.geometry.coordinates[j]); } } else coorSets.push(areaItem.geometry.coordinates); if (utils.isEmptyArray(coorSets)) continue; if (typeof (isUTM) === 'undefined') { const p = coorSets[0][0][0]; isUTM = !((p[1] >= -90.0 && p[1] <= 90.0) && (p[0] >= -180.0 && p[1] <= 360.0)); if (isUTM && (!fromPrj || !fromPrj.startsWith('PROJ'))) AppError.throw(Errors.PRJ_NOT_FOUND); // return null; // Invalid .prj => will not take items from this invalid shape file } for (let corSet of coorSets) { coors = corSet[0]; if (isUTM) { try { coors = geoUtil.UTMtoLnLats(fromPrj, coors); } catch (error) { debug(error); continue; } } if (utils.isEmptyArray(coors)) continue; if (corSet.length === 1) { if (utils.stringToBoolean(utils.getProp(areaItem.properties, 'XCL'))) { const xcls = createGeoAreas(areaItem, coors, areaItems.length > 1 ? sitecodes[i] : { 'name': bareFilename }, 1, knownFields, defaultName); if (!utils.isEmptyArray(xcls)) jobItem.xclAreas = jobItem.xclAreas.concat(xcls); } else { const areas = createGeoAreas(areaItem, coors, areaItems.length > 1 ? sitecodes[i] : { 'name': bareFilename }, 0, knownFields, defaultName, ops.sprZoneColor, ops.appRate, ops.crop); if (!utils.isEmptyArray(areas)) jobItem.sprayAreas = jobItem.sprayAreas.concat(areas); } } else { // First part is the spray polygon const areas = createGeoAreas(areaItem, coors, areaItems.length > 1 ? sitecodes[i] : { 'name': bareFilename }, 0, knownFields, defaultName, ops.sprZoneColor, ops.appRate, ops.crop); if (!utils.isEmptyArray(areas)) jobItem.sprayAreas = jobItem.sprayAreas.concat(areas); // Create xcl areas (polgons from 2nd part - holes) from the same shape record for (let k = 1; k < corSet.length; k++) { coors = corSet[k]; if (isUTM) { try { coors = geoUtil.UTMtoLnLats(fromPrj, coors); } catch (error) { debug(error); continue; } } if (utils.isEmptyArray(coors)) continue; const xcls = createGeoAreas(areaItem, coors, areaItems.length > 1 ? sitecodes[i] : { 'name': bareFilename }, 1, knownFields, defaultName); if (!utils.isEmptyArray(xcls)) jobItem.xclAreas = jobItem.xclAreas.concat(xcls); } } } } if (!utils.isEmptyArray(wptItems)) { let name, nameIndex = 1; for (let j = 0; j < wptItems.length; j++) { name = wptItems[j].properties.Name; if (name && name.trim().toUpperCase() != "COR1") { coors = wptItems[j].geometry.coordinates; if (isUTM) { try { coors = geoUtil.UTMtoLnLats(fromPrj, coors)[0]; } catch (error) { debug(error); continue; } } const wpt = polyUtil.createGeoPoint(2, coors, name ? name : `WPT_${nameIndex++}`) if (jobItem.waypoints.length < FILE.MAX_ITEM && wpt) jobItem.waypoints.push(wpt); } } } } if (cb) cb(null, jobItem); else return jobItem; }) .catch(err => { if (cb) { return cb(err); } else throw err; }); } function readSHP4Items(shapePath, dbfPath, shpType) { let geoItems = []; return shp.open(shapePath, dbfPath) .then(source => source.read() .then(function log(result) { if (result.done) return; const geoItem = result.value; if ((shpType !== 1 && shpType !== 5) || ((shpType === 1 && geoItem.geometry.type !== 'Point') || ((shpType === 5 && (geoItem.geometry.type !== 'Polygon' && geoItem.geometry.type !== 'MultiPolygon')))) ) { AppInputError.throw(Errors.UNSUPPORTED_SHAPE_TYPE); } geoItems.push(geoItem); return source.read().then(log); }) ) .then(() => { // (util.inspect(geoItems, { showHidden: false, depth: null })); return geoItems; }) .catch(err => { debug(err); return []; }); } function getKnownFieldNames(item) { let rateFieldName, nameFieldName; if (!utils.isEmptyObj(item) && utils.hasProperty(item, 'properties') && !utils.isEmptyObj(item.properties)) { rateFieldName = utils.getProp(item.properties, 'Flies') ? 'Flies' : utils.getProp(item.properties, 'Rate') ? 'Rate' : null; const names = ["GISKEY", "ID", "OBJECTID", "BLOCKID", "TITLE", "SITECODE", "NAME", "BLK_SEC", "PLAN_NUM", "NOPCL"]; for (let i = 0; i < names.length; i++) { if (!nameFieldName && !utils.isBlank(utils.getProp(item.properties, names[i]))) { nameFieldName = names[i]; break; } } if (!nameFieldName) { // If none of the known name field found, assume taking the 1st field for the polygon name (quite common scenario for most recent customers) nameFieldName = Object.keys(item.properties)[0]; } } return ({ rate: rateFieldName, name: nameFieldName }); } function createGeoAreas(geoPoly, coors, siteCodeObj, type, knownFields, defaultName, color = null, appRate = 0, crop = null) { let _appRate = appRate; if (_appRate === 0) _appRate = utils.getProp(geoPoly.properties, knownFields.rate) ? Number(utils.getProp(geoPoly.properties, knownFields.rate)) : 0; const _name = utils.getProp(geoPoly.properties, knownFields.name); let zoneName = siteCodeObj ? utils.getFirstProp(siteCodeObj, _name) : _name; if (utils.isBlank(zoneName)) zoneName = defaultName; return polyUtil.createAreas(type, coors, String(zoneName), color, _appRate, crop); } module.exports = { readSHPToGeoItems, readSHP4Items, readDBF4Items, readDBF4ItemsAsync };