module.exports = function (mongoose) { const isProd = (process.env.NODE_ENV && process.env.NODE_ENV.toLowerCase() == "production"), debug = require('debug')('maintainer:db-utils'), async = require('async'), fs = require('fs-extra'), path = require('path'), util = require('util'), moment = require('moment'), unzip = require('extract-zip'), nightmare = require('nightmare')({ show: false }), cheerio = require('cheerio'), utils = require(isProd ? '../agmission/helpers/utils' : '../server/helpers/utils'), obsUtil = require(isProd ? '../agmission/helpers/file_obstacle' : '../server/helpers/file_obstacle'), Obstacle = require('../shared/model/obstacles')(mongoose); var module = {}; const stateFileName = 'state.json'; let getDOFLinks = html => { var data = [], a; const $ = cheerio.load(html); $("table caption:contains('Digital Obstacle Files'), tbody > tr > td").each((i, td) => { a = $(td).find('a'); if (a.length && (a.text() && a.text().toUpperCase() === "DOF")) { data.push({ product: a.text(), link: a.attr('href') }); } }); return data; } function getLatestFAAObsZip(lastState) { const url = 'https://www.faa.gov/air_traffic/flight_info/aeronav/digital_products/dof/'; return nightmare .goto(url) .wait('body') .evaluate(() => document.querySelector('body').innerHTML) .end() .then(response => { let dofs = getDOFLinks(response); if (dofs.length) { let dof = dofs[dofs.length - 1]; if (!dof.link) throw new Error('no_obs_file'); if (lastState.dofFile && path.basename(lastState.dofFile.toLowerCase()) === path.basename(dof.link).toLowerCase()) throw new Error('obs_file_updated'); return dof.link; } return null; }) .catch(err => { throw err; }); } module.updateFAAObstacles = function (cb) { const unzipPath = isProd ? '/media/ssd1/agmission/.tmp/obs/' : '../server/.tmp/obs/', dofFilePath = path.join(unzipPath, 'DOF.DAT'), lastState = fs.readJsonSync(stateFileName, { throws: false }) || {}; var fileURL, zipFile; /** * Steps: Get the lastest DOF zip file from FAAA website Download the file Remove the old records then read the file; Import Update last state with the last download file */ var obstacles = []; async.series([ function (callback) { return getLatestFAAObsZip(lastState) .then(url => { if (url) { fileURL = url; zipFile = path.join(unzipPath, path.basename(fileURL)); } callback(); }) .catch(err => callback(err)); }, function (callback) { if (fileURL) { debug("Downloading FAA Obstacle file :" + fileURL); fs.ensureDirSync(unzipPath); utils.download(fileURL, zipFile, callback); } else return callback('no_obs_file'); }, function (callback) { debug("Unzipping :" + zipFile); unzip(zipFile, { dir: path.resolve(unzipPath) }, (err) => { if (err) return callback("corrupted_zip"); if (!fs.pathExistsSync(dofFilePath)) return callback('no_obs_dof_file'); callback(); }); }, function (callback) { obsUtil.readObstacles(dofFilePath, (err, obs) => { if (err) return callback(err); obstacles = obs; callback(); }); }, function (callback) { debug("Removing old obstacles !"); return Obstacle.deleteMany({ "properties.type": { $ne: "USER" } }, callback); }, function (callback) { if (!obstacles || !obstacles.length) return callback(); debug(`Importing new ${obstacles.length} records ...`); const chunks = utils.chunkArray(obstacles, 1000); async.eachSeries(chunks, (chunk, cb) => { Obstacle.insertMany(chunk, cb); }, callback); } ], err => { if (err) { if (err.message && !err.message.includes('_')) debug('Error when updating FAA Obstacles', err); return cb(err); } else { debug(`Imported Obstacle file ${dofFilePath} with ${obstacles.length ? obstacles.length : 0} records`); lastState['dofFile'] = fileURL; lastState['total'] = obstacles.length; lastState['date'] = moment.utc().toISOString(); fs.writeJsonSync(stateFileName, lastState); return cb(null, obstacles.length); } }); } module.updateFAAObstaclesASync = util.promisify(module.updateFAAObstacles); return module; }