agmission/Development/maintainer/db-utils.js

module.exports = function (mongoose) {
  const
    isProd = (process.env.NODE_ENV && process.env.NODE_ENV.toLowerCase() == "production"),
    debug = require('debug')('maintainer:db-utils'),
    async = require('async'),
    fs = require('fs-extra'),
    path = require('path'),
    util = require('util'),
    moment = require('moment'),
    unzip = require('extract-zip'),
    nightmare = require('nightmare')({ show: false }),
    cheerio = require('cheerio'),
    utils = require(isProd ? '../agmission/helpers/utils' : '../server/helpers/utils'),
    obsUtil = require(isProd ? '../agmission/helpers/file_obstacle' : '../server/helpers/file_obstacle'),
    Obstacle = require('../shared/model/obstacles')(mongoose);

  var module = {};
  const stateFileName = 'state.json';

  let getDOFLinks = html => {
    var data = [], a;
    const $ = cheerio.load(html);
    $("table caption:contains('Digital Obstacle Files'), tbody > tr > td").each((i, td) => {
      a = $(td).find('a');
      if (a.length && (a.text() && a.text().toUpperCase() === "DOF")) {
        data.push({ product: a.text(), link: a.attr('href') });
      }
    });
    return data;
  }

  function getLatestFAAObsZip(lastState) {

    const url = 'https://www.faa.gov/air_traffic/flight_info/aeronav/digital_products/dof/';
    return nightmare
      .goto(url)
      .wait('body')
      .evaluate(() => document.querySelector('body').innerHTML)
      .end()
      .then(response => {
        let dofs = getDOFLinks(response);
        if (dofs.length) {
          let dof = dofs[dofs.length - 1];
          if (!dof.link)
            throw new Error('no_obs_file');

          if (lastState.dofFile && path.basename(lastState.dofFile.toLowerCase()) === path.basename(dof.link).toLowerCase())
            throw new Error('obs_file_updated');

          return dof.link;
        }
        return null;
      })
      .catch(err => {
        throw err;
      });
  }

  module.updateFAAObstacles = function (cb) {
    const
      unzipPath = isProd ? '/media/ssd1/agmission/.tmp/obs/' : '../server/.tmp/obs/',
      dofFilePath = path.join(unzipPath, 'DOF.DAT'),
      lastState = fs.readJsonSync(stateFileName, { throws: false }) || {};

    var fileURL, zipFile;
    /**
     * Steps:
      Get the lastest DOF zip file from FAAA website
      Download the file
      Remove the old records then read the file; Import
      Update last state with the last download file
     */
    var obstacles = [];
    async.series([
      function (callback) {
        return getLatestFAAObsZip(lastState)
          .then(url => {
            if (url) {
              fileURL = url;
              zipFile = path.join(unzipPath, path.basename(fileURL));
            }
            callback();
          })
          .catch(err => callback(err));
      },
      function (callback) {
        if (fileURL) {
          debug("Downloading FAA Obstacle file :" + fileURL);
          fs.ensureDirSync(unzipPath);
          utils.download(fileURL, zipFile, callback);
        }
        else
          return callback('no_obs_file');
      },
      function (callback) {
        debug("Unzipping :" + zipFile);
        unzip(zipFile, { dir: path.resolve(unzipPath) }, (err) => {
          if (err)
            return callback("corrupted_zip");

          if (!fs.pathExistsSync(dofFilePath))
            return callback('no_obs_dof_file');

          callback();
        });
      },
      function (callback) {
        obsUtil.readObstacles(dofFilePath, (err, obs) => {
          if (err)
            return callback(err);
          obstacles = obs;
          callback();
        });
      },
      function (callback) {
        debug("Removing old obstacles !");
        return Obstacle.deleteMany({ "properties.type": { $ne: "USER" } }, callback);
      },
      function (callback) {
        if (!obstacles || !obstacles.length)
          return callback();

        debug(`Importing new ${obstacles.length} records ...`);
        const chunks = utils.chunkArray(obstacles, 1000);
        async.eachSeries(chunks, (chunk, cb) => {
          Obstacle.insertMany(chunk, cb);
        }, callback);
      }
    ], err => {
      if (err) {
        if (err.message && !err.message.includes('_'))
          debug('Error when updating FAA Obstacles', err);
        return cb(err);
      }
      else {
        debug(`Imported Obstacle file ${dofFilePath} with ${obstacles.length ? obstacles.length : 0} records`);

        lastState['dofFile'] = fileURL;
        lastState['total'] = obstacles.length;
        lastState['date'] = moment.utc().toISOString();
        fs.writeJsonSync(stateFileName, lastState);

        return cb(null, obstacles.length);
      }
    });
  }

  module.updateFAAObstaclesASync = util.promisify(module.updateFAAObstacles);

  return module;
}