agmission/server/model/application.js

// Application: AppId, JobId, datetime, zipFileName, status, lastUpdate
const debug = require('debug')('agm:application'),
  mongoose = require('mongoose'), Schema = mongoose.Schema,
  path = require('path'),
  AppFile = require('./application_file'),
  AppDetail = require('./application_detail'),
  { Fields } = require('../helpers/constants'),
  fileUtil = require('../helpers/file_helper'),
  mongoUtil = require('../helpers/mongo'),
  env = require('../helpers/env');

const schema = new Schema({
  jobId: { type: Number, required: false, default: null },
  fileName: { type: String, required: true },
  fileSize: { type: Number, required: true },            // in number of bytes 
  savedFilename: { type: String },
  /** 
   * The upload type determine how to process the uploaded file for this application
   * Upload type: 1: dataOnly, 2: append, 3: overwrite, 4: exclusion areas 
   * Added from version 2.6.9 for tracing purposes
   */
  updateOp: { type: Number },

  createdDate: { type: Date, default: Date.now },
  updateDate: { type: Date, default: Date.now },          // Notes: only update with operations on which the application is processed

  // Legacy hybrid timestamps: local calendar date + UTC time-of-day.
  // UTC companion fields are stored below for query/filter use.
  startDateTime: { type: String },
  endDateTime: { type: String },
  startDateTimeUTC: { type: Date },
  endDateTimeUTC: { type: Date },
  utcOffset: { type: Number },       // Minutes east of UTC based on flight location/timezone
  appRate: { type: Number, required: false },             // Applied application rate in average

  totalSprLength: { type: Number, required: false },      // always in meter(s). Computed for all file types: AgNav binary (.nt), Shape (.shp), and SatLoc (.log)
  totalFlightLength: { type: Number, required: false },   // always in meter(s). Total distance travelled including turns (all GPS segments)

  totalSprayTime: { type: Number, required: false },      // always in seconds
  totalTurnTime: { type: Number, required: false },       // always in seconds
  totalFlightTime: { type: Number, required: false },     // always in seconds

  totalSprayed: { type: Number, required: false },        // always in hectare(s)
  totalSprayMat: { type: Number, required: false },       // Total Sprayed material amount. Always in metric (Lit or Kg)
  totalSprayMatUnit: { type: Number, required: false },   // 3 or 4 (Liter or Kg)

  avgSpraySpeed: { type: Number, required: false },       // Average ground speed (m/s) during spray-on periods, computed at import time
  avgXtError: { type: Number, required: false },          // Average absolute cross-track error (m) across spray-on (sprayStat 1 & 3) records within valid segments
  avgHdop: { type: Number, required: false },             // Average HDOP across spray-on records (lower = better; <1 excellent, 1–2 good, >5 poor)
  flowAccuracyPct: { type: Number, required: false },     // Flow control accuracy: (totalSprayMat/totalSprayed / appRate) × 100; null when appRate=0 or no spray material

  status: { type: Number, required: true, default: 1 },   // -1: was cancelled - to be deleted soon, 0: error, 1: created, 2: in progress, 3: done
  proStatus: { type: Number, default: 0 },                // 0: not fully processed (disrupted while reading or processing files). 1: with data, 2: no data. +10 if items were updated
  errorMsg: { type: String },
  warnMsg: { type: Map },
  cid: { type: String, required: false },                 // Client Id for quickly identifying the uploaded file belongs to which client => Note: going to be removed after the "users migration" done
  byUser: { type: Schema.Types.ObjectId, ref: 'User', required: false },  // Who uploaded the application file, available from recent versions
  byImport: { type: Boolean, default: false },            // Whether the file was uploaded manually for submitted by a console system

  markedDelete: { type: Boolean, default: false }
});

schema.index({ jobId: 1, status: 1, markedDelete: 1, startDateTimeUTC: 1 });

async function deleteAppFiles(fileName) {
  let files = [];
  if (fileName) {
    if (!/.*(.kml|.kmz)+$/i.test(fileName)) {
      // Include the unzipped folder if it is not a kmz or kml
      files = [...files, path.join(env.UNZIP_DIR, fileName), path.join(env.UNZIP_DIR, fileName.replace(path.extname(fileName), ''))];
    }
    files = [...files, path.join(env.UPLOAD_DIR, fileName)]; // The uploaded file

    if (files.length) {
      try {
        await fileUtil.removeFilesAsync(files);
      } catch (err) { // Ignore I/O error }
        console.log(err);
      }
    }
  }
  return files;
}

async function markDeleteApp(session) {
  session && (session.startTransaction(mongoUtil.getTranOps()));
  try {
    this.markedDelete = true;
    await this.save({ session });
    await AppFile.updateMany({ appId: this._id }, { $set: { markedDelete: true } }, { session });
    await mongoUtil.commitWithRetry(session);
  } catch (error) {
    session && (session.abortTransaction());
    throw error;
  }
}

async function deleteAppFilesData(appId) {
  const appFiles = await AppFile.find({ appId: appId }, { _id: 1 }, { lean: true });
  for (let i = 0; i < appFiles.length; i++) {
    await AppDetail.deleteMany({ fileId: appFiles[i]._id });
  }
}

async function deleteApp(session) {
  session && (session.startTransaction(mongoUtil.getTranOps()));
  try {
    await this.remove({ session });
    await AppFile.deleteMany({ appId: this._id }, { session });
    await mongoUtil.commitWithRetry(session);
  } catch (error) {
    session && (session.abortTransaction());
    throw error;
  }
}

/**
 * Remove the Application (and uploaded kmz/kml or zip file) and its related files and data records
 * @param {ClientSession} ses the transactional db session. If ses is null, create a session and end when done.
 * @param {boolean} markDeletedOnly Determine whether to just mark the item as deleted only. Default is true.
 */
schema.methods.removeFull = async function (ses = null, markDeletedOnly = true) {
  const endSesDone = !ses;
  const session = ses || await this.db.startSession({ readPreference: { mode: "primary" } });

  try {
    // 1. Mark this application and all its files as deleted - use transaction
    if (!this[Fields.MARKED_DELETE])
      await mongoUtil.runTransactionWithRetry(markDeleteApp.bind(this), session);

    if (!markDeletedOnly) {
      // 2. Go ahead and try to remove each application files's data records 
      /* Not use transaction to avoid mongo's 'WriteConfict' error pitfall, should cleanup later using a separate maintenane process for marked as deleted apps */
      await deleteAppFilesData(this._id);

      // 3. Remove the application's physical files
      await deleteAppFiles(this.savedFilename);

      // 4. Delete the application
      await mongoUtil.runTransactionWithRetry(deleteApp.bind(this), session);
    }
  } catch (error) {
    debug(error);
    throw error;
  }
  finally {
    if (endSesDone && session) await session.endSession();
  }
}

module.exports = mongoose.model('Application', schema);