360 lines
11 KiB
JavaScript
360 lines
11 KiB
JavaScript
/**
|
||
* SatLoc Test Data Cleanup Utility
|
||
* Removes test data created by the SatLoc Application Processor tests
|
||
*
|
||
* Usage:
|
||
* node scripts/cleanup_satloc_test_data.js [options]
|
||
*
|
||
* Options:
|
||
* --env <path> Path to environment file (default: ./environment.env)
|
||
* --dry-run Show what would be deleted without actually deleting
|
||
* --force Skip confirmation prompts
|
||
* --orphaned Only clean orphaned data
|
||
* --all Clean all test data including production-like test entries
|
||
*/
|
||
|
||
const path = require('path');
|
||
|
||
// Parse command line arguments first
|
||
const args = process.argv.slice(2);
|
||
let envFile = './environment.env';
|
||
|
||
for (let i = 0; i < args.length; i++) {
|
||
if (args[i] === '--env' && args[i + 1]) {
|
||
envFile = args[i + 1];
|
||
i++;
|
||
}
|
||
}
|
||
|
||
// Load environment variables
|
||
const envPath = path.resolve(process.cwd(), envFile);
|
||
console.log(`Loading environment from: ${envPath}`);
|
||
require('dotenv').config({ path: envPath });
|
||
|
||
const mongoose = require('mongoose');
|
||
const readline = require('readline');
|
||
|
||
// Import models
|
||
const Application = require('../model/application');
|
||
const ApplicationFile = require('../model/application_file');
|
||
const ApplicationDetail = require('../model/application_detail');
|
||
|
||
const args = process.argv.slice(2);
|
||
const dryRun = args.includes('--dry-run');
|
||
const force = args.includes('--force');
|
||
const orphanedOnly = args.includes('--orphaned');
|
||
const cleanAll = args.includes('--all');
|
||
|
||
/**
|
||
* Clean up test data based on various criteria
|
||
*/
|
||
async function cleanupSatLocTestData() {
|
||
console.log('🧹 SatLoc Test Data Cleanup Utility');
|
||
console.log('=====================================');
|
||
|
||
if (dryRun) {
|
||
console.log('🔍 DRY RUN MODE - No data will be deleted');
|
||
}
|
||
|
||
try {
|
||
// PRODUCTION SAFETY CHECK
|
||
if (process.env.NODE_ENV === 'production' && !process.env.ALLOW_TEST_CLEANUP) {
|
||
console.log('⚠️ SAFETY: Test cleanup is disabled in production environment');
|
||
console.log(' Set ALLOW_TEST_CLEANUP=true environment variable to enable');
|
||
return;
|
||
}
|
||
|
||
// Connect to database
|
||
await mongoose.connect('mongodb://agm:agm@127.0.0.1:27017/agmission?authSource=agmission');
|
||
console.log('✅ Connected to database');
|
||
|
||
let testApplications = [];
|
||
|
||
if (orphanedOnly) {
|
||
console.log('\n🔍 Cleaning orphaned data only...');
|
||
await cleanupOrphanedData();
|
||
return;
|
||
}
|
||
|
||
// ULTRA-SAFE TEST DATA CRITERIA - Multiple required conditions
|
||
const TEST_MARKER = 'test_processor';
|
||
|
||
const testCriteria = {
|
||
$and: [
|
||
// REQUIRED: Must be explicitly marked as test data
|
||
{ 'meta.source': TEST_MARKER },
|
||
|
||
// REQUIRED: Must match one of these specific test patterns
|
||
{
|
||
$or: [
|
||
// Known test job IDs only
|
||
{ $and: [{ jobId: 123456 }, { fileName: 'satloc_logs.zip' }] },
|
||
{ $and: [{ jobId: 999999 }, { fileName: 'satloc_logs.zip' }] },
|
||
|
||
// Very specific test file patterns with test marker
|
||
{ $and: [
|
||
{ fileName: { $regex: /^test.*\.log$/i } },
|
||
{ 'meta.source': TEST_MARKER }
|
||
]},
|
||
{ $and: [
|
||
{ fileName: { $regex: /^liquid_if2_g4\.log$/i } },
|
||
{ 'meta.source': TEST_MARKER },
|
||
{ jobId: { $in: [123456, 999999] } }
|
||
]}
|
||
]
|
||
}
|
||
]
|
||
};
|
||
|
||
// Find test applications
|
||
testApplications = await Application.find(testCriteria);
|
||
|
||
if (testApplications.length === 0) {
|
||
console.log('ℹ️ No test applications found');
|
||
await cleanupOrphanedData();
|
||
return;
|
||
}
|
||
|
||
console.log(`\n📊 Found ${testApplications.length} test applications:`);
|
||
|
||
// Show what will be deleted
|
||
for (const app of testApplications.slice(0, 10)) { // Show first 10
|
||
console.log(`- App ${app._id}: Job ${app.jobId || 'N/A'}, File: ${app.fileName}, User: ${app.byUser || 'N/A'}`);
|
||
}
|
||
|
||
if (testApplications.length > 10) {
|
||
console.log(`... and ${testApplications.length - 10} more`);
|
||
}
|
||
|
||
// Count related data
|
||
const applicationIds = testApplications.map(app => app._id);
|
||
|
||
const fileCount = await ApplicationFile.countDocuments({
|
||
appId: { $in: applicationIds }
|
||
});
|
||
|
||
const detailCount = await ApplicationDetail.countDocuments({
|
||
$or: [
|
||
{ appId: { $in: applicationIds } },
|
||
{ fileId: { $exists: true } }
|
||
]
|
||
});
|
||
|
||
console.log(`\n📈 Impact Summary:`);
|
||
console.log(`- Applications: ${testApplications.length}`);
|
||
console.log(`- Application Files: ${fileCount}`);
|
||
console.log(`- Application Details: ${detailCount}`);
|
||
|
||
if (!dryRun && !force) {
|
||
const confirmed = await askConfirmation('Are you sure you want to delete this test data? (y/N): ');
|
||
if (!confirmed) {
|
||
console.log('❌ Cleanup cancelled by user');
|
||
return;
|
||
}
|
||
}
|
||
|
||
if (!dryRun) {
|
||
// ULTRA-SAFE DELETION: Multiple verification layers
|
||
console.log('\n🗑️ Deleting test data with safety checks...');
|
||
|
||
// SAFETY: Re-verify test applications before deletion
|
||
const verifiedTestApps = await Application.find({
|
||
$and: [
|
||
{ _id: { $in: applicationIds } },
|
||
{ 'meta.source': TEST_MARKER } // Must have test marker
|
||
]
|
||
});
|
||
|
||
const verifiedAppIds = verifiedTestApps.map(app => app._id);
|
||
|
||
if (verifiedAppIds.length !== applicationIds.length) {
|
||
console.log(`⚠️ WARNING: Found ${applicationIds.length} apps but only ${verifiedAppIds.length} verified as test data`);
|
||
console.log('❌ Aborting cleanup for safety');
|
||
return;
|
||
}
|
||
|
||
// SAFETY: Only delete ApplicationDetails for verified test applications
|
||
const deletedDetails = await ApplicationDetail.deleteMany({
|
||
$and: [
|
||
{ appId: { $in: verifiedAppIds } },
|
||
// EXTRA SAFETY: Cross-check with test applications
|
||
{ appId: { $in: await Application.find({ 'meta.source': TEST_MARKER }).distinct('_id') } }
|
||
]
|
||
});
|
||
console.log(`✅ Deleted ${deletedDetails.deletedCount} ApplicationDetails`);
|
||
|
||
// SAFETY: Only delete ApplicationFiles for verified test applications
|
||
const deletedFiles = await ApplicationFile.deleteMany({
|
||
$and: [
|
||
{ appId: { $in: verifiedAppIds } },
|
||
// EXTRA SAFETY: Cross-check with test applications
|
||
{ appId: { $in: await Application.find({ 'meta.source': TEST_MARKER }).distinct('_id') } }
|
||
]
|
||
});
|
||
console.log(`✅ Deleted ${deletedFiles.deletedCount} ApplicationFiles`);
|
||
|
||
// SAFETY: Final verification before deleting Applications
|
||
const finalDeletedApps = await Application.deleteMany({
|
||
$and: [
|
||
{ _id: { $in: verifiedAppIds } },
|
||
{ 'meta.source': TEST_MARKER }, // Triple-check test marker
|
||
{ jobId: { $in: [123456, 999999] } } // Only known test job IDs
|
||
]
|
||
});
|
||
console.log(`✅ Deleted ${finalDeletedApps.deletedCount} Applications`);
|
||
|
||
console.log('\n✅ Test data cleanup completed successfully');
|
||
} else {
|
||
console.log('\n🔍 DRY RUN: Would delete the above data');
|
||
}
|
||
|
||
// Always clean orphaned data
|
||
await cleanupOrphanedData();
|
||
|
||
} catch (error) {
|
||
console.error('❌ Cleanup error:', error.message);
|
||
throw error;
|
||
} finally {
|
||
await mongoose.connection.close();
|
||
console.log('✅ Database connection closed');
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Clean up orphaned ApplicationDetails and ApplicationFiles
|
||
*/
|
||
async function cleanupOrphanedData() {
|
||
console.log('\n🔍 Checking for orphaned data...');
|
||
|
||
try {
|
||
let orphanCount = 0;
|
||
|
||
// Find ApplicationDetails without valid Application references
|
||
const orphanedDetails = await ApplicationDetail.find({
|
||
$or: [
|
||
{ appId: { $exists: false } },
|
||
{ appId: null }
|
||
]
|
||
});
|
||
|
||
if (orphanedDetails.length > 0) {
|
||
if (!dryRun) {
|
||
const deletedOrphans = await ApplicationDetail.deleteMany({
|
||
_id: { $in: orphanedDetails.map(d => d._id) }
|
||
});
|
||
console.log(`✅ Deleted ${deletedOrphans.deletedCount} orphaned ApplicationDetails`);
|
||
orphanCount += deletedOrphans.deletedCount;
|
||
} else {
|
||
console.log(`🔍 Would delete ${orphanedDetails.length} orphaned ApplicationDetails`);
|
||
}
|
||
}
|
||
|
||
// Find ApplicationFiles without valid Application references
|
||
const allApplications = await Application.find({}, '_id');
|
||
const validAppIds = allApplications.map(app => app._id);
|
||
|
||
const orphanedFiles = await ApplicationFile.find({
|
||
appId: { $nin: validAppIds }
|
||
});
|
||
|
||
if (orphanedFiles.length > 0) {
|
||
if (!dryRun) {
|
||
// Delete ApplicationDetails linked to orphaned files first
|
||
await ApplicationDetail.deleteMany({
|
||
fileId: { $in: orphanedFiles.map(f => f._id) }
|
||
});
|
||
|
||
const deletedOrphanedFiles = await ApplicationFile.deleteMany({
|
||
_id: { $in: orphanedFiles.map(f => f._id) }
|
||
});
|
||
console.log(`✅ Deleted ${deletedOrphanedFiles.deletedCount} orphaned ApplicationFiles`);
|
||
orphanCount += deletedOrphanedFiles.deletedCount;
|
||
} else {
|
||
console.log(`🔍 Would delete ${orphanedFiles.length} orphaned ApplicationFiles`);
|
||
}
|
||
}
|
||
|
||
if (orphanCount === 0 && !dryRun) {
|
||
console.log('ℹ️ No orphaned data found');
|
||
}
|
||
|
||
} catch (error) {
|
||
console.error('❌ Error during orphaned data cleanup:', error.message);
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Ask for user confirmation
|
||
*/
|
||
function askConfirmation(question) {
|
||
const rl = readline.createInterface({
|
||
input: process.stdin,
|
||
output: process.stdout
|
||
});
|
||
|
||
return new Promise((resolve) => {
|
||
rl.question(question, (answer) => {
|
||
rl.close();
|
||
resolve(answer.toLowerCase().startsWith('y'));
|
||
});
|
||
});
|
||
}
|
||
|
||
/**
|
||
* Show usage information
|
||
*/
|
||
function showUsage() {
|
||
console.log(`
|
||
🧹 SatLoc Test Data Cleanup Utility
|
||
|
||
Usage:
|
||
node scripts/cleanup_satloc_test_data.js [options]
|
||
|
||
Options:
|
||
--dry-run Show what would be deleted without actually deleting
|
||
--force Skip confirmation prompts
|
||
--orphaned Only clean orphaned data (no confirmation needed)
|
||
--all Clean all test data including production-like entries
|
||
--help Show this help message
|
||
|
||
Examples:
|
||
# Safe dry run to see what would be deleted
|
||
node scripts/cleanup_satloc_test_data.js --dry-run
|
||
|
||
# Clean test data with confirmation
|
||
node scripts/cleanup_satloc_test_data.js
|
||
|
||
# Clean without confirmation (dangerous!)
|
||
node scripts/cleanup_satloc_test_data.js --force
|
||
|
||
# Only clean orphaned data
|
||
node scripts/cleanup_satloc_test_data.js --orphaned
|
||
|
||
# Clean everything including production-like test data (VERY dangerous!)
|
||
node scripts/cleanup_satloc_test_data.js --all --force
|
||
`);
|
||
}
|
||
|
||
// Main execution
|
||
if (require.main === module) {
|
||
if (args.includes('--help')) {
|
||
showUsage();
|
||
process.exit(0);
|
||
}
|
||
|
||
cleanupSatLocTestData()
|
||
.then(() => {
|
||
console.log('\n🎉 Cleanup completed successfully!');
|
||
process.exit(0);
|
||
})
|
||
.catch((error) => {
|
||
console.error('\n💥 Cleanup failed:', error.message);
|
||
process.exit(1);
|
||
});
|
||
}
|
||
|
||
module.exports = {
|
||
cleanupSatLocTestData,
|
||
cleanupOrphanedData
|
||
};
|