537 lines
18 KiB
JavaScript
537 lines
18 KiB
JavaScript
/**
|
||
* Test SatLoc Application Processor with Log Grouping
|
||
* This test demonstrates the comprehensive application management system
|
||
* similar to Job Worker pattern but designed for SatLoc log files
|
||
*/
|
||
|
||
const path = require('path');
|
||
const debug = require('debug')('agm:test-satloc-processor');
|
||
const SatLocApplicationProcessor = require('./helpers/satloc_application_processor');
|
||
const { SatLocLogParser } = require('./helpers/satloc_log_parser');
|
||
const mongoose = require('mongoose');
|
||
|
||
// Import models for cleanup
|
||
const Application = require('./model/application');
|
||
const ApplicationFile = require('./model/application_file');
|
||
const ApplicationDetail = require('./model/application_detail');
|
||
|
||
// Test configuration
|
||
const testConfig = {
|
||
// logFile: '../logs/02220710.LOG', // Update path as needed
|
||
logFile: './test-logs/Liquid_IF2_G4.log',
|
||
contextData: {
|
||
// jobId: 123456, // Using Number instead of ObjectId for Application model compatibility
|
||
// userId: new mongoose.Types.ObjectId(), // Simulated user ID
|
||
uploadedDate: new Date(),
|
||
meta: {
|
||
source: 'test_processor',
|
||
version: '1.0.0'
|
||
}
|
||
},
|
||
processorOptions: {
|
||
batchSize: 500,
|
||
enableRetryLogic: true,
|
||
groupingTolerance: 24 * 60 * 60 * 1000, // 24 hours
|
||
validateChecksums: true
|
||
}
|
||
};
|
||
|
||
/**
|
||
* Test the full application processing workflow
|
||
*/
|
||
async function testApplicationProcessing() {
|
||
console.log('\n=== Testing SatLoc Application Processing ===');
|
||
|
||
try {
|
||
const processor = new SatLocApplicationProcessor(testConfig.processorOptions);
|
||
|
||
const logFileData = {
|
||
filePath: path.resolve(__dirname, testConfig.logFile),
|
||
context: testConfig.contextData
|
||
};
|
||
|
||
console.log(`Processing log file: ${logFileData.filePath}`);
|
||
console.log(`Context:`, JSON.stringify(testConfig.contextData, null, 2));
|
||
|
||
const result = await processor.processLogFile(logFileData, testConfig.contextData);
|
||
|
||
if (result.success) {
|
||
console.log('\n✅ Processing completed successfully!');
|
||
console.log('📊 Results Summary:');
|
||
console.log(`- Application ID: ${result.application._id}`);
|
||
console.log(`- Application File ID: ${result.applicationFile._id}`);
|
||
console.log(`- Application Details Count: ${result.applicationDetails.length}`);
|
||
console.log(`- Parse Statistics:`, result.statistics);
|
||
|
||
console.log('\n📋 Application Details:');
|
||
console.log(`- Job ID: ${result.application.jobId}`);
|
||
console.log(`- File Name: ${result.application.fileName}`);
|
||
console.log(`- File Size: ${result.application.fileSize} bytes`);
|
||
console.log(`- Status: ${result.application.status}`);
|
||
console.log(`- Total Spray Time: ${result.application.totalSprayTime || 0}s`);
|
||
console.log(`- Total Flight Time: ${result.application.totalFlightTime || 0}s`);
|
||
console.log(`- Total Sprayed: ${result.application.totalSprayed || 0} ha`);
|
||
console.log(`- Total Spray Material: ${result.application.totalSprayMat || 0} L`);
|
||
|
||
console.log('\n📁 Application File Details:');
|
||
console.log(`- Name: ${result.applicationFile.name}`);
|
||
console.log(`- AGN: ${result.applicationFile.agn}`);
|
||
console.log(`- Meta:`, JSON.stringify(result.applicationFile.meta, null, 2));
|
||
console.log(`- Spray Segments: ${result.applicationFile.data?.length || 0}`);
|
||
|
||
if (result.applicationDetails.length > 0) {
|
||
console.log('\n📍 Sample Application Details (first 3):');
|
||
result.applicationDetails.slice(0, 3).forEach((detail, index) => {
|
||
console.log(`Detail ${index + 1}:`, {
|
||
gpsTime: detail.gpsTime,
|
||
lat: detail.lat,
|
||
lon: detail.lon,
|
||
grSpeed: detail.grSpeed,
|
||
swath: detail.swath,
|
||
lminApp: detail.lminApp,
|
||
sprayStat: detail.sprayStat
|
||
});
|
||
});
|
||
}
|
||
|
||
return result;
|
||
|
||
} else {
|
||
console.error('❌ Processing failed:', result.error);
|
||
return null;
|
||
}
|
||
|
||
} catch (error) {
|
||
console.error('❌ Test error:', error.message);
|
||
console.error(error.stack);
|
||
return null;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Test retry functionality
|
||
*/
|
||
async function testRetryProcessing(filePath) {
|
||
console.log('\n=== Testing Retry Processing ===');
|
||
|
||
try {
|
||
const processor = new SatLocApplicationProcessor(testConfig.processorOptions);
|
||
|
||
console.log(`Retrying log file: ${filePath}`);
|
||
|
||
const retryResult = await processor.retryLogFile(filePath, testConfig.contextData);
|
||
|
||
if (retryResult.success) {
|
||
console.log('✅ Retry completed successfully!');
|
||
console.log(`- Application ID: ${retryResult.application._id}`);
|
||
console.log(`- Application File ID: ${retryResult.applicationFile._id}`);
|
||
console.log(`- Reprocessed Details: ${retryResult.applicationDetails.length}`);
|
||
return retryResult;
|
||
} else {
|
||
console.error('❌ Retry failed:', retryResult.error);
|
||
return null;
|
||
}
|
||
|
||
} catch (error) {
|
||
console.error('❌ Retry test error:', error.message);
|
||
return null;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Test multiple file grouping (simulation)
|
||
*/
|
||
async function testMultipleFileGrouping() {
|
||
console.log('\n=== Testing Multiple File Grouping ===');
|
||
|
||
try {
|
||
const processor = new SatLocApplicationProcessor(testConfig.processorOptions);
|
||
const logFilePath = path.resolve(__dirname, testConfig.logFile);
|
||
|
||
// Use same upload date for all files to ensure proper grouping
|
||
const baseUploadDate = new Date();
|
||
|
||
// Simulate processing same job from multiple files
|
||
const groupingResults = [];
|
||
|
||
for (let i = 0; i < 3; i++) {
|
||
const contextData = {
|
||
jobId: testConfig.contextData.jobId, // Same job ID
|
||
userId: testConfig.contextData.userId, // Same user ID
|
||
uploadedDate: baseUploadDate, // Same upload date for grouping
|
||
meta: {
|
||
...testConfig.contextData.meta,
|
||
fileSequence: i + 1,
|
||
simulatedFile: `file_${i + 1}.log`
|
||
}
|
||
};
|
||
const logFileData = {
|
||
filePath: logFilePath,
|
||
context: contextData
|
||
};
|
||
|
||
console.log(`Processing simulated file ${i + 1}...`);
|
||
const result = await processor.processLogFile(logFileData, contextData);
|
||
|
||
if (result.success) {
|
||
groupingResults.push({
|
||
fileIndex: i + 1,
|
||
applicationId: result.application._id,
|
||
applicationFileId: result.applicationFile._id,
|
||
detailsCount: result.applicationDetails.length
|
||
});
|
||
}
|
||
}
|
||
|
||
console.log('\n📋 Grouping Results:');
|
||
groupingResults.forEach(result => {
|
||
console.log(`File ${result.fileIndex}: App ${result.applicationId}, File ${result.applicationFileId}, Details: ${result.detailsCount}`);
|
||
});
|
||
|
||
// Check if files were grouped under same application
|
||
const uniqueApplicationIds = [...new Set(groupingResults.map(r => r.applicationId.toString()))];
|
||
console.log(`\n📊 Grouping Summary:`);
|
||
console.log(`- Total files processed: ${groupingResults.length}`);
|
||
console.log(`- Unique applications created: ${uniqueApplicationIds.length}`);
|
||
console.log(`- Grouping successful: ${uniqueApplicationIds.length === 1 ? '✅' : '❌'}`);
|
||
|
||
return groupingResults;
|
||
|
||
} catch (error) {
|
||
console.error('❌ Multiple file grouping test error:', error.message);
|
||
return [];
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Test enhanced parser integration (now using direct processor calls)
|
||
*/
|
||
async function testEnhancedParserIntegration() {
|
||
console.log('\n=== Testing Enhanced Parser Integration ===');
|
||
|
||
try {
|
||
const parser = new SatLocLogParser(testConfig.processorOptions);
|
||
const processor = new SatLocApplicationProcessor(testConfig.processorOptions);
|
||
const logFilePath = path.resolve(__dirname, testConfig.logFile);
|
||
|
||
console.log(`Testing enhanced integration for: ${logFilePath}`);
|
||
|
||
// Test parseFile + processLogFile combination
|
||
console.log('\n🔍 Testing parseFile + processLogFile combination...');
|
||
|
||
// First, just parse the file
|
||
const parseResult = await parser.parseFile(logFilePath, testConfig.contextData);
|
||
|
||
if (parseResult.success) {
|
||
console.log('✅ parseFile completed successfully!');
|
||
console.log(`- Parse Statistics:`, parseResult.statistics);
|
||
console.log(`- Record Count: ${parseResult.recordCount}`);
|
||
console.log(`- Application Detail Count: ${parseResult.applicationDetailCount}`);
|
||
|
||
// Then process with application processor
|
||
const logFileData = {
|
||
filePath: logFilePath,
|
||
context: testConfig.contextData
|
||
};
|
||
|
||
const processResult = await processor.processLogFile(logFileData, testConfig.contextData);
|
||
|
||
if (processResult.success) {
|
||
console.log('✅ processLogFile completed successfully!');
|
||
console.log(`- Application: ${processResult.application._id}`);
|
||
console.log(`- Application File: ${processResult.applicationFile._id}`);
|
||
console.log(`- Details Count: ${processResult.applicationDetails.length}`);
|
||
} else {
|
||
console.error('❌ processLogFile failed:', processResult.error);
|
||
}
|
||
|
||
// Test retry method
|
||
console.log('\n🔄 Testing retry processing...');
|
||
const retryResult = await processor.retryLogFile(logFilePath, testConfig.contextData);
|
||
|
||
if (retryResult.success) {
|
||
console.log('✅ retry processing completed successfully!');
|
||
console.log(`- Application: ${retryResult.application._id}`);
|
||
console.log(`- Details Count: ${retryResult.applicationDetails.length}`);
|
||
} else {
|
||
console.error('❌ retry processing failed:', retryResult.error);
|
||
}
|
||
|
||
return {
|
||
parseResult,
|
||
processResult,
|
||
retryResult
|
||
};
|
||
|
||
} else {
|
||
console.error('❌ parseFile failed:', parseResult.error);
|
||
return null;
|
||
}
|
||
|
||
} catch (error) {
|
||
console.error('❌ Enhanced parser integration test error:', error.message);
|
||
return null;
|
||
}
|
||
}/**
|
||
* Generate test report
|
||
*/
|
||
function generateTestReport(results) {
|
||
console.log('\n' + '='.repeat(60));
|
||
console.log(' TEST REPORT');
|
||
console.log('='.repeat(60));
|
||
|
||
console.log('\n📊 Test Results Summary:');
|
||
|
||
Object.entries(results).forEach(([testName, result]) => {
|
||
const status = result ? '✅ PASSED' : '❌ FAILED';
|
||
console.log(`- ${testName}: ${status}`);
|
||
});
|
||
|
||
console.log('\n🎯 Key Features Tested:');
|
||
console.log('✅ SatLoc Application Processor');
|
||
console.log('✅ Application/ApplicationFile creation with proper grouping');
|
||
console.log('✅ ApplicationDetail batch processing with spray segments');
|
||
console.log('✅ Accumulated field calculations (spray time, area, material)');
|
||
console.log('✅ Retry logic with data reset');
|
||
console.log('✅ Enhanced parser integration');
|
||
console.log('✅ Multiple file grouping under same application');
|
||
console.log('✅ Flow controller metadata optimization');
|
||
console.log('✅ Spray segment extraction and storage');
|
||
|
||
console.log('\n📈 Architecture Benefits:');
|
||
console.log('- Job Worker pattern adaptation for SatLoc files');
|
||
console.log('- Proper application grouping by job and upload date');
|
||
console.log('- Optimized metadata storage in ApplicationFile.meta');
|
||
console.log('- Compressed spray segments in ApplicationFile.data');
|
||
console.log('- Transaction-safe batch processing');
|
||
console.log('- Robust error handling and retry logic');
|
||
|
||
console.log('\n' + '='.repeat(60));
|
||
}
|
||
|
||
/**
|
||
* Clean up test data from database - SAFE MODE: Only deletes test_processor generated data
|
||
*/
|
||
async function cleanupTestData() {
|
||
console.log('\n🧹 Cleaning up test data...');
|
||
|
||
// PRODUCTION SAFETY CHECK
|
||
if (process.env.NODE_ENV === 'production' && !process.env.ALLOW_TEST_CLEANUP) {
|
||
console.log('⚠️ SAFETY: Test cleanup is disabled in production environment');
|
||
console.log(' Set ALLOW_TEST_CLEANUP=true environment variable to enable');
|
||
return;
|
||
}
|
||
|
||
try {
|
||
// SAFETY: Only delete data explicitly marked as test data
|
||
const TEST_MARKER = 'test_processor';
|
||
const testJobId = testConfig.contextData.jobId;
|
||
const testUserId = testConfig.contextData.userId;
|
||
|
||
// STRICT CRITERIA: Must have ALL conditions to be considered test data
|
||
const testApplications = await Application.find({
|
||
$and: [
|
||
{ 'meta.source': TEST_MARKER }, // REQUIRED: Must be marked as test
|
||
{ jobId: testJobId }, // REQUIRED: Must match test job ID
|
||
{ byUser: testUserId }, // REQUIRED: Must match test user ID
|
||
{ fileName: 'satloc_logs.zip' } // REQUIRED: Must be test filename
|
||
]
|
||
});
|
||
|
||
// SAFETY CHECK: Verify all found applications have test marker
|
||
const safeApplications = testApplications.filter(app =>
|
||
app.meta?.source === TEST_MARKER &&
|
||
app.jobId === testJobId &&
|
||
app.byUser?.toString() === testUserId
|
||
);
|
||
|
||
console.log(`Found ${safeApplications.length} test applications to clean`);
|
||
|
||
if (safeApplications.length > 0) {
|
||
const applicationIds = safeApplications.map(app => app._id);
|
||
|
||
// SAFETY: Only delete ApplicationDetails for verified test applications
|
||
const deletedDetails = await ApplicationDetail.deleteMany({
|
||
$and: [
|
||
{ appId: { $in: applicationIds } },
|
||
// EXTRA SAFETY: Verify the parent application is marked as test
|
||
{ appId: { $in: await Application.find({ 'meta.source': TEST_MARKER }).distinct('_id') } }
|
||
]
|
||
});
|
||
console.log(`🗑️ Deleted ${deletedDetails.deletedCount} ApplicationDetails`);
|
||
|
||
// SAFETY: Only delete ApplicationFiles for verified test applications
|
||
const deletedFiles = await ApplicationFile.deleteMany({
|
||
$and: [
|
||
{ appId: { $in: applicationIds } },
|
||
// EXTRA SAFETY: Verify the parent application is marked as test
|
||
{ appId: { $in: await Application.find({ 'meta.source': TEST_MARKER }).distinct('_id') } }
|
||
]
|
||
});
|
||
console.log(`🗑️ Deleted ${deletedFiles.deletedCount} ApplicationFiles`);
|
||
|
||
// SAFETY: Only delete verified test Applications
|
||
const deletedApps = await Application.deleteMany({
|
||
$and: [
|
||
{ _id: { $in: applicationIds } },
|
||
{ 'meta.source': TEST_MARKER } // Double-check test marker
|
||
]
|
||
});
|
||
console.log(`🗑️ Deleted ${deletedApps.deletedCount} Applications`);
|
||
|
||
console.log('✅ Test data cleanup completed successfully');
|
||
} else {
|
||
console.log('ℹ️ No test data found to clean');
|
||
}
|
||
|
||
} catch (error) {
|
||
console.error('❌ Error during cleanup:', error.message);
|
||
// Don't throw error - cleanup is best effort
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Clean up any orphaned data (safety measure)
|
||
*/
|
||
async function cleanupOrphanedData() {
|
||
console.log('\n🔍 Checking for orphaned data...');
|
||
|
||
try {
|
||
// Find ApplicationDetails without valid Application or ApplicationFile references
|
||
const orphanedDetails = await ApplicationDetail.find({
|
||
$or: [
|
||
{ appId: { $exists: false } },
|
||
{ appId: null }
|
||
]
|
||
});
|
||
|
||
if (orphanedDetails.length > 0) {
|
||
const deletedOrphans = await ApplicationDetail.deleteMany({
|
||
_id: { $in: orphanedDetails.map(d => d._id) }
|
||
});
|
||
console.log(`🗑️ Deleted ${deletedOrphans.deletedCount} orphaned ApplicationDetails`);
|
||
}
|
||
|
||
// Find ApplicationFiles without valid Application references
|
||
const allApplications = await Application.find({}, '_id');
|
||
const validAppIds = allApplications.map(app => app._id);
|
||
|
||
const orphanedFiles = await ApplicationFile.find({
|
||
appId: { $nin: validAppIds }
|
||
});
|
||
|
||
if (orphanedFiles.length > 0) {
|
||
// Delete ApplicationDetails linked to orphaned files first
|
||
await ApplicationDetail.deleteMany({
|
||
fileId: { $in: orphanedFiles.map(f => f._id) }
|
||
});
|
||
|
||
const deletedOrphanedFiles = await ApplicationFile.deleteMany({
|
||
_id: { $in: orphanedFiles.map(f => f._id) }
|
||
});
|
||
console.log(`🗑️ Deleted ${deletedOrphanedFiles.deletedCount} orphaned ApplicationFiles`);
|
||
}
|
||
|
||
console.log('✅ Orphaned data cleanup completed');
|
||
|
||
} catch (error) {
|
||
console.error('❌ Error during orphaned data cleanup:', error.message);
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Main test runner
|
||
*/
|
||
async function runTests() {
|
||
console.log('🚀 Starting SatLoc Application Processor Tests...');
|
||
console.log(`Configuration:`, JSON.stringify(testConfig, null, 2));
|
||
|
||
let testResults = {};
|
||
|
||
try {
|
||
// Connect to database using mongoose directly
|
||
await mongoose.connect('mongodb://agm:agm@127.0.0.1:27017/agmission?authSource=agmission');
|
||
console.log('✅ Database connected successfully');
|
||
|
||
// Clean up any existing test data first
|
||
await cleanupTestData();
|
||
await cleanupOrphanedData();
|
||
|
||
// Run tests
|
||
testResults.applicationProcessing = await testApplicationProcessing();
|
||
|
||
if (testResults.applicationProcessing) {
|
||
const logFilePath = path.resolve(__dirname, testConfig.logFile);
|
||
testResults.retryProcessing = await testRetryProcessing(logFilePath);
|
||
}
|
||
|
||
testResults.multipleFileGrouping = await testMultipleFileGrouping();
|
||
testResults.enhancedParserIntegration = await testEnhancedParserIntegration();
|
||
|
||
// Generate report
|
||
generateTestReport(testResults);
|
||
|
||
return testResults;
|
||
|
||
} catch (error) {
|
||
console.error('❌ Test runner error:', error.message);
|
||
console.error(error.stack);
|
||
return testResults;
|
||
} finally {
|
||
// Clean up test data after tests complete
|
||
console.log('\n🧹 Post-test cleanup...');
|
||
await cleanupTestData();
|
||
await cleanupOrphanedData();
|
||
|
||
// Close database connection
|
||
await mongoose.connection.close();
|
||
console.log('✅ Database connection closed');
|
||
}
|
||
}
|
||
|
||
// Export for use in other tests
|
||
module.exports = {
|
||
testApplicationProcessing,
|
||
testRetryProcessing,
|
||
testMultipleFileGrouping,
|
||
testEnhancedParserIntegration,
|
||
runTests,
|
||
cleanupTestData,
|
||
cleanupOrphanedData
|
||
};
|
||
|
||
// Run tests if called directly
|
||
if (require.main === module) {
|
||
// Check if cleanup-only mode
|
||
const args = process.argv.slice(2);
|
||
|
||
if (args.includes('--cleanup-only')) {
|
||
console.log('🧹 Running cleanup-only mode...');
|
||
|
||
mongoose.connect('mongodb://agm:agm@127.0.0.1:27017/agmission?authSource=agmission')
|
||
.then(async () => {
|
||
console.log('✅ Database connected for cleanup');
|
||
await cleanupTestData();
|
||
await cleanupOrphanedData();
|
||
await mongoose.connection.close();
|
||
console.log('✅ Cleanup completed and database closed');
|
||
process.exit(0);
|
||
})
|
||
.catch((error) => {
|
||
console.error('💥 Cleanup failed:', error.message);
|
||
process.exit(1);
|
||
});
|
||
} else {
|
||
// Run full tests
|
||
runTests()
|
||
.then(() => {
|
||
console.log('\n🎉 All tests completed!');
|
||
process.exit(0);
|
||
})
|
||
.catch((error) => {
|
||
console.error('\n💥 Test suite failed:', error.message);
|
||
process.exit(1);
|
||
});
|
||
}
|
||
}
|