copy of satloc-resume branch as of April 22 2026

2026-04-22 15:04:54 -04:00
10 changed files with 537 additions and 167 deletions
--- a/.gitea/workflows/sync-to-svn.yaml
+++ b/.gitea/workflows/sync-to-svn.yaml
@ -1,77 +0,0 @@
-# Gitea Actions – Sync git repository to SVN
-# Equivalent of .circleci/config.yml, translated to GitHub Actions syntax
-# which Gitea Actions supports natively.
-#
-# Prerequisites (set as repository Secrets in Gitea → Settings → Secrets):
-#   SVN_USERNAME  – SVN commit username
-#   SVN_PASSWORD  – SVN commit password
-#   SVN_REPO_URL  – Base SVN repo URL (e.g. https://svn.example.com/repos/myproject)
-
-name: Sync to SVN
-on:
-  push:
-    branches:
-      - master
-
-jobs:
-  push-to-svn:
-    runs-on: self-hosted
-
-    steps:
-      - name: Checkout git repository
-        uses: actions/checkout@v4
-
-      # SVN and rsync are pre-installed in the container image, but the
-      # apt-get call is kept so the workflow works on a stock runner too.
-      - name: Install SVN and rsync
-        run: |
-          if ! command -v svn &>/dev/null || ! command -v rsync &>/dev/null; then
-            sudo apt-get update -qq && sudo apt-get install -y subversion rsync
-          fi
-
-      - name: Verify SVN credentials are set
-        run: |
-          if [ -z "${{ secrets.SVN_USERNAME }}" ]; then echo "ERROR: SVN_USERNAME secret is not set" && exit 1; fi
-          if [ -z "${{ secrets.SVN_PASSWORD }}" ]; then echo "ERROR: SVN_PASSWORD secret is not set" && exit 1; fi
-          if [ -z "${{ secrets.SVN_REPO_URL }}" ];  then echo "ERROR: SVN_REPO_URL secret is not set"  && exit 1; fi
-          echo "All SVN secrets are set."
-
-      - name: Checkout SVN branch
-        run: |
-          svn checkout \
-            --username "${{ secrets.SVN_USERNAME }}" \
-            --password "${{ secrets.SVN_PASSWORD }}" \
-            --no-auth-cache \
-            --non-interactive \
-            --trust-server-cert \
-            "${{ secrets.SVN_REPO_URL }}/branches/data-export-api-copy" svn-branch
-
-      - name: Sync files to SVN working copy
-        run: |
-          rsync -a --delete \
-            --exclude='.git/' \
-            --exclude='.gitea/' \
-            --exclude='.svn/' \
-            --exclude='svn-branch/' \
-            . svn-branch/
-      - name: Stage and commit to SVN
-        run: |
-          cd svn-branch
-
-          # Add all new/unversioned files and directories in one pass
-          svn add --force .
-
-          # Delete files removed from git (handles paths with spaces).
-          # '|| true' prevents pipefail aborting when grep finds no '!' lines.
-          svn status | grep '^!' | while IFS= read -r line; do
-            svn delete "${line:8}@"
-          done || true
-
-          # Attempt commit; svn commit exits 0 silently when nothing changed
-          svn commit \
-            --username "${{ secrets.SVN_USERNAME }}" \
-            --password "${{ secrets.SVN_PASSWORD }}" \
-            --no-auth-cache \
-            --non-interactive \
-            --trust-server-cert \
-            -m "Gitea CI sync from commit ${{ github.sha }} [ci skip]"
--- a/.githooks/pre-commit
+++ b/.githooks/pre-commit
@ -1,64 +0,0 @@
-#!/bin/sh
-
-branch=$(git rev-parse --abbrev-ref HEAD)
-
-# Allow master and development
-if [ "$branch" = "master" ] || [ "$branch" = "development" ]; then
-  exit 0
-fi
-
-# Validate feature/* and bugfix/* branches
-case "$branch" in
-  feature/*|bugfix/*)
-    name="${branch#*/}"
-
-    # Name must not be empty
-    if [ -z "$name" ]; then
-      echo "ERROR: Branch '$branch' has an empty name after the prefix."
-      exit 1
-    fi
-
-    # Only letters, digits, and dashes allowed
-    if echo "$name" | grep -qE '[^a-zA-Z0-9-]'; then
-      echo "ERROR: Branch '$branch' contains invalid characters."
-      echo "       Only letters, digits, and dashes are allowed after the prefix."
-      exit 1
-    fi
-
-    # No leading or trailing dashes
-    if echo "$name" | grep -qE '^-|-$'; then
-      echo "ERROR: Branch '$branch' must not start or end with a dash."
-      exit 1
-    fi
-
-    # No consecutive dashes
-    if echo "$name" | grep -q -- '--'; then
-      echo "ERROR: Branch '$branch' must not contain consecutive dashes."
-      exit 1
-    fi
-
-    # Max 25 characters excluding dashes
-    name_no_dashes=$(echo "$name" | tr -d '-')
-    char_count=${#name_no_dashes}
-    if [ "$char_count" -gt 25 ]; then
-      echo "ERROR: Branch '$branch': the name after the prefix is $char_count characters (excluding dashes), max is 25."
-      exit 1
-    fi
-
-    exit 0
-    ;;
-esac
-
-echo "ERROR: Branch name '$branch' does not follow the naming convention."
-echo ""
-echo "  Allowed formats:"
-echo "    master"
-echo "    development"
-echo "    feature/{name}"
-echo "    bugfix/{name}"
-echo ""
-echo "  Rules for feature/bugfix names:"
-echo "    - Letters, digits, and dashes only (no spaces)"
-echo "    - No leading, trailing, or consecutive dashes"
-echo "    - Max 20 characters excluding dashes"
-exit 1
--- a/.gitignore
+++ b/.gitignore
@ -1,19 +0,0 @@
-# Dependencies
-**/node_modules/
-
-# Logs
-*.log
-*.rlog
-npm-debug.log*
-
-# Environment files
-**/*.env
-**/environment.env
-
-# Build output
-**/dist/
-**/build/
-
-# OS files
-.DS_Store
-Thumbs.db
--- a/Development/libs/phantomjs
+++ b/Development/libs/phantomjs
--- a/Development/libs/ultima-ng-9.0.0.zip
+++ b/Development/libs/ultima-ng-9.0.0.zip
--- a/Development/server/.tmp/export_69e28118a9673b08101159b1.csv
+++ b/Development/server/.tmp/export_69e28118a9673b08101159b1.csv
@ -1 +0,0 @@
-jobId,orderNumber,jobName,sessionId,fileName,pilotName,timestampUtc,gpsTime,lat,lon,utmX,utmY,alt_m,groundSpeed_ms,heading,crossTrackError_m,lockedLine,hdop,satsInView,correctionId,waasId,sprayStat,flowRateApplied_Lmin,flowRateRequired_Lmin,appRateRequired_Lha,appRateApplied_Lha,swathWidth_m,boomPressure_psi,sprayOnLag_s,sprayOffLag_s,pulsesPerLitre,windSpeed_ms,windDir_deg,temp_c,humidity_pct
--- a/Development/server/.tmp/export_69e28122a9673b08101159bc.csv
+++ b/Development/server/.tmp/export_69e28122a9673b08101159bc.csv
@ -1 +0,0 @@
-jobId,orderNumber,jobName,sessionId,fileName,pilotName,timestampUtc,gpsTime,lat,lon,utmX,utmY,alt_m,groundSpeed_ms,heading,crossTrackError_m,lockedLine,hdop,satsInView,correctionId,waasId,sprayStat,flowRateApplied_Lmin,flowRateRequired_Lmin,appRateRequired_Lha,appRateApplied_Lha,swathWidth_m,boomPressure_psi,sprayOnLag_s,sprayOffLag_s,pulsesPerLitre,windSpeed_ms,windDir_deg,temp_c,humidity_pct
--- a/Development/server/model/index.js
+++ b/Development/server/model/index.js
@ -29,4 +29,4 @@ module.exports = {
  Partner: require('./partner').Partner,
  PartnerSystemUser: require('./partner').PartnerSystemUser,
  PartnerLogTracker: require('./partner_log_tracker')
-}
+}
--- a/Development/server/public/partner-dlq-monitor.html
+++ b/Development/server/public/partner-dlq-monitor.html
@ -0,0 +1,535 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Partner DLQ Monitor</title>
+    <style>
+        * {
+            margin: 0;
+            padding: 0;
+            box-sizing: border-box;
+        }
+        
+        body {
+            font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, sans-serif;
+            background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
+            min-height: 100vh;
+            padding: 20px;
+        }
+        
+        .container {
+            max-width: 1400px;
+            margin: 0 auto;
+        }
+        
+        h1 {
+            color: white;
+            margin-bottom: 30px;
+            text-align: center;
+            font-size: 2.5em;
+            text-shadow: 2px 2px 4px rgba(0,0,0,0.2);
+        }
+        
+        .stats-grid {
+            display: grid;
+            grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));
+            gap: 20px;
+            margin-bottom: 30px;
+        }
+        
+        .stat-card {
+            background: white;
+            border-radius: 10px;
+            padding: 25px;
+            box-shadow: 0 4px 6px rgba(0,0,0,0.1);
+            transition: transform 0.2s;
+        }
+        
+        .stat-card:hover {
+            transform: translateY(-5px);
+            box-shadow: 0 6px 12px rgba(0,0,0,0.15);
+        }
+        
+        .stat-card h3 {
+            color: #667eea;
+            font-size: 0.9em;
+            margin-bottom: 10px;
+            text-transform: uppercase;
+            letter-spacing: 1px;
+        }
+        
+        .stat-card .value {
+            font-size: 2.5em;
+            font-weight: bold;
+            color: #333;
+        }
+        
+        .stat-card.danger .value {
+            color: #e74c3c;
+        }
+        
+        .stat-card.success .value {
+            color: #27ae60;
+        }
+        
+        .stat-card.warning .value {
+            color: #f39c12;
+        }
+        
+        .actions {
+            background: white;
+            border-radius: 10px;
+            padding: 25px;
+            margin-bottom: 30px;
+            box-shadow: 0 4px 6px rgba(0,0,0,0.1);
+        }
+        
+        .actions h2 {
+            color: #667eea;
+            margin-bottom: 20px;
+        }
+        
+        .button-group {
+            display: flex;
+            gap: 15px;
+            flex-wrap: wrap;
+        }
+        
+        button {
+            padding: 12px 24px;
+            border: none;
+            border-radius: 5px;
+            font-size: 1em;
+            cursor: pointer;
+            transition: all 0.3s;
+            font-weight: 600;
+        }
+        
+        button:disabled {
+            opacity: 0.5;
+            cursor: not-allowed;
+        }
+        
+        .btn-primary {
+            background: #667eea;
+            color: white;
+        }
+        
+        .btn-primary:hover:not(:disabled) {
+            background: #5568d3;
+        }
+        
+        .btn-success {
+            background: #27ae60;
+            color: white;
+        }
+        
+        .btn-success:hover:not(:disabled) {
+            background: #229954;
+        }
+        
+        .btn-danger {
+            background: #e74c3c;
+            color: white;
+        }
+        
+        .btn-danger:hover:not(:disabled) {
+            background: #c0392b;
+        }
+        
+        .btn-warning {
+            background: #f39c12;
+            color: white;
+        }
+        
+        .btn-warning:hover:not(:disabled) {
+            background: #e67e22;
+        }
+        
+        .failures-section {
+            background: white;
+            border-radius: 10px;
+            padding: 25px;
+            box-shadow: 0 4px 6px rgba(0,0,0,0.1);
+        }
+        
+        .failures-section h2 {
+            color: #667eea;
+            margin-bottom: 20px;
+        }
+        
+        .failure-item {
+            background: #f8f9fa;
+            border-left: 4px solid #e74c3c;
+            padding: 15px;
+            margin-bottom: 15px;
+            border-radius: 5px;
+        }
+        
+        .failure-item .header {
+            display: flex;
+            justify-content: space-between;
+            align-items: center;
+            margin-bottom: 10px;
+        }
+        
+        .failure-item .filename {
+            font-weight: bold;
+            color: #333;
+            font-size: 1.1em;
+        }
+        
+        .failure-item .time {
+            color: #7f8c8d;
+            font-size: 0.9em;
+        }
+        
+        .failure-item .details {
+            display: grid;
+            grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
+            gap: 10px;
+            margin-bottom: 10px;
+        }
+        
+        .failure-item .detail {
+            font-size: 0.9em;
+        }
+        
+        .failure-item .detail strong {
+            color: #667eea;
+        }
+        
+        .failure-item .error {
+            background: #fee;
+            padding: 10px;
+            border-radius: 5px;
+            font-family: monospace;
+            font-size: 0.85em;
+            color: #c0392b;
+            margin-bottom: 10px;
+        }
+        
+        .failure-item .actions {
+            display: flex;
+            gap: 10px;
+            padding: 0;
+            background: transparent;
+            box-shadow: none;
+            margin: 0;
+        }
+        
+        .failure-item button {
+            padding: 8px 16px;
+            font-size: 0.9em;
+        }
+        
+        .loading {
+            text-align: center;
+            padding: 40px;
+            color: white;
+            font-size: 1.2em;
+        }
+        
+        .error-message {
+            background: #e74c3c;
+            color: white;
+            padding: 15px;
+            border-radius: 5px;
+            margin-bottom: 20px;
+            display: none;
+        }
+        
+        .success-message {
+            background: #27ae60;
+            color: white;
+            padding: 15px;
+            border-radius: 5px;
+            margin-bottom: 20px;
+            display: none;
+        }
+        
+        .last-updated {
+            text-align: center;
+            color: white;
+            margin-top: 20px;
+            font-size: 0.9em;
+        }
+        
+        .category-badge {
+            display: inline-block;
+            padding: 4px 8px;
+            border-radius: 3px;
+            font-size: 0.8em;
+            font-weight: bold;
+            margin-right: 5px;
+        }
+        
+        .category-transient { background: #3498db; color: white; }
+        .category-validation { background: #e74c3c; color: white; }
+        .category-processing { background: #f39c12; color: white; }
+        .category-infrastructure { background: #95a5a6; color: white; }
+        .category-partner_api { background: #9b59b6; color: white; }
+        .category-unknown { background: #7f8c8d; color: white; }
+    </style>
+</head>
+<body>
+    <div class="container">
+        <h1>🔄 Partner DLQ Monitor</h1>
+        
+        <div id="error-message" class="error-message"></div>
+        <div id="success-message" class="success-message"></div>
+        
+        <div class="stats-grid">
+            <div class="stat-card danger">
+                <h3>DLQ Messages</h3>
+                <div class="value" id="dlq-count">-</div>
+            </div>
+            <div class="stat-card danger">
+                <h3>Failed Tasks</h3>
+                <div class="value" id="failed-count">-</div>
+            </div>
+            <div class="stat-card warning">
+                <h3>Processing</h3>
+                <div class="value" id="processing-count">-</div>
+            </div>
+            <div class="stat-card">
+                <h3>Downloaded</h3>
+                <div class="value" id="downloaded-count">-</div>
+            </div>
+            <div class="stat-card success">
+                <h3>Processed</h3>
+                <div class="value" id="processed-count">-</div>
+            </div>
+            <div class="stat-card">
+                <h3>Archived</h3>
+                <div class="value" id="archived-count">-</div>
+            </div>
+        </div>
+        
+        <div class="actions">
+            <h2>Actions</h2>
+            <div class="button-group">
+                <button class="btn-primary" onclick="refreshStats()">🔄 Refresh</button>
+                <button class="btn-success" onclick="processDLQ(false)" id="process-btn">⚙️ Process DLQ</button>
+                <button class="btn-warning" onclick="processDLQ(true)">🔍 Dry Run</button>
+                <button class="btn-danger" onclick="purgeDLQ()">🗑️ Purge DLQ</button>
+            </div>
+        </div>
+        
+        <div class="failures-section">
+            <h2>Recent Failures</h2>
+            <div id="failures-list">
+                <div class="loading">Loading...</div>
+            </div>
+        </div>
+        
+        <div class="last-updated">
+            Last updated: <span id="last-updated">Never</span>
+        </div>
+    </div>
+    
+    <script>
+        let refreshInterval;
+        
+        // Auto-refresh every 30 seconds
+        function startAutoRefresh() {
+            refreshInterval = setInterval(refreshStats, 30000);
+        }
+        
+        function stopAutoRefresh() {
+            if (refreshInterval) {
+                clearInterval(refreshInterval);
+            }
+        }
+        
+        async function refreshStats() {
+            try {
+                const response = await fetch('/api/partners/dlq/stats');
+                const data = await response.json();
+                
+                // Update stats
+                document.getElementById('dlq-count').textContent = data.dlq.messageCount;
+                document.getElementById('failed-count').textContent = data.trackers.failed;
+                document.getElementById('processing-count').textContent = data.trackers.processing;
+                document.getElementById('downloaded-count').textContent = data.trackers.downloaded;
+                document.getElementById('processed-count').textContent = data.trackers.processed;
+                document.getElementById('archived-count').textContent = data.trackers.archived;
+                
+                // Update failures list
+                const failuresList = document.getElementById('failures-list');
+                if (data.recentFailures.length === 0) {
+                    failuresList.innerHTML = '<p style="text-align:center;color:#7f8c8d;">No recent failures</p>';
+                } else {
+                    failuresList.innerHTML = data.recentFailures.map(failure => {
+                        const category = categorizeError(failure.errorMessage);
+                        return `
+                            <div class="failure-item">
+                                <div class="header">
+                                    <span class="filename">${failure.logFileName}</span>
+                                    <span class="time">${new Date(failure.failedAt).toLocaleString()}</span>
+                                </div>
+                                <div class="details">
+                                    <div class="detail"><strong>Partner:</strong> ${failure.partner?.name || 'N/A'} (${failure.partner?.code || 'N/A'})</div>
+                                    <div class="detail"><strong>Customer:</strong> ${failure.customer?.name || 'N/A'}</div>
+                                    <div class="detail"><strong>Retries:</strong> ${failure.retryCount}</div>
+                                    <div class="detail"><span class="category-badge category-${category}">${category}</span></div>
+                                </div>
+                                <div class="error">${failure.errorMessage || 'No error message'}</div>
+                                <div class="actions">
+                                    <button class="btn-success" onclick="retryTask('${failure.id}')">🔄 Retry</button>
+                                    <button class="btn-danger" onclick="archiveTask('${failure.id}')">📦 Archive</button>
+                                </div>
+                            </div>
+                        `;
+                    }).join('');
+                }
+                
+                document.getElementById('last-updated').textContent = new Date().toLocaleString();
+                
+            } catch (error) {
+                showError('Failed to refresh stats: ' + error.message);
+            }
+        }
+        
+        async function processDLQ(dryRun = false) {
+            const btn = document.getElementById('process-btn');
+            btn.disabled = true;
+            btn.textContent = '⏳ Processing...';
+            
+            try {
+                const response = await fetch('/api/partners/dlq/process', {
+                    method: 'POST',
+                    headers: { 'Content-Type': 'application/json' },
+                    body: JSON.stringify({ dryRun, maxMessages: 100 })
+                });
+                
+                const data = await response.json();
+                
+                if (dryRun) {
+                    showSuccess(`Dry run completed: ${data.processed} analyzed. ` +
+                        `Categorization: ${JSON.stringify(data.categorization)}`);
+                } else {
+                    showSuccess(`Processed ${data.processed} messages. ` +
+                        `Retried: ${data.retried}, Archived: ${data.archived}`);
+                }
+                
+                await refreshStats();
+                
+            } catch (error) {
+                showError('Failed to process DLQ: ' + error.message);
+            } finally {
+                btn.disabled = false;
+                btn.textContent = '⚙️ Process DLQ';
+            }
+        }
+        
+        async function retryTask(id) {
+            if (!confirm('Retry this task?')) return;
+            
+            try {
+                const response = await fetch(`/api/partners/dlq/retry/${id}`, {
+                    method: 'POST'
+                });
+                
+                const data = await response.json();
+                showSuccess(data.message);
+                await refreshStats();
+                
+            } catch (error) {
+                showError('Failed to retry task: ' + error.message);
+            }
+        }
+        
+        async function archiveTask(id) {
+            const reason = prompt('Archive reason (optional):');
+            if (reason === null) return;
+            
+            try {
+                const response = await fetch(`/api/partners/dlq/archive/${id}`, {
+                    method: 'POST',
+                    headers: { 'Content-Type': 'application/json' },
+                    body: JSON.stringify({ reason: reason || 'Manually archived' })
+                });
+                
+                const data = await response.json();
+                showSuccess(data.message);
+                await refreshStats();
+                
+            } catch (error) {
+                showError('Failed to archive task: ' + error.message);
+            }
+        }
+        
+        async function purgeDLQ() {
+            if (!confirm('⚠️ WARNING: This will permanently delete ALL messages from the DLQ. Are you sure?')) {
+                return;
+            }
+            
+            if (!confirm('⚠️ FINAL CONFIRMATION: This action cannot be undone!')) {
+                return;
+            }
+            
+            try {
+                const response = await fetch('/api/partners/dlq/purge', {
+                    method: 'DELETE',
+                    headers: { 'Content-Type': 'application/json' },
+                    body: JSON.stringify({ confirm: true })
+                });
+                
+                const data = await response.json();
+                showSuccess(data.message);
+                await refreshStats();
+                
+            } catch (error) {
+                showError('Failed to purge DLQ: ' + error.message);
+            }
+        }
+        
+        function categorizeError(errorMessage) {
+            if (!errorMessage) return 'unknown';
+            
+            const msg = errorMessage.toLowerCase();
+            
+            if (msg.includes('timeout') || msg.includes('econnrefused') || msg.includes('network')) {
+                return 'transient';
+            }
+            if (msg.includes('validation') || msg.includes('invalid') || msg.includes('required')) {
+                return 'validation';
+            }
+            if (msg.includes('parse') || msg.includes('calculation') || msg.includes('processing')) {
+                return 'processing';
+            }
+            if (msg.includes('database') || msg.includes('mongo') || msg.includes('filesystem')) {
+                return 'infrastructure';
+            }
+            if (msg.includes('api') || msg.includes('authentication') || msg.includes('unauthorized')) {
+                return 'partner_api';
+            }
+            
+            return 'unknown';
+        }
+        
+        function showError(message) {
+            const el = document.getElementById('error-message');
+            el.textContent = message;
+            el.style.display = 'block';
+            setTimeout(() => el.style.display = 'none', 5000);
+        }
+        
+        function showSuccess(message) {
+            const el = document.getElementById('success-message');
+            el.textContent = message;
+            el.style.display = 'block';
+            setTimeout(() => el.style.display = 'none', 5000);
+        }
+        
+        // Initialize
+        refreshStats();
+        startAutoRefresh();
+        
+        // Cleanup on page unload
+        window.addEventListener('beforeunload', stopAutoRefresh);
+    </script>
+</body>
+</html>
--- a/Development/server/routes/index.js
+++ b/Development/server/routes/index.js
@ -25,12 +25,9 @@ module.exports = function (app) {
  require('./billing')(app);
  require('./subscription')(app);
  require('./invoice')(app),
-  require('./invoice_settings')(app);
+    require('./invoice_settings')(app);
  require('./costing_items')(app);
  require('./log_payment')(app);
  require('./partner')(app);
  require('./health')(app);
-  // Data Export public API (X-API-Key auth) and key management (JWT auth)
-  require('./api_pub')(app);
-  require('./api_keys')(app);
 };
				`@ -1 +0,0 @@`
				`jobId,orderNumber,jobName,sessionId,fileName,pilotName,timestampUtc,gpsTime,lat,lon,utmX,utmY,alt_m,groundSpeed_ms,heading,crossTrackError_m,lockedLine,hdop,satsInView,correctionId,waasId,sprayStat,flowRateApplied_Lmin,flowRateRequired_Lmin,appRateRequired_Lha,appRateApplied_Lha,swathWidth_m,boomPressure_psi,sprayOnLag_s,sprayOffLag_s,pulsesPerLitre,windSpeed_ms,windDir_deg,temp_c,humidity_pct`