def _run_training_with_callbacks(self, trainer, train_loader, val_loader, device, config):
"""Run training with batch-level progress updates"""
import torch
import csv
from [Link] import autocast, GradScaler
from datetime import datetime
import threading
scaler = GradScaler() if [Link] else None
optimizer = [Link]
model = [Link]
criterion = [Link]
# Create output directories
vis_dir = Path(config.VIS_DIR)
vis_dir.mkdir(parents=True, exist_ok=True)
log_dir = Path(config.LOG_DIR)
log_dir.mkdir(parents=True, exist_ok=True)
# Create metrics log file
metrics_csv_path = log_dir / "training_metrics.csv"
with open(metrics_csv_path, 'w', newline='') as f:
csv_writer = [Link](f)
csv_writer.writerow(['epoch', 'loss', 'map50', 'best_map', 'learning_rate', 'timestamp'])
cache_clear_freq = MEMORY_CONFIG['empty_cache_frequency']
# ✅ VALIDATION MONITORING VARIABLES
validation_in_progress = False
validation_start_time = None
validation_timeout = 300 # 5 minutes
for epoch_idx in range(self.total_epochs):
if self.stop_requested:
[Link]("Training stopped by user", "WARNING")
break
# Set current epoch
self.current_epoch = epoch_idx + 1
self.epoch_start_time = [Link]()
trainer.current_epoch = self.current_epoch
[Link]()
epoch_loss = 0.0
num_batches = len(train_loader)
[Link](f"\n[Epoch {self.current_epoch}/{self.total_epochs}]", "PROGRESS")
# Training loop
for batch_idx, (images, targets) in enumerate(train_loader):
if self.stop_requested:
break
self.current_batch = batch_idx + 1
try:
images = [Link](device, non_blocking=True)
for target in targets:
for key in target:
if isinstance(target[key], [Link]):
target[key] = target[key].to(device, non_blocking=True)
batch = trainer._prepare_batch(images, targets)
optimizer.zero_grad()
if scaler is not None:
with autocast():
outputs = model(images)
loss, loss_items = criterion(outputs, batch)
[Link](loss).backward()
scaler.unscale_(optimizer)
[Link].clip_grad_norm_([Link](), config.GRADIENT_CLIP)
[Link](optimizer)
[Link]()
else:
outputs = model(images)
loss, loss_items = criterion(outputs, batch)
[Link]()
[Link].clip_grad_norm_([Link](), config.GRADIENT_CLIP)
[Link]()
batch_loss = [Link]()
epoch_loss += batch_loss
if batch_idx % cache_clear_freq == 0:
[Link].empty_cache()
# Update GUI
total_batches = len(train_loader)
batch_progress = ((batch_idx + 1) / total_batches) * 100
elapsed = [Link]() - self.epoch_start_time
batch_per_sec = (batch_idx + 1) / elapsed if elapsed > 0 else 0
remaining_batches = total_batches - (batch_idx + 1)
epoch_eta_secs = remaining_batches / batch_per_sec if batch_per_sec > 0 else 0
epoch_eta_mins = int(epoch_eta_secs // 60)
epoch_eta_secs_rem = int(epoch_eta_secs % 60)
avg_loss = epoch_loss / (batch_idx + 1)
self.batch_queue.put({
'batch': batch_idx + 1,
'total_batches': total_batches,
'batch_loss': batch_loss,
'avg_loss': avg_loss,
'batch_per_sec': batch_per_sec,
'epoch_eta_mins': epoch_eta_mins,
'epoch_eta_secs': epoch_eta_secs_rem,
'progress': batch_progress
})
except RuntimeError as e:
if "out of memory" in str(e):
[Link](f"OOM at batch {batch_idx}, clearing cache...", "WARNING")
[Link].empty_cache()
[Link]()
continue
raise
# End of epoch
avg_epoch_loss = epoch_loss / max(num_batches, 1)
current_lr = optimizer.param_groups[0]['lr']
if hasattr(trainer, 'scheduler') and [Link]:
[Link]()
# ✅ FIXED VALIDATION WITH TIMEOUT MONITORING
current_map = 0.0
if self.current_epoch % config.VAL_INTERVAL == 0:
[Link]("Running validation...", "PROGRESS")
# ✅ RUN VALIDATION IN SEPARATE THREAD WITH MONITORING
validation_result = {'completed': False, 'val_losses': None, 'metrics': None, 'error': None}
def run_validation():
try:
val_losses, metrics = [Link]()
validation_result['val_losses'] = val_losses
validation_result['metrics'] = metrics
validation_result['completed'] = True
except Exception as e:
validation_result['error'] = str(e)
validation_result['completed'] = True
# Start validation in thread
validation_thread = [Link](target=run_validation, daemon=True)
validation_thread.start()
# ✅ MONITOR VALIDATION WITH TIMEOUT
validation_start_time = [Link]()
validation_timeout = 300 # 5 minutes
while validation_thread.is_alive():
elapsed = [Link]() - validation_start_time
# Update GUI with validation progress
if int(elapsed) % 10 == 0: # Every 10 seconds
[Link](f"Validation in progress... ({int(elapsed)}s elapsed)", "PROGRESS")
# ✅ CHECK FOR TIMEOUT
if elapsed > validation_timeout:
[Link](f"⚠️VALIDATION TIMEOUT at epoch {self.current_epoch}!", "ERROR")
[Link](f"Validation exceeded {validation_timeout}s", "ERROR")
[Link]("Skipping this validation and continuing...", "WARNING")
# Force cleanup
[Link].empty_cache()
[Link]()
# Skip to next epoch
break
# Sleep briefly to avoid busy-waiting
[Link](0.5)
# ✅ PROCESS VALIDATION RESULTS
if validation_result['completed']:
if validation_result['error']:
[Link](f"Validation error: {validation_result['error']}", "ERROR")
else:
val_losses = validation_result['val_losses']
metrics = validation_result['metrics']
validation_time = [Link]() - validation_start_time
[Link](f"✅ Validation complete in {validation_time:.1f}s")
if validation_time > 120:
[Link](f"⚠️Validation took {validation_time:.1f}s (unusually long)", "WARNING")
current_map = [Link]('mAP50', 0.0)
# Check if best
if current_map > trainer.best_map:
trainer.best_map = current_map
[Link](f"🎯 NEW BEST mAP: {current_map:.4f}!")
trainer.save_checkpoint(is_best=True)
# Save at intervals
if self.current_epoch % config.SAVE_INTERVAL == 0:
trainer.save_checkpoint(is_best=False)
[Link](f"💾 Saved checkpoint at epoch {self.current_epoch}")
[Link].empty_cache()
[Link]()
else:
# Timeout occurred
[Link](f"Continuing to epoch {self.current_epoch + 1}...", "WARNING")
current_map = trainer.best_map # Use previous best
# Log metrics to CSV
try:
timestamp = [Link]().strftime("%Y-%m-%d %H:%M:%S")
with open(metrics_csv_path, 'a', newline='') as f:
csv_writer = [Link](f)
csv_writer.writerow([self.current_epoch, avg_epoch_loss, current_map,
trainer.best_map, current_lr, timestamp])
except Exception as e:
[Link](f"Failed to log metrics: {e}", "WARNING")
# Send metrics to GUI
self.metrics_queue.put({
'epoch': self.current_epoch,
'loss': avg_epoch_loss,
'map': current_map,
'best_map': trainer.best_map
})
[Link](f"Epoch {self.current_epoch}: Loss={avg_epoch_loss:.4f}, mAP={current_map:.4f},
Best={trainer.best_map:.4f}, LR={current_lr:.6f}")
# Cleanup
[Link].empty_cache()
[Link]()
# Training complete
[Link]("=" * 60)
[Link]("✅ TRAINING COMPLETED!")
[Link](f"Best mAP: {trainer.best_map:.4f}")
[Link](f"Checkpoints: {config.CHECKPOINT_DIR}")
[Link](f"Visualizations: {config.VIS_DIR}")
[Link](f"Logs: {config.LOG_DIR}")
[Link]("=" * 60)
[Link](0, lambda: self.status_label.config(text="Completed!", foreground="green"))
[Link](0, lambda: [Link](
"Training Complete",
f"Incremental training finished!\n\nBest mAP: {trainer.best_map:.4f}\n\n"
f"Checkpoints: {config.CHECKPOINT_DIR}\n"
f"Visualizations: {config.VIS_DIR}\n"
f"Logs: {config.LOG_DIR}"
))