387 lines
13 KiB
Python
387 lines
13 KiB
Python
"""
|
|
Complexity Model Tuner
|
|
|
|
Implements feedback loop and retraining for the complexity analysis model.
|
|
Collects actual vs predicted agent counts and adjusts the model over time.
|
|
"""
|
|
|
|
import json
|
|
import time
|
|
from typing import Dict, List, Optional, Tuple
|
|
from dataclasses import dataclass, asdict
|
|
from datetime import datetime, timedelta
|
|
import redis
|
|
import logging
|
|
from collections import defaultdict
|
|
import statistics
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
@dataclass
|
|
class ComplexityPrediction:
|
|
"""Record of a complexity prediction"""
|
|
task_id: str
|
|
timestamp: float
|
|
task_description: str
|
|
predicted_agents: int
|
|
predicted_complexity: str # "simple", "moderate", "complex"
|
|
confidence: float
|
|
model_version: str
|
|
|
|
|
|
@dataclass
|
|
class ComplexityActual:
|
|
"""Actual outcome of a task"""
|
|
task_id: str
|
|
timestamp: float
|
|
actual_agents: int
|
|
actual_duration_seconds: float
|
|
success: bool
|
|
user_feedback: Optional[str] = None # "too_many", "too_few", "just_right"
|
|
user_rating: Optional[int] = None # 1-5 scale
|
|
|
|
|
|
@dataclass
|
|
class ModelMetrics:
|
|
"""Model performance metrics"""
|
|
model_version: str
|
|
total_predictions: int
|
|
mean_absolute_error: float
|
|
accuracy_within_1: float # % predictions within ±1 agent
|
|
accuracy_within_2: float # % predictions within ±2 agents
|
|
user_satisfaction: float # Average user rating
|
|
last_updated: float
|
|
|
|
|
|
class ComplexityModelTuner:
|
|
"""Manages complexity model feedback and tuning"""
|
|
|
|
def __init__(self, redis_client: redis.Redis):
|
|
self.redis = redis_client
|
|
self.current_model_version = "v1.0"
|
|
self.data_ttl = 86400 * 30 # 30 days
|
|
|
|
def record_prediction(self, prediction: ComplexityPrediction) -> bool:
|
|
"""Record a complexity prediction"""
|
|
try:
|
|
key = f"prediction:{prediction.task_id}"
|
|
data = json.dumps(asdict(prediction))
|
|
self.redis.setex(key, self.data_ttl, data)
|
|
|
|
# Add to predictions list
|
|
list_key = f"predictions:{prediction.model_version}"
|
|
self.redis.lpush(list_key, prediction.task_id)
|
|
self.redis.expire(list_key, self.data_ttl)
|
|
|
|
logger.info(
|
|
f"Recorded prediction for task {prediction.task_id}: "
|
|
f"{prediction.predicted_agents} agents"
|
|
)
|
|
return True
|
|
except Exception as e:
|
|
logger.error(f"Failed to record prediction: {e}")
|
|
return False
|
|
|
|
def record_actual(self, actual: ComplexityActual) -> bool:
|
|
"""Record actual task outcome"""
|
|
try:
|
|
key = f"actual:{actual.task_id}"
|
|
data = json.dumps(asdict(actual))
|
|
self.redis.setex(key, self.data_ttl, data)
|
|
|
|
# Add to actuals list
|
|
list_key = "actuals:all"
|
|
self.redis.lpush(list_key, actual.task_id)
|
|
self.redis.expire(list_key, self.data_ttl)
|
|
|
|
logger.info(
|
|
f"Recorded actual for task {actual.task_id}: "
|
|
f"{actual.actual_agents} agents, feedback: {actual.user_feedback}"
|
|
)
|
|
return True
|
|
except Exception as e:
|
|
logger.error(f"Failed to record actual: {e}")
|
|
return False
|
|
|
|
def get_prediction(self, task_id: str) -> Optional[ComplexityPrediction]:
|
|
"""Retrieve a prediction"""
|
|
try:
|
|
key = f"prediction:{task_id}"
|
|
data = self.redis.get(key)
|
|
if not data:
|
|
return None
|
|
return ComplexityPrediction(**json.loads(data))
|
|
except Exception as e:
|
|
logger.error(f"Failed to get prediction: {e}")
|
|
return None
|
|
|
|
def get_actual(self, task_id: str) -> Optional[ComplexityActual]:
|
|
"""Retrieve actual outcome"""
|
|
try:
|
|
key = f"actual:{task_id}"
|
|
data = self.redis.get(key)
|
|
if not data:
|
|
return None
|
|
return ComplexityActual(**json.loads(data))
|
|
except Exception as e:
|
|
logger.error(f"Failed to get actual: {e}")
|
|
return None
|
|
|
|
def calculate_metrics(
|
|
self,
|
|
model_version: Optional[str] = None,
|
|
limit: int = 1000
|
|
) -> Optional[ModelMetrics]:
|
|
"""Calculate model performance metrics"""
|
|
if model_version is None:
|
|
model_version = self.current_model_version
|
|
|
|
try:
|
|
# Get predictions for this model version
|
|
list_key = f"predictions:{model_version}"
|
|
task_ids = self.redis.lrange(list_key, 0, limit - 1)
|
|
|
|
if not task_ids:
|
|
logger.warning(f"No predictions found for model {model_version}")
|
|
return None
|
|
|
|
errors = []
|
|
within_1 = 0
|
|
within_2 = 0
|
|
ratings = []
|
|
|
|
for task_id_bytes in task_ids:
|
|
task_id = task_id_bytes.decode('utf-8')
|
|
|
|
prediction = self.get_prediction(task_id)
|
|
actual = self.get_actual(task_id)
|
|
|
|
if not prediction or not actual:
|
|
continue
|
|
|
|
# Calculate error
|
|
error = abs(prediction.predicted_agents - actual.actual_agents)
|
|
errors.append(error)
|
|
|
|
# Check accuracy thresholds
|
|
if error <= 1:
|
|
within_1 += 1
|
|
if error <= 2:
|
|
within_2 += 1
|
|
|
|
# Collect user ratings
|
|
if actual.user_rating:
|
|
ratings.append(actual.user_rating)
|
|
|
|
if not errors:
|
|
logger.warning(f"No matched predictions/actuals for model {model_version}")
|
|
return None
|
|
|
|
total = len(errors)
|
|
mae = statistics.mean(errors)
|
|
acc_1 = (within_1 / total) * 100
|
|
acc_2 = (within_2 / total) * 100
|
|
avg_rating = statistics.mean(ratings) if ratings else 0.0
|
|
|
|
metrics = ModelMetrics(
|
|
model_version=model_version,
|
|
total_predictions=total,
|
|
mean_absolute_error=mae,
|
|
accuracy_within_1=acc_1,
|
|
accuracy_within_2=acc_2,
|
|
user_satisfaction=avg_rating,
|
|
last_updated=time.time()
|
|
)
|
|
|
|
# Cache metrics
|
|
metrics_key = f"metrics:{model_version}"
|
|
self.redis.setex(metrics_key, 3600, json.dumps(asdict(metrics)))
|
|
|
|
logger.info(
|
|
f"Model {model_version} metrics: MAE={mae:.2f}, "
|
|
f"Acc±1={acc_1:.1f}%, Acc±2={acc_2:.1f}%, "
|
|
f"Satisfaction={avg_rating:.2f}/5"
|
|
)
|
|
|
|
return metrics
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to calculate metrics: {e}")
|
|
return None
|
|
|
|
def get_error_patterns(self, limit: int = 100) -> Dict[str, List[Tuple[str, int, int]]]:
|
|
"""
|
|
Analyze error patterns to identify systematic biases
|
|
|
|
Returns dict with categories:
|
|
- overestimated: tasks where we predicted too many agents
|
|
- underestimated: tasks where we predicted too few agents
|
|
- accurate: tasks where prediction was close
|
|
"""
|
|
patterns = {
|
|
"overestimated": [],
|
|
"underestimated": [],
|
|
"accurate": []
|
|
}
|
|
|
|
try:
|
|
list_key = "actuals:all"
|
|
task_ids = self.redis.lrange(list_key, 0, limit - 1)
|
|
|
|
for task_id_bytes in task_ids:
|
|
task_id = task_id_bytes.decode('utf-8')
|
|
|
|
prediction = self.get_prediction(task_id)
|
|
actual = self.get_actual(task_id)
|
|
|
|
if not prediction or not actual:
|
|
continue
|
|
|
|
error = prediction.predicted_agents - actual.actual_agents
|
|
|
|
entry = (
|
|
task_id,
|
|
prediction.predicted_agents,
|
|
actual.actual_agents
|
|
)
|
|
|
|
if error > 1:
|
|
patterns["overestimated"].append(entry)
|
|
elif error < -1:
|
|
patterns["underestimated"].append(entry)
|
|
else:
|
|
patterns["accurate"].append(entry)
|
|
|
|
return patterns
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to analyze error patterns: {e}")
|
|
return patterns
|
|
|
|
def get_feedback_summary(self, limit: int = 100) -> Dict[str, int]:
|
|
"""Summarize user feedback"""
|
|
feedback_counts = defaultdict(int)
|
|
|
|
try:
|
|
list_key = "actuals:all"
|
|
task_ids = self.redis.lrange(list_key, 0, limit - 1)
|
|
|
|
for task_id_bytes in task_ids:
|
|
task_id = task_id_bytes.decode('utf-8')
|
|
actual = self.get_actual(task_id)
|
|
|
|
if actual and actual.user_feedback:
|
|
feedback_counts[actual.user_feedback] += 1
|
|
|
|
return dict(feedback_counts)
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to get feedback summary: {e}")
|
|
return {}
|
|
|
|
def generate_tuning_recommendations(self) -> List[str]:
|
|
"""Generate recommendations for model tuning based on data"""
|
|
recommendations = []
|
|
|
|
try:
|
|
# Get current metrics
|
|
metrics = self.calculate_metrics()
|
|
if not metrics:
|
|
return ["Insufficient data for recommendations"]
|
|
|
|
# Check accuracy
|
|
if metrics.accuracy_within_1 < 60:
|
|
recommendations.append(
|
|
f"Low accuracy ({metrics.accuracy_within_1:.1f}%). "
|
|
"Consider retraining with more diverse examples."
|
|
)
|
|
|
|
# Check error patterns
|
|
patterns = self.get_error_patterns()
|
|
overestimated = len(patterns["overestimated"])
|
|
underestimated = len(patterns["underestimated"])
|
|
total = overestimated + underestimated + len(patterns["accurate"])
|
|
|
|
if total > 0:
|
|
over_pct = (overestimated / total) * 100
|
|
under_pct = (underestimated / total) * 100
|
|
|
|
if over_pct > 40:
|
|
recommendations.append(
|
|
f"Model overestimates in {over_pct:.1f}% of cases. "
|
|
"Consider reducing base agent count or adjusting complexity thresholds."
|
|
)
|
|
|
|
if under_pct > 40:
|
|
recommendations.append(
|
|
f"Model underestimates in {under_pct:.1f}% of cases. "
|
|
"Consider increasing base agent count or lowering complexity thresholds."
|
|
)
|
|
|
|
# Check user satisfaction
|
|
if metrics.user_satisfaction < 3.5:
|
|
recommendations.append(
|
|
f"Low user satisfaction ({metrics.user_satisfaction:.1f}/5). "
|
|
"Review user feedback and adjust model accordingly."
|
|
)
|
|
|
|
# Check feedback
|
|
feedback = self.get_feedback_summary()
|
|
if feedback.get("too_many", 0) > feedback.get("too_few", 0) * 2:
|
|
recommendations.append(
|
|
"Users frequently report 'too many agents'. "
|
|
"Consider reducing default agent counts."
|
|
)
|
|
elif feedback.get("too_few", 0) > feedback.get("too_many", 0) * 2:
|
|
recommendations.append(
|
|
"Users frequently report 'too few agents'. "
|
|
"Consider increasing default agent counts."
|
|
)
|
|
|
|
if not recommendations:
|
|
recommendations.append(
|
|
f"Model performing well (MAE={metrics.mean_absolute_error:.2f}, "
|
|
f"Acc±1={metrics.accuracy_within_1:.1f}%). Continue monitoring."
|
|
)
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to generate recommendations: {e}")
|
|
recommendations.append(f"Error generating recommendations: {e}")
|
|
|
|
return recommendations
|
|
|
|
def export_training_data(self, limit: int = 1000) -> List[Dict]:
|
|
"""Export prediction/actual pairs for model retraining"""
|
|
training_data = []
|
|
|
|
try:
|
|
list_key = "actuals:all"
|
|
task_ids = self.redis.lrange(list_key, 0, limit - 1)
|
|
|
|
for task_id_bytes in task_ids:
|
|
task_id = task_id_bytes.decode('utf-8')
|
|
|
|
prediction = self.get_prediction(task_id)
|
|
actual = self.get_actual(task_id)
|
|
|
|
if not prediction or not actual:
|
|
continue
|
|
|
|
training_data.append({
|
|
"task_description": prediction.task_description,
|
|
"predicted_agents": prediction.predicted_agents,
|
|
"actual_agents": actual.actual_agents,
|
|
"duration_seconds": actual.actual_duration_seconds,
|
|
"success": actual.success,
|
|
"user_feedback": actual.user_feedback,
|
|
"user_rating": actual.user_rating
|
|
})
|
|
|
|
logger.info(f"Exported {len(training_data)} training examples")
|
|
return training_data
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to export training data: {e}")
|
|
return []
|