1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149
| import json from datetime import datetime
class RAGEvaluator: """RAG系统自动评测器""" def __init__(self, rag_system, eval_dataset): """ Args: rag_system: RAG系统实例 eval_dataset: 评测数据集 """ self.rag_system = rag_system self.dataset = eval_dataset self.metrics = RAGMetrics() self.results = [] def run_evaluation(self, batch_size=10): """运行完整评测""" print("=" * 60) print("开始RAG系统评测") print("=" * 60) predictions = [] ground_truths = [] for i, test_case in enumerate(self.dataset.test_cases): print(f"\r评测进度: {i+1}/{len(self.dataset.test_cases)}", end="", flush=True) prediction = self.rag_system.query(test_case['query']) predictions.append(prediction['answer']) ground_truths.append(test_case['expected_answer']) self.results.append({ 'test_case': test_case, 'prediction': prediction, 'metrics': self._calculate_case_metrics( prediction, test_case ) }) print("\n") overall_metrics = self._calculate_overall_metrics( predictions, ground_truths ) report = self._generate_report(overall_metrics) return report def _calculate_case_metrics(self, prediction, test_case): """计算单个测试用例的指标""" return { 'accuracy': self.metrics.accuracy( [prediction['answer']], [test_case['expected_answer']] ), 'relevance': self.metrics.relevance( prediction.get('retrieved_docs', []), test_case['query'] ), 'completeness': self.metrics.completeness( prediction['answer'], test_case.get('expected_info', []) ) } def _calculate_overall_metrics(self, predictions, ground_truths): """计算整体指标""" return { 'accuracy': self.metrics.accuracy(predictions, ground_truths), 'avg_relevance': np.mean([ r['metrics']['relevance'] for r in self.results ]), 'avg_completeness': np.mean([ r['metrics']['completeness'] for r in self.results ]), 'consistency': self._measure_consistency() } def _measure_consistency(self): """测量一致性(对同一问题多次查询)""" sample_cases = self.dataset.test_cases[:10] consistency_scores = [] for case in sample_cases: responses = [] for _ in range(3): response = self.rag_system.query(case['query']) responses.append(response['answer']) score = self.metrics.consistency(responses) consistency_scores.append(score) return np.mean(consistency_scores) def _generate_report(self, overall_metrics): """生成评测报告""" report = { 'timestamp': datetime.now().isoformat(), 'dataset_size': len(self.dataset.test_cases), 'overall_metrics': overall_metrics, 'detailed_results': self.results, 'summary': self._generate_summary(overall_metrics) } report_path = f"eval_report_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json" with open(report_path, 'w', encoding='utf-8') as f: json.dump(report, f, ensure_ascii=False, indent=2) print(f"评测报告已保存: {report_path}") return report def _generate_summary(self, metrics): """生成评测摘要""" summary = f""" ============================================================ RAG系统评测报告摘要 ============================================================
评测时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')} 测试用例数: {len(self.dataset.test_cases)}
核心指标: - 准确率 (Accuracy): {metrics['accuracy']:.2%} - 平均相关性 (Relevance): {metrics['avg_relevance']:.2%} - 平均完整性 (Completeness): {metrics['avg_completeness']:.2%} - 一致性 (Consistency): {metrics['consistency']:.2%}
总体评价: {'优秀' if metrics['accuracy'] >= 0.9 else '良好' if metrics['accuracy'] >= 0.8 else '需改进'} ============================================================ """ print(summary) return summary
evaluator = RAGEvaluator(rag_system, dataset) report = evaluator.run_evaluation()
|