Feature - Kullback-Leibler divergence

api/evaluation/metrics/metrics.py

@@ -136,6 +136,15 @@ def get_chrf_meta(task=None):
     }
 
 
+def get_kullback_leibler_divergence_meta(task=None):
+    return {
+        "unit": "%",
+        "pretty_name": "Kullback-Leibler Divergence",
+        "utility_direction": -1,
+        "offset": 0,
+    }
+
+
 def get_dataperf_fraction_of_fixes(required_fixes, total_fixes):
     fraction_of_fixes = required_fixes / total_fixes
     return fraction_of_fixes
@@ -290,6 +299,15 @@ def get_chrf(predictions: list, targets: list):
     return chrf.score
 
 
+def get_kullback_leibler_divergence(y_true, y_pred, epsilon=0.00001):
+    y = np.clip(y_true, epsilon, 1 - epsilon)
+    x = np.clip(y_pred, epsilon, 1 - epsilon)
+
+    kl = np.sum(y * np.log(y / x), axis=1)
+
+    return np.mean(kl)
+
+
 def get_bleu_meta(task=None):
     return {"unit": "", "pretty_name": "BLEU", "utility_direction": 1, "offset": 0}
 

api/evaluation/metrics/metrics_dicts.py

@@ -23,6 +23,7 @@
     "dataperf_fraction_of_fixes": metrics.get_dataperf_fraction_of_fixes,
     "dataperf_balanced_accuracy": metrics.get_dataperf_balanced_accuracy,
     "chrf": metrics.get_chrf,
+    "kullback_leibler_divergence": metrics.get_kullback_leibler_divergence,
 }
 
 delta_metrics_dict = {
@@ -53,4 +54,5 @@
     "dataperf_fraction_of_fixes": metrics.get_dataperf_fraction_of_fixes_meta,
     "dataperf_balanced_accuracy": metrics.get_dataperf_balanced_accuracy_meta,
     "chrf": metrics.get_chrf_meta,
+    "kullback_leibler_divergence": metrics.get_kullback_leibler_divergence_meta,
 }

backend/app/domain/services/builder_and_evaluation/eval_utils/metrics.py

@@ -166,6 +166,15 @@ def get_chrf_meta(task=None):
     }
 
 
+def get_kullback_leibler_divergence_meta(task=None):
+    return {
+        "unit": "%",
+        "pretty_name": "Kullback-Leibler Divergence",
+        "utility_direction": -1,
+        "offset": 0,
+    }
+
+
 def get_chrf_pp(predictions: list, targets: list):
     """Chrf++ metric.
 

backend/app/domain/services/builder_and_evaluation/eval_utils/metrics_dicts.py

@@ -27,6 +27,7 @@
     get_f1,
     get_f1_meta,
     get_fairness_meta,
+    get_kullback_leibler_divergence_meta,
     get_macro_f1,
     get_macro_f1_meta,
     get_matthews_correlation,
@@ -95,4 +96,5 @@
     "dataperf_fraction_of_fixes": get_dataperf_fraction_of_fixes_meta,
     "dataperf_balanced_accuracy": get_dataperf_balanced_accuracy_meta,
     "chrf": get_chrf_meta,
+    "kullback_leibler_divergence": get_kullback_leibler_divergence_meta,
 }