Add generic callback configuration logic

frameworks/AutoGluon/README.md

@@ -4,6 +4,17 @@ To run v0.8.2: ```python3 ../automlbenchmark/runbenchmark.py autogluon ...```
 
 To run mainline: ```python3 ../automlbenchmark/runbenchmark.py autogluon:latest ...```
 
+## Callbacks
+Callbacks from the `autogluon.core.callbacks` module can be configured from the configuration file by specifying the classname and any hyperparameters (see AutoGluonES example below).
+This configuration is case-sensitive.
+```yaml
+AutoGluonES:
+  extends: AutoGluon
+  params:
+    callbacks:
+      EarlyStoppingEnsembleCallback:
+        patience: 5
+```
 
 ## Running with Learning Curves Enabled
 
@@ -18,16 +29,17 @@ To summarize these steps:
 5. For example, if running the most basic framework listed below in the example user_dir, the command would look like: \
     `python3 runbenchmark.py AutoGluon_curves_true -u examples/custom ...`
 
+
 ### Sample Framework Definitions
-```
+```yaml
 # simplest usage
 AutoGluon_curves_true:
   extends: AutoGluon
   params:
     learning_curves: True
     _save_artifacts: ['learning_curves']
 ```
-```
+```yaml
 # including test data
 AutoGluon_curves_test:
   extends: AutoGluon
@@ -36,7 +48,7 @@ AutoGluon_curves_test:
     _include_test_during_fit: True
     _save_artifacts: ['learning_curves']
 ```
-```
+```yaml
 # parameterizing learning_curves dictionary
 AutoGluon_curves_parameters:
   extends: AutoGluon
@@ -50,7 +62,7 @@ AutoGluon_curves_parameters:
     _include_test_during_fit: True
     _save_artifacts: ['learning_curves']
 ```
-```
+```yaml
 # adding custom hyperparameters
 Defaults: &defaults
   NN_TORCH:
@@ -78,4 +90,4 @@ AutoGluon_curves_hyperparameters:
         multiclass: ["accuracy", "precision_weighted", "recall_weighted"]
     _include_test_during_fit: True
     _save_artifacts: ['learning_curves']
-```
+```

frameworks/AutoGluon/exec.py

@@ -48,6 +48,10 @@ def run(dataset, config):
 
     is_classification = config.type == 'classification'
     training_params = {k: v for k, v in config.framework_params.items() if not k.startswith('_')}
+
+    if callback_settings := training_params.get("callbacks", {}):
+        training_params["callbacks"] = initialize_callbacks(callback_settings)
+
     time_limit = config.max_runtime_seconds
     presets = training_params.get("presets", [])
     presets = presets if isinstance(presets, list) else [presets]
@@ -171,6 +175,20 @@ def inference_time_regression(data: Union[str, pd.DataFrame]):
                   inference_times=inference_times,)
 
 
+def initialize_callbacks(callback_settings):
+    callbacks = []
+    try:
+        import autogluon.core.callbacks
+    except ImportError:
+        raise ValueError("Callbacks are only available for AutoGluon>=1.1.2")
+    for callback, hyperparameters in callback_settings.items():
+        callback_cls = getattr(autogluon.core.callbacks, callback, None)
+        if not callback_cls:
+            raise ValueError(f"Callback {callback} is not a valid callback")
+        callbacks.append(callback_cls(**hyperparameters))
+    return callbacks
+
+
 def save_artifacts(predictor, leaderboard, learning_curves, config):
     artifacts = config.framework_params.get('_save_artifacts', ['leaderboard'])
     try: