updtes sim-to-real experiments (regression)

lasgroup · Oct 19, 2023 · c82166f · c82166f
1 parent 2710e62
commit c82166f
Show file tree

Hide file tree

Showing 3 changed files with 38 additions and 25 deletions.
diff --git a/experiments/sim_real_transfer_exp/inspect_results.py b/experiments/sim_real_transfer_exp/inspect_results.py
@@ -43,6 +43,10 @@ def different_method_plot(df_agg: pd.DataFrame, metric: str = 'nll', display: bo
 def main(args, drop_nan=False):
     df_full, param_names = collect_exp_results(exp_name=args.exp_name)
     df_full = df_full[df_full['data_source'] == args.data_source]
+    #df_full = df_full[df_full['lea'] == args.data_source]
+
+    df_full['mae_pose'] = (df_full['per_dim_metrics/mae_0'] + df_full['per_dim_metrics/mae_1'] +
+                           0.5 * (df_full['per_dim_metrics/mae_2'] + 0.5 * df_full['per_dim_metrics/mae_3'])) / 3
 
     for col in ['bandwidth_kde', 'bandwidth_ssge', 'bandwidth_score_estim']:
         if col in df_full.columns:
@@ -92,13 +96,13 @@ def main(args, drop_nan=False):
     print('Models:', set(df_agg['model']))
 
     different_method_plot(df_agg, metric='nll')
-    different_method_plot(df_agg, metric='rmse')
+    different_method_plot(df_agg, metric='mae_pose')
 
-    df_method = df_agg[(df_agg['model'] == 'BNN_FSVGD_SimPrior_gp')]
+    df_method = df_agg[(df_agg['model'] == 'BNN_FSVGD_SimPrior_gp_no_add_gp')]
 
     #df_method = df_method[df_method['bandwidth_score_estim'] > 0.1]
 
-    metric = 'rmse'
+    metric = 'nll'
     for param in ['num_measurement_points', 'added_gp_lengthscale', 'added_gp_outputscale']:# ['num_f_samples', 'bandwidth_score_estim', 'bandwidth_svgd', 'num_measurement_points']:
         plt.scatter(df_method[param], df_method[(metric, 'mean')])
         plt.xlabel(param)
@@ -166,7 +170,7 @@ def main(args, drop_nan=False):
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser(description='Inspect results of a regression experiment.')
-    parser.add_argument('--exp_name', type=str, default='aug21_str')
-    parser.add_argument('--data_source', type=str, default='real_racecar')
+    parser.add_argument('--exp_name', type=str, default='sep12_str_std')
+    parser.add_argument('--data_source', type=str, default='real_racecar_new')
     args = parser.parse_args()
     main(args)
diff --git a/experiments/sim_real_transfer_exp/run_regression_exp.py b/experiments/sim_real_transfer_exp/run_regression_exp.py
@@ -28,14 +28,15 @@
     'swish': jax.nn.swish,
 }
 
-OUTPUTSCALES_RCCAR = [0.003, 0.003, 0.004, 0.004, 0.021, 0.021, 0.11]
+OUTPUTSCALES_RCCAR = [0.008, 0.008, 0.009, 0.009, 0.05, 0.05, 0.2]
 
 
 def regression_experiment(
                           # data parameters
                           data_source: str,
                           num_samples_train: int,
                           data_seed: int = 981648,
+                          pred_diff: bool = False,
 
                           # logging parameters
                           use_wandb: bool = False,
@@ -53,6 +54,7 @@ def regression_experiment(
                           normalize_likelihood_std: bool = False,
                           learn_likelihood_std: bool = False,
                           likelihood_exponent: float = 1.0,
+                          likelihood_reg: float = 0.0,
 
                           # SVGD parameters
                           num_particles: int = 20,
@@ -79,7 +81,8 @@ def regression_experiment(
     # provide data and sim
     x_train, y_train, x_test, y_test, sim_lf = provide_data_and_sim(
         data_source=data_source,
-        data_spec={'num_samples_train': num_samples_train},
+        data_spec={'num_samples_train': num_samples_train,
+                   'pred_diff': bool(pred_diff)},
         data_seed=data_seed)
 
     if model.endswith('_no_add_gp'):
@@ -123,12 +126,14 @@ def regression_experiment(
                          bandwidth_svgd=bandwidth_svgd,
                          weight_prior_std=weight_prior_std,
                          bias_prior_std=bias_prior_std,
+                         likelihood_reg=likelihood_reg,
                          **standard_model_params)
     elif model == 'BNN_FSVGD':
         model = BNN_FSVGD(domain=sim.domain,
                           num_particles=num_particles,
                           bandwidth_svgd=bandwidth_svgd,
                           bandwidth_gp_prior=bandwidth_gp_prior,
+                          likelihood_reg=likelihood_reg,
                           num_measurement_points=num_measurement_points,
                           **standard_model_params)
     elif 'BNN_FSVGD_SimPrior' in model:
@@ -169,7 +174,7 @@ def regression_experiment(
     model.fit(x_train, y_train, x_test, y_test, log_to_wandb=use_wandb, log_period=1000)
 
     # eval model
-    eval_metrics = model.eval(x_test, y_test)
+    eval_metrics = model.eval(x_test, y_test, per_dim_metrics=True)
     return eval_metrics
 
 
@@ -267,15 +272,17 @@ def main(args):
     parser.add_argument('--use_wandb', type=bool, default=False)
 
     # data parameters
-    parser.add_argument('--data_source', type=str, default='real_racecar')
+    parser.add_argument('--data_source', type=str, default='real_racecar_new')
+    parser.add_argument('--pred_diff', type=int, default=1)
     parser.add_argument('--num_samples_train', type=int, default=200)
     parser.add_argument('--data_seed', type=int, default=77698)
 
     # standard BNN parameters
-    parser.add_argument('--model', type=str, default='BNN_FSVGD_SimPrior_gp')
+    parser.add_argument('--model', type=str, default='BNN_FSVGD')
     parser.add_argument('--model_seed', type=int, default=892616)
     parser.add_argument('--likelihood_std', type=float, default=None)
-    parser.add_argument('--learn_likelihood_std', type=int, default=0)
+    parser.add_argument('--learn_likelihood_std', type=int, default=1)
+    parser.add_argument('--likelihood_reg', type=float, default=0.0)
     parser.add_argument('--data_batch_size', type=int, default=8)
     parser.add_argument('--num_train_steps', type=int, default=20000)
     parser.add_argument('--lr', type=float, default=1e-3)

diff --git a/experiments/sim_real_transfer_exp/sweep_regression_exp.py b/experiments/sim_real_transfer_exp/sweep_regression_exp.py
@@ -11,38 +11,39 @@
 MODEL_SPECIFIC_CONFIG = {
     'BNN_SVGD': {
         'bandwidth_svgd': {'distribution': 'log_uniform', 'min': -1., 'max': 4.},
-        'num_train_steps': {'values': [20000, 40000]}
+        'num_train_steps': {'values': [5000, 10000, 20000]},
+        'likelihood_reg': {'distribution': 'log_uniform_10', 'min': 0.0, 'max': 1.5}
     },
     'BNN_FSVGD': {
         'bandwidth_svgd': {'distribution': 'log_uniform_10', 'min': -1.0, 'max': 0.0},
         'bandwidth_gp_prior': {'distribution': 'log_uniform', 'min': -2., 'max': 0.},
-        'num_train_steps': {'values': [20000]},
-        'num_measurement_points': {'values': [16, 32, 64, 128]},
+        'num_train_steps': {'values': [5000, 10000, 20000]},
+        'num_measurement_points': {'values': [64]},
+        'likelihood_reg': {'distribution': 'log_uniform_10', 'min': 0.0, 'max': 1.5},
     },
     'BNN_FSVGD_SimPrior_gp': {
         'bandwidth_svgd': {'distribution': 'log_uniform_10', 'min': -1.0, 'max': 0.0},
         'num_train_steps': {'values': [40000]},
         'num_measurement_points': {'values': [32]},
         'num_f_samples': {'values': [1024]},
-        'added_gp_lengthscale': {'distribution': 'uniform', 'min': 4., 'max': 20.0},   # racecar: 4 - 8
-        #'added_gp_outputscale': {'distribution': 'uniform', 'min': 0.05, 'max': 0.5},  # racecar: 0.2 - 0.8
+        'added_gp_lengthscale': {'distribution': 'log_uniform_10', 'min': -1., 'max': 0.5},   # racecar: 4 - 8
         'added_gp_outputscale': {'distribution': 'uniform', 'min': 0.5, 'max': 4.0},   # racecar: 4 - 8
     },
-    'BNN_FSVGD_SimPrior_ssge': {
+    'BNN_FSVGD_SimPrior_nu-method': {
         'bandwidth_svgd': {'distribution': 'log_uniform_10', 'min': -1.0, 'max': 0.0},
         'num_train_steps': {'values': [40000]},
-        'num_measurement_points': {'values': [8, 16, 32]},
+        'num_measurement_points': {'values': [32]},
         'num_f_samples': {'values': [512]},
-        'bandwidth_score_estim': {'distribution': 'log_uniform_10', 'min': -0.5, 'max': 1.},
+        'bandwidth_score_estim': {'distribution': 'uniform', 'min': 0.8, 'max': 2.0},
+        'added_gp_lengthscale': {'distribution': 'uniform', 'min': 5., 'max': 10.0},
+        'added_gp_outputscale': {'distribution': 'uniform', 'min': 0.5, 'max': 2.0},  # racecar: 4 - 8
     },
-    'BNN_FSVGD_SimPrior_nu-method': {
+    'BNN_FSVGD_SimPrior_ssge': {
         'bandwidth_svgd': {'distribution': 'log_uniform_10', 'min': -1.0, 'max': 0.0},
         'num_train_steps': {'values': [40000]},
-        'num_measurement_points': {'values': [16, 32]},
+        'num_measurement_points': {'values': [8, 16, 32]},
         'num_f_samples': {'values': [512]},
-        'bandwidth_score_estim': {'distribution': 'log_uniform_10', 'min': 0.0, 'max': 0.0},
-        'added_gp_lengthscale': {'distribution': 'uniform', 'min': 5., 'max': 10.0},
-        'added_gp_outputscale': {'distribution': 'uniform', 'min': 0.5, 'max': 2.0},   # racecar: 4 - 8
+        'bandwidth_score_estim': {'distribution': 'log_uniform_10', 'min': -0.5, 'max': 1.},
     },
     'BNN_FSVGD_SimPrior_gp+nu-method': {
         'bandwidth_svgd': {'distribution': 'log_uniform_10', 'min': -1.0, 'max': 0.0},
@@ -81,10 +82,10 @@ def main(args):
 
     sweep_config = {
         'data_source': {'value': args.data_source},
+        'pred_diff': {'value': args.pred_diff},
         'num_samples_train': DATASET_CONFIGS[args.data_source]['num_samples_train'],
         'model': {'value': args.model},
         'learn_likelihood_std': {'value': args.learn_likelihood_std},
-        #'likelihood_std': {'value': None},
         'num_particles': {'value': 20},
         'data_batch_size': {'value': 8},
     }
@@ -135,6 +136,7 @@ def main(args):
 
     # data parameters
     parser.add_argument('--data_source', type=str, default='pendulum')
+    parser.add_argument('--pred_diff', type=int, default=0)
 
     # # standard BNN parameters
     parser.add_argument('--model', type=str, default='BNN_SVGD')