diff --git a/experiments/sim_real_transfer_exp/inspect_results.py b/experiments/sim_real_transfer_exp/inspect_results.py index 259dc62..a2c2a0a 100644 --- a/experiments/sim_real_transfer_exp/inspect_results.py +++ b/experiments/sim_real_transfer_exp/inspect_results.py @@ -43,6 +43,10 @@ def different_method_plot(df_agg: pd.DataFrame, metric: str = 'nll', display: bo def main(args, drop_nan=False): df_full, param_names = collect_exp_results(exp_name=args.exp_name) df_full = df_full[df_full['data_source'] == args.data_source] + #df_full = df_full[df_full['lea'] == args.data_source] + + df_full['mae_pose'] = (df_full['per_dim_metrics/mae_0'] + df_full['per_dim_metrics/mae_1'] + + 0.5 * (df_full['per_dim_metrics/mae_2'] + 0.5 * df_full['per_dim_metrics/mae_3'])) / 3 for col in ['bandwidth_kde', 'bandwidth_ssge', 'bandwidth_score_estim']: if col in df_full.columns: @@ -92,13 +96,13 @@ def main(args, drop_nan=False): print('Models:', set(df_agg['model'])) different_method_plot(df_agg, metric='nll') - different_method_plot(df_agg, metric='rmse') + different_method_plot(df_agg, metric='mae_pose') - df_method = df_agg[(df_agg['model'] == 'BNN_FSVGD_SimPrior_gp')] + df_method = df_agg[(df_agg['model'] == 'BNN_FSVGD_SimPrior_gp_no_add_gp')] #df_method = df_method[df_method['bandwidth_score_estim'] > 0.1] - metric = 'rmse' + metric = 'nll' for param in ['num_measurement_points', 'added_gp_lengthscale', 'added_gp_outputscale']:# ['num_f_samples', 'bandwidth_score_estim', 'bandwidth_svgd', 'num_measurement_points']: plt.scatter(df_method[param], df_method[(metric, 'mean')]) plt.xlabel(param) @@ -166,7 +170,7 @@ def main(args, drop_nan=False): if __name__ == '__main__': parser = argparse.ArgumentParser(description='Inspect results of a regression experiment.') - parser.add_argument('--exp_name', type=str, default='aug21_str') - parser.add_argument('--data_source', type=str, default='real_racecar') + parser.add_argument('--exp_name', type=str, default='sep12_str_std') + parser.add_argument('--data_source', type=str, default='real_racecar_new') args = parser.parse_args() main(args) \ No newline at end of file diff --git a/experiments/sim_real_transfer_exp/run_regression_exp.py b/experiments/sim_real_transfer_exp/run_regression_exp.py index a71a709..fb8db9c 100644 --- a/experiments/sim_real_transfer_exp/run_regression_exp.py +++ b/experiments/sim_real_transfer_exp/run_regression_exp.py @@ -28,7 +28,7 @@ 'swish': jax.nn.swish, } -OUTPUTSCALES_RCCAR = [0.003, 0.003, 0.004, 0.004, 0.021, 0.021, 0.11] +OUTPUTSCALES_RCCAR = [0.008, 0.008, 0.009, 0.009, 0.05, 0.05, 0.2] def regression_experiment( @@ -36,6 +36,7 @@ def regression_experiment( data_source: str, num_samples_train: int, data_seed: int = 981648, + pred_diff: bool = False, # logging parameters use_wandb: bool = False, @@ -53,6 +54,7 @@ def regression_experiment( normalize_likelihood_std: bool = False, learn_likelihood_std: bool = False, likelihood_exponent: float = 1.0, + likelihood_reg: float = 0.0, # SVGD parameters num_particles: int = 20, @@ -79,7 +81,8 @@ def regression_experiment( # provide data and sim x_train, y_train, x_test, y_test, sim_lf = provide_data_and_sim( data_source=data_source, - data_spec={'num_samples_train': num_samples_train}, + data_spec={'num_samples_train': num_samples_train, + 'pred_diff': bool(pred_diff)}, data_seed=data_seed) if model.endswith('_no_add_gp'): @@ -123,12 +126,14 @@ def regression_experiment( bandwidth_svgd=bandwidth_svgd, weight_prior_std=weight_prior_std, bias_prior_std=bias_prior_std, + likelihood_reg=likelihood_reg, **standard_model_params) elif model == 'BNN_FSVGD': model = BNN_FSVGD(domain=sim.domain, num_particles=num_particles, bandwidth_svgd=bandwidth_svgd, bandwidth_gp_prior=bandwidth_gp_prior, + likelihood_reg=likelihood_reg, num_measurement_points=num_measurement_points, **standard_model_params) elif 'BNN_FSVGD_SimPrior' in model: @@ -169,7 +174,7 @@ def regression_experiment( model.fit(x_train, y_train, x_test, y_test, log_to_wandb=use_wandb, log_period=1000) # eval model - eval_metrics = model.eval(x_test, y_test) + eval_metrics = model.eval(x_test, y_test, per_dim_metrics=True) return eval_metrics @@ -267,15 +272,17 @@ def main(args): parser.add_argument('--use_wandb', type=bool, default=False) # data parameters - parser.add_argument('--data_source', type=str, default='real_racecar') + parser.add_argument('--data_source', type=str, default='real_racecar_new') + parser.add_argument('--pred_diff', type=int, default=1) parser.add_argument('--num_samples_train', type=int, default=200) parser.add_argument('--data_seed', type=int, default=77698) # standard BNN parameters - parser.add_argument('--model', type=str, default='BNN_FSVGD_SimPrior_gp') + parser.add_argument('--model', type=str, default='BNN_FSVGD') parser.add_argument('--model_seed', type=int, default=892616) parser.add_argument('--likelihood_std', type=float, default=None) - parser.add_argument('--learn_likelihood_std', type=int, default=0) + parser.add_argument('--learn_likelihood_std', type=int, default=1) + parser.add_argument('--likelihood_reg', type=float, default=0.0) parser.add_argument('--data_batch_size', type=int, default=8) parser.add_argument('--num_train_steps', type=int, default=20000) parser.add_argument('--lr', type=float, default=1e-3) diff --git a/experiments/sim_real_transfer_exp/sweep_regression_exp.py b/experiments/sim_real_transfer_exp/sweep_regression_exp.py index 1fa7c74..19614ba 100644 --- a/experiments/sim_real_transfer_exp/sweep_regression_exp.py +++ b/experiments/sim_real_transfer_exp/sweep_regression_exp.py @@ -11,38 +11,39 @@ MODEL_SPECIFIC_CONFIG = { 'BNN_SVGD': { 'bandwidth_svgd': {'distribution': 'log_uniform', 'min': -1., 'max': 4.}, - 'num_train_steps': {'values': [20000, 40000]} + 'num_train_steps': {'values': [5000, 10000, 20000]}, + 'likelihood_reg': {'distribution': 'log_uniform_10', 'min': 0.0, 'max': 1.5} }, 'BNN_FSVGD': { 'bandwidth_svgd': {'distribution': 'log_uniform_10', 'min': -1.0, 'max': 0.0}, 'bandwidth_gp_prior': {'distribution': 'log_uniform', 'min': -2., 'max': 0.}, - 'num_train_steps': {'values': [20000]}, - 'num_measurement_points': {'values': [16, 32, 64, 128]}, + 'num_train_steps': {'values': [5000, 10000, 20000]}, + 'num_measurement_points': {'values': [64]}, + 'likelihood_reg': {'distribution': 'log_uniform_10', 'min': 0.0, 'max': 1.5}, }, 'BNN_FSVGD_SimPrior_gp': { 'bandwidth_svgd': {'distribution': 'log_uniform_10', 'min': -1.0, 'max': 0.0}, 'num_train_steps': {'values': [40000]}, 'num_measurement_points': {'values': [32]}, 'num_f_samples': {'values': [1024]}, - 'added_gp_lengthscale': {'distribution': 'uniform', 'min': 4., 'max': 20.0}, # racecar: 4 - 8 - #'added_gp_outputscale': {'distribution': 'uniform', 'min': 0.05, 'max': 0.5}, # racecar: 0.2 - 0.8 + 'added_gp_lengthscale': {'distribution': 'log_uniform_10', 'min': -1., 'max': 0.5}, # racecar: 4 - 8 'added_gp_outputscale': {'distribution': 'uniform', 'min': 0.5, 'max': 4.0}, # racecar: 4 - 8 }, - 'BNN_FSVGD_SimPrior_ssge': { + 'BNN_FSVGD_SimPrior_nu-method': { 'bandwidth_svgd': {'distribution': 'log_uniform_10', 'min': -1.0, 'max': 0.0}, 'num_train_steps': {'values': [40000]}, - 'num_measurement_points': {'values': [8, 16, 32]}, + 'num_measurement_points': {'values': [32]}, 'num_f_samples': {'values': [512]}, - 'bandwidth_score_estim': {'distribution': 'log_uniform_10', 'min': -0.5, 'max': 1.}, + 'bandwidth_score_estim': {'distribution': 'uniform', 'min': 0.8, 'max': 2.0}, + 'added_gp_lengthscale': {'distribution': 'uniform', 'min': 5., 'max': 10.0}, + 'added_gp_outputscale': {'distribution': 'uniform', 'min': 0.5, 'max': 2.0}, # racecar: 4 - 8 }, - 'BNN_FSVGD_SimPrior_nu-method': { + 'BNN_FSVGD_SimPrior_ssge': { 'bandwidth_svgd': {'distribution': 'log_uniform_10', 'min': -1.0, 'max': 0.0}, 'num_train_steps': {'values': [40000]}, - 'num_measurement_points': {'values': [16, 32]}, + 'num_measurement_points': {'values': [8, 16, 32]}, 'num_f_samples': {'values': [512]}, - 'bandwidth_score_estim': {'distribution': 'log_uniform_10', 'min': 0.0, 'max': 0.0}, - 'added_gp_lengthscale': {'distribution': 'uniform', 'min': 5., 'max': 10.0}, - 'added_gp_outputscale': {'distribution': 'uniform', 'min': 0.5, 'max': 2.0}, # racecar: 4 - 8 + 'bandwidth_score_estim': {'distribution': 'log_uniform_10', 'min': -0.5, 'max': 1.}, }, 'BNN_FSVGD_SimPrior_gp+nu-method': { 'bandwidth_svgd': {'distribution': 'log_uniform_10', 'min': -1.0, 'max': 0.0}, @@ -81,10 +82,10 @@ def main(args): sweep_config = { 'data_source': {'value': args.data_source}, + 'pred_diff': {'value': args.pred_diff}, 'num_samples_train': DATASET_CONFIGS[args.data_source]['num_samples_train'], 'model': {'value': args.model}, 'learn_likelihood_std': {'value': args.learn_likelihood_std}, - #'likelihood_std': {'value': None}, 'num_particles': {'value': 20}, 'data_batch_size': {'value': 8}, } @@ -135,6 +136,7 @@ def main(args): # data parameters parser.add_argument('--data_source', type=str, default='pendulum') + parser.add_argument('--pred_diff', type=int, default=0) # # standard BNN parameters parser.add_argument('--model', type=str, default='BNN_SVGD')