From 3aa264b8bdcb9026fb9e56a8fde61f01bbe0a46a Mon Sep 17 00:00:00 2001 From: Jonas Rothfuss Date: Wed, 24 Jan 2024 14:31:31 +0100 Subject: [PATCH] updates on the sim_real_exp --- experiments/data_provider.py | 2 +- .../sim_real_transfer_exp/inspect_results.py | 75 +++---------- .../run_regression_exp.py | 54 +++++---- .../sweep_regression_exp.py | 13 ++- .../sweep_regression_exp_num_data.py | 105 ++++++++++-------- 5 files changed, 114 insertions(+), 135 deletions(-) diff --git a/experiments/data_provider.py b/experiments/data_provider.py index 57b785b..f54cf09 100644 --- a/experiments/data_provider.py +++ b/experiments/data_provider.py @@ -203,7 +203,7 @@ def get_rccar_recorded_data_new(encode_angle: bool = True, skip_first_n_points: num_test_points = 20_000 elif dataset in ['v2', 'v3', 'v4']: recordings_dir = [os.path.join(DATA_DIR, f'recordings_rc_car_{dataset}')] - num_test_points = 10_000 + num_test_points = 6_000 else: raise ValueError(f"Unknown dataset {dataset} for car_id {car_id}") else: diff --git a/experiments/sim_real_transfer_exp/inspect_results.py b/experiments/sim_real_transfer_exp/inspect_results.py index a2c2a0a..aa071ba 100644 --- a/experiments/sim_real_transfer_exp/inspect_results.py +++ b/experiments/sim_real_transfer_exp/inspect_results.py @@ -95,82 +95,35 @@ def main(args, drop_nan=False): print('Models:', set(df_agg['model'])) - different_method_plot(df_agg, metric='nll') - different_method_plot(df_agg, metric='mae_pose') + different_method_plot(df_agg, metric='nll', filter_std_higher_than=20.) + different_method_plot(df_agg, metric='rmse') - df_method = df_agg[(df_agg['model'] == 'BNN_FSVGD_SimPrior_gp_no_add_gp')] + df_method = df_agg[(df_agg['model'] == 'BNN_FSVGD_SimPrior_gp')] #df_method = df_method[df_method['bandwidth_score_estim'] > 0.1] metric = 'nll' - for param in ['num_measurement_points', 'added_gp_lengthscale', 'added_gp_outputscale']:# ['num_f_samples', 'bandwidth_score_estim', 'bandwidth_svgd', 'num_measurement_points']: + for param in [ + 'added_gp_lengthscale', + 'added_gp_outputscale', + # 'bandwidth_svgd', + # 'likelihood_reg', + #'bandwidth_svgd' + ]: + # ['num_f_samples', 'bandwidth_score_estim', 'bandwidth_svgd', 'num_measurement_points']: plt.scatter(df_method[param], df_method[(metric, 'mean')]) plt.xlabel(param) plt.xscale('log') plt.ylabel(metric) #plt.scatter(5., -7, color='red') - #plt.ylim(-8, 0) + plt.ylim(-10, -6) plt.show() - plt.scatter(df_method['added_gp_lengthscale'], df_method['added_gp_outputscale'], c=df_method[(metric, 'mean')], - cmap='viridis') - plt.xlabel('added_gp_lengthscale') - plt.ylabel('added_gp_outputscale') - plt.colorbar(label=metric) - plt.show() - - plt.scatter(df_method['added_gp_lengthscale'], df_method['added_gp_outputscale'], s=df_method[(metric, 'mean')]) - - QUANTILE_BASED_CI = True - METHODS = ['random_search', 'hill_search'] - METRICS = ['x_diff', 'f_diff'] - PLOT_N_BEST = 2 - n_metrics = len(METRICS) - - fig, axes = plt.subplots(ncols=n_metrics, nrows=1, figsize=(n_metrics * 4, 4)) - for k, method in enumerate(METHODS): - df_plot = df_agg.loc[df_agg['method'] == method] - df_plot.sort_values(by=('x_diff', 'mean'), ascending=True, inplace=True) - - if df_plot.empty: - continue - - for i, metric in enumerate(METRICS): - for j in range(PLOT_N_BEST): - row = df_plot.iloc[j] - num_seeds = row[(metric, 'count')] - ci_factor = 1 / np.sqrt(num_seeds) - - if QUANTILE_BASED_CI: - metric_median = row[(metric, 'median')] - axes[i].scatter(k, metric_median, label=f'{method}_{j}') - lower_err = - (row[(metric, 'lcb')] - metric_median) * ci_factor - upper_err = (row[(metric, 'ucb')] - metric_median) * ci_factor - axes[i].errorbar(k, metric_median, yerr=np.array([[lower_err, upper_err]]).T, - capsize=5) - else: - metric_mean = row[(metric, 'mean')] - metric_std = row[(metric, 'std')] - axes[i].scatter(k, metric_mean, label=f'{method}_{j}') - axes[i].errorbar(k, metric_mean, yerr=2 * metric_std * ci_factor) - - if i == 0: - print(f'{method}_{j}', row['exp_result_folder'][0]) - if k == 0: - axes[i].set_ylabel(metric) - axes[i].set_xticks(np.arange(len(METHODS))) - axes[i].set_xticklabels(METHODS) - axes[i].set_xlim((-0.5, len(METHODS) - 0.5)) - axes[i].set_yscale('log') - - plt.legend() - plt.tight_layout() - plt.show() if __name__ == '__main__': parser = argparse.ArgumentParser(description='Inspect results of a regression experiment.') - parser.add_argument('--exp_name', type=str, default='sep12_str_std') - parser.add_argument('--data_source', type=str, default='real_racecar_new') + parser.add_argument('--exp_name', type=str, default='jan17_std_hf') + parser.add_argument('--data_source', type=str, default='real_racecar_v3') args = parser.parse_args() main(args) \ No newline at end of file diff --git a/experiments/sim_real_transfer_exp/run_regression_exp.py b/experiments/sim_real_transfer_exp/run_regression_exp.py index f6beaee..39f4e90 100644 --- a/experiments/sim_real_transfer_exp/run_regression_exp.py +++ b/experiments/sim_real_transfer_exp/run_regression_exp.py @@ -38,7 +38,7 @@ def regression_experiment( data_source: str, num_samples_train: int, data_seed: int = 981648, - pred_diff: bool = False, + pred_diff: bool = True, # logging parameters use_wandb: bool = False, @@ -48,7 +48,8 @@ def regression_experiment( model_seed: int = 892616, likelihood_std: Union[List[float], float] = 0.1, data_batch_size: int = 8, - num_train_steps: int = 20000, + min_train_steps: int = 2500, + num_epochs: int = 60, lr: float = 1e-3, hidden_activation: str = 'leaky_relu', num_layers: int = 3, @@ -66,12 +67,14 @@ def regression_experiment( # FSVGD parameters bandwidth_gp_prior: float = 0.4, + outputscale_gp_prior: float = 1.0, num_measurement_points: int = 32, # FSVGD_Sim_Prior parameters bandwidth_score_estim: float = None, ssge_kernel_type: str = 'SE', num_f_samples: int = 128, + use_hf_sim: bool = True, switch_score_estimator_frac: float = 0.6667, added_gp_lengthscale: float = 5., @@ -80,19 +83,24 @@ def regression_experiment( # BNN_SVGD_DistillPrior num_distill_steps: int = 500000, ): + # determine number of train steps + num_train_steps = num_epochs * num_samples_train // data_batch_size + min_train_steps + # provide data and sim - x_train, y_train, x_test, y_test, sim_lf = provide_data_and_sim( + x_train, y_train, x_test, y_test, sim = provide_data_and_sim( data_source=data_source, - data_spec={'num_samples_train': num_samples_train, - 'sampling': 'consecutive'}, + data_spec={'num_samples_train': 6500, 'sampling': 'iid', + 'use_hf_sim': bool(use_hf_sim), 'num_samples_test': 6000}, data_seed=data_seed) + x_train = x_train[:num_samples_train] + y_train = y_train[:num_samples_train] # handle pred diff mode if pred_diff: - assert x_train.shape[-1] == sim_lf.input_size and y_train.shape[-1] == sim_lf.output_size - y_train = y_train - x_train[..., :sim_lf.output_size] - y_test = y_test - x_test[..., :sim_lf.output_size] - sim_lf = PredictStateChangeWrapper(sim_lf) + assert x_train.shape[-1] == sim.input_size and y_train.shape[-1] == sim.output_size + y_train = y_train - x_train[..., :sim.output_size] + y_test = y_test - x_test[..., :sim.output_size] + sim = PredictStateChangeWrapper(sim) if model.endswith('_no_add_gp'): no_added_gp = True @@ -105,12 +113,12 @@ def regression_experiment( # create additive sim with a GP on top of the sim prior to model the fidelity gap if no_added_gp: - sim = sim_lf + sim = sim else: - sim = AdditiveSim(base_sims=[sim_lf, + sim = AdditiveSim(base_sims=[sim, GaussianProcessSim( - sim_lf.input_size, - sim_lf.output_size, + sim.input_size, + sim.output_size, output_scale=added_gp_outputscale, length_scale=added_gp_lengthscale)]) @@ -144,6 +152,7 @@ def regression_experiment( num_particles=num_particles, bandwidth_svgd=bandwidth_svgd, bandwidth_gp_prior=bandwidth_gp_prior, + outputscale_gp_prior=outputscale_gp_prior, likelihood_reg=likelihood_reg, num_measurement_points=num_measurement_points, **standard_model_params) @@ -166,6 +175,7 @@ def regression_experiment( num_particles=num_particles, bandwidth_svgd=bandwidth_svgd, bandwidth_gp_prior=bandwidth_gp_prior, + outputscale_gp_prior=outputscale_gp_prior, likelihood_reg=likelihood_reg, num_measurement_points=num_measurement_points, **standard_model_params @@ -174,7 +184,7 @@ def regression_experiment( base_bnn=base_bnn, sim=sim, use_base_bnn=(model == 'GreyBox'), - num_sim_model_train_steps=5_000, + num_sim_model_train_steps=20_000, ) elif model == 'BNN_MMD_SimPrior': model = BNN_MMD_SimPrior(domain=sim.domain, @@ -299,23 +309,24 @@ def main(args): # data parameters parser.add_argument('--data_source', type=str, default='real_racecar_v3') parser.add_argument('--pred_diff', type=int, default=1) - parser.add_argument('--num_samples_train', type=int, default=200) + parser.add_argument('--num_samples_train', type=int, default=50) parser.add_argument('--data_seed', type=int, default=77698) # standard BNN parameters - parser.add_argument('--model', type=str, default='BNN_FSVGD') + parser.add_argument('--model', type=str, default='SysID') parser.add_argument('--model_seed', type=int, default=892616) parser.add_argument('--likelihood_std', type=float, default=None) parser.add_argument('--learn_likelihood_std', type=int, default=1) - parser.add_argument('--likelihood_reg', type=float, default=1.0) + parser.add_argument('--likelihood_reg', type=float, default=0.0) parser.add_argument('--data_batch_size', type=int, default=8) - parser.add_argument('--num_train_steps', type=int, default=20000) + parser.add_argument('--min_train_steps', type=int, default=2500) + parser.add_argument('--num_epochs', type=int, default=60) parser.add_argument('--lr', type=float, default=1e-3) parser.add_argument('--hidden_activation', type=str, default='leaky_relu') parser.add_argument('--num_layers', type=int, default=3) parser.add_argument('--layer_size', type=int, default=64) parser.add_argument('--normalize_likelihood_std', type=bool, default=True) - parser.add_argument('--likelihood_exponent', type=float, default=0.5) + parser.add_argument('--likelihood_exponent', type=float, default=1.0) # SVGD parameters parser.add_argument('--num_particles', type=int, default=20) @@ -324,7 +335,8 @@ def main(args): parser.add_argument('--bias_prior_std', type=float, default=1.0) # FSVGD parameters - parser.add_argument('--bandwidth_gp_prior', type=float, default=0.4) + parser.add_argument('--bandwidth_gp_prior', type=float, default=0.1) + parser.add_argument('--outputscale_gp_prior', type=float, default=1.0) parser.add_argument('--num_measurement_points', type=int, default=32) # FSVGD_SimPrior parameters @@ -332,6 +344,8 @@ def main(args): parser.add_argument('--ssge_kernel_type', type=str, default='IMQ') parser.add_argument('--num_f_samples', type=int, default=128) parser.add_argument('--switch_score_estimator_frac', type=float, default=0.6667) + parser.add_argument('--use_hf_sim', type=int, default=1) + # Additive SimPrior GP parameters parser.add_argument('--added_gp_lengthscale', type=float, default=5.) diff --git a/experiments/sim_real_transfer_exp/sweep_regression_exp.py b/experiments/sim_real_transfer_exp/sweep_regression_exp.py index a9ae222..88537e1 100644 --- a/experiments/sim_real_transfer_exp/sweep_regression_exp.py +++ b/experiments/sim_real_transfer_exp/sweep_regression_exp.py @@ -11,23 +11,24 @@ MODEL_SPECIFIC_CONFIG = { 'BNN_SVGD': { 'bandwidth_svgd': {'distribution': 'log_uniform', 'min': -1., 'max': 4.}, - 'num_train_steps': {'values': [20000]}, - 'likelihood_reg': {'values': [0., 1.]}, + 'min_train_steps': {'values': [2000]}, + 'num_epochs': {'values': [60]}, + 'likelihood_reg': {'values': [1.]}, }, 'BNN_FSVGD': { 'bandwidth_svgd': {'values': [10.]}, 'bandwidth_gp_prior': {'distribution': 'log_uniform', 'min': -2., 'max': 0.}, 'num_train_steps': {'values': [20000]}, 'num_measurement_points': {'values': [32]}, - 'likelihood_reg': {'values': [0., 1.]}, + 'likelihood_reg': {'values': [1.]}, }, 'BNN_FSVGD_SimPrior_gp': { 'bandwidth_svgd': {'values': [10.]}, 'num_train_steps': {'values': [40000]}, 'num_measurement_points': {'values': [32]}, 'num_f_samples': {'values': [1024]}, - 'added_gp_lengthscale': {'distribution': 'uniform', 'min': 0., 'max': 20.}, # racecar: 4 - 8 - 'added_gp_outputscale': {'distribution': 'uniform', 'min': 0., 'max': 10.}, # racecar: 4 - 8 + 'added_gp_lengthscale': {'distribution': 'uniform', 'min': 10., 'max': 40.}, # racecar: 4 - 8 + 'added_gp_outputscale': {'distribution': 'uniform', 'min': 0., 'max': 40.}, # racecar: 4 - 8 }, 'BNN_FSVGD_SimPrior_nu-method': { 'bandwidth_svgd': {'values': [0.2]}, @@ -77,6 +78,7 @@ def main(args): 'learn_likelihood_std': {'value': args.learn_likelihood_std}, 'num_particles': {'value': 20}, 'data_batch_size': {'value': 8}, + 'use_hf_sim': {'value': args.use_hf_sim}, } # update with model specific sweep ranges model_name = args.model.replace('_no_add_gp', '') @@ -130,6 +132,7 @@ def main(args): # # standard BNN parameters parser.add_argument('--model', type=str, default='BNN_FSVGD') parser.add_argument('--learn_likelihood_std', type=int, default=0) + parser.add_argument('--use_hf_sim', type=int, default=0) args = parser.parse_args() main(args) diff --git a/experiments/sim_real_transfer_exp/sweep_regression_exp_num_data.py b/experiments/sim_real_transfer_exp/sweep_regression_exp_num_data.py index 73b3385..ecbfc6e 100644 --- a/experiments/sim_real_transfer_exp/sweep_regression_exp_num_data.py +++ b/experiments/sim_real_transfer_exp/sweep_regression_exp_num_data.py @@ -8,49 +8,56 @@ import argparse import os -MODEL_SPECIFIC_CONFIG = { - 'BNN_SVGD': { - 'bandwidth_svgd': {'values': [10.]}, - 'num_train_steps': {'values': [20000]}, - 'likelihood_reg': {'values': [0.0]}, - }, - 'BNN_FSVGD': { - 'bandwidth_svgd': {'values': [10.]}, - 'bandwidth_gp_prior': {'values': [0.4]}, - 'num_train_steps': {'values': [20000]}, - 'num_measurement_points': {'values': [32]}, - 'likelihood_reg': {'values': [0.0]}, - }, - 'BNN_FSVGD_SimPrior_gp': { - 'bandwidth_svgd': {'values': [10.]}, - 'num_train_steps': {'values': [40000]}, - 'num_measurement_points': {'values': [32]}, - 'num_f_samples': {'values': [1024]}, - 'added_gp_lengthscale': {'values': [10.]}, - 'added_gp_outputscale': {'values': [2.]}, - }, - # 'BNN_FSVGD_SimPrior_nu-method': { - # 'bandwidth_svgd': {'values': [10.]}, - # 'num_train_steps': {'values': [60000]}, - # 'num_measurement_points': {'values': [32]}, - # 'num_f_samples': {'values': [512]}, - # 'bandwidth_score_estim': {'distribution': 'uniform', 'min': 0.8, 'max': 2.0}, - # 'added_gp_lengthscale': {'distribution': 'uniform', 'min': 5., 'max': 10.0}, - # 'added_gp_outputscale': {'distribution': 'uniform', 'min': 0.5, 'max': 2.0}, # racecar: 4 - 8 - # }, - 'SysID': { - }, - 'GreyBox': { - 'bandwidth_svgd': {'values': [10.]}, - 'bandwidth_gp_prior': {'values': [0.4]}, - 'num_train_steps': {'values': [20000]}, - 'num_measurement_points': {'values': [64]}, - 'likelihood_reg': {'values': [0.0]}, - }, -} - def main(args): + MODEL_SPECIFIC_CONFIG = { + 'BNN_SVGD': { + 'min_train_steps': {'values': [1000]}, + 'num_epochs': {'values': [60]}, + 'bandwidth_svgd': {'values': [10.]}, + #'likelihood_reg': {'values': [10.0]}, + }, + 'BNN_FSVGD': { + 'min_train_steps': {'values': [1000]}, + 'num_epochs': {'values': [60]}, + 'bandwidth_svgd': {'values': [10.]}, + 'bandwidth_gp_prior': {'values': [0.4]}, + 'num_measurement_points': {'values': [32]}, + 'likelihood_reg': {'values': [10.0]}, + + }, + 'BNN_FSVGD_SimPrior_gp': { + 'min_train_steps': {'values': [5000]}, + 'num_epochs': {'values': [75]}, + 'bandwidth_svgd': {'values': [10.]}, + 'num_measurement_points': {'values': [32]}, + 'num_f_samples': {'values': [1024]}, + 'added_gp_lengthscale': {'values': [10.]}, + 'added_gp_outputscale': {'values': [2. if args.use_hf_sim else 4.]}, + }, + # 'BNN_FSVGD_SimPrior_nu-method': { + # 'bandwidth_svgd': {'values': [10.]}, + # 'num_train_steps': {'values': [60000]}, + # 'num_measurement_points': {'values': [32]}, + # 'num_f_samples': {'values': [512]}, + # 'bandwidth_score_estim': {'distribution': 'uniform', 'min': 0.8, 'max': 2.0}, + # 'added_gp_lengthscale': {'distribution': 'uniform', 'min': 5., 'max': 10.0}, + # 'added_gp_outputscale': {'distribution': 'uniform', 'min': 0.5, 'max': 2.0}, # racecar: 4 - 8 + # }, + 'SysID': { + 'likelihood_reg': {'values': [10.0]}, + }, + 'GreyBox': { + 'min_train_steps': {'values': [1000]}, + 'num_epochs': {'values': [60]}, + 'bandwidth_svgd': {'values': [10.]}, + 'bandwidth_gp_prior': {'values': [0.4]}, + 'outputscale_gp_prior': {'values': [0.05]}, + 'num_measurement_points': {'values': [64]}, + 'likelihood_reg': {'values': [10.0]}, + }, + } + # setup random seeds rds = np.random.RandomState(args.seed) model_seeds = list(rds.randint(0, 10**6, size=(100,))) @@ -63,6 +70,7 @@ def main(args): 'pred_diff': {'value': args.pred_diff}, 'num_particles': {'value': 20}, 'data_batch_size': {'value': 8}, + 'use_hf_sim': {'value': args.use_hf_sim}, } # update with model specific sweep ranges model_name = args.model.replace('_no_add_gp', '') @@ -73,7 +81,7 @@ def main(args): exp_base_path = os.path.join(RESULT_DIR, args.exp_name) exp_path = os.path.join(exp_base_path, f'{args.data_source}_{args.model}') - N_SAMPLES_LIST = [50, 100, 200, 400, 800, 1600, 3200, 6400] + N_SAMPLES_LIST = [50, 200, 800, 1600, 3200, 6400] command_list = [] output_file_list = [] for _ in range(args.num_hparam_samples): @@ -104,8 +112,8 @@ def main(args): # sweep args parser.add_argument('--num_hparam_samples', type=int, default=1) - parser.add_argument('--num_model_seeds', type=int, default=5, help='number of model seeds per hparam') - parser.add_argument('--num_data_seeds', type=int, default=5, help='number of model seeds per hparam') + parser.add_argument('--num_model_seeds', type=int, default=3, help='number of model seeds per hparam') + parser.add_argument('--num_data_seeds', type=int, default=3, help='number of model seeds per hparam') parser.add_argument('--num_cpus', type=int, default=1, help='number of cpus to use') parser.add_argument('--run_mode', type=str, default='euler') @@ -116,12 +124,13 @@ def main(args): parser.add_argument('--yes', default=False, action='store_true') # data parameters - parser.add_argument('--data_source', type=str, default='pendulum') - parser.add_argument('--pred_diff', type=int, default=0) + parser.add_argument('--data_source', type=str, default='real_racecar_v3') + parser.add_argument('--pred_diff', type=int, default=1) # # standard BNN parameters - parser.add_argument('--model', type=str, default='BNN_SVGD') - parser.add_argument('--learn_likelihood_std', type=int, default=0) + parser.add_argument('--model', type=str, default='SysID') + parser.add_argument('--learn_likelihood_std', type=int, default=1) + parser.add_argument('--use_hf_sim', type=int, default=1) args = parser.parse_args() main(args)