Skip to content

Commit

Permalink
updates on the sim_real_exp
Browse files Browse the repository at this point in the history
  • Loading branch information
@jonasrothfuss
jonasrothfuss committed Jan 24, 2024
1 parent 9d4ccd8 commit 3aa264b
Show file tree
Hide file tree
Showing 5 changed files with 114 additions and 135 deletions.
2 changes: 1 addition & 1 deletion experiments/data_provider.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -203,7 +203,7 @@ def get_rccar_recorded_data_new(encode_angle: bool = True, skip_first_n_points:
num_test_points = 20_000
elif dataset in ['v2', 'v3', 'v4']:
recordings_dir = [os.path.join(DATA_DIR, f'recordings_rc_car_{dataset}')]
num_test_points = 10_000
num_test_points = 6_000
else:
raise ValueError(f"Unknown dataset {dataset} for car_id {car_id}")
else:
Expand Down
75 changes: 14 additions & 61 deletions experiments/sim_real_transfer_exp/inspect_results.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,82 +95,35 @@ def main(args, drop_nan=False):

print('Models:', set(df_agg['model']))

different_method_plot(df_agg, metric='nll')
different_method_plot(df_agg, metric='mae_pose')
different_method_plot(df_agg, metric='nll', filter_std_higher_than=20.)
different_method_plot(df_agg, metric='rmse')

df_method = df_agg[(df_agg['model'] == 'BNN_FSVGD_SimPrior_gp_no_add_gp')]
df_method = df_agg[(df_agg['model'] == 'BNN_FSVGD_SimPrior_gp')]

#df_method = df_method[df_method['bandwidth_score_estim'] > 0.1]

metric = 'nll'
for param in ['num_measurement_points', 'added_gp_lengthscale', 'added_gp_outputscale']:# ['num_f_samples', 'bandwidth_score_estim', 'bandwidth_svgd', 'num_measurement_points']:
for param in [
'added_gp_lengthscale',
'added_gp_outputscale',
# 'bandwidth_svgd',
# 'likelihood_reg',
#'bandwidth_svgd'
]:
# ['num_f_samples', 'bandwidth_score_estim', 'bandwidth_svgd', 'num_measurement_points']:
plt.scatter(df_method[param], df_method[(metric, 'mean')])
plt.xlabel(param)
plt.xscale('log')
plt.ylabel(metric)
#plt.scatter(5., -7, color='red')
#plt.ylim(-8, 0)
plt.ylim(-10, -6)
plt.show()

plt.scatter(df_method['added_gp_lengthscale'], df_method['added_gp_outputscale'], c=df_method[(metric, 'mean')],
cmap='viridis')
plt.xlabel('added_gp_lengthscale')
plt.ylabel('added_gp_outputscale')
plt.colorbar(label=metric)
plt.show()

plt.scatter(df_method['added_gp_lengthscale'], df_method['added_gp_outputscale'], s=df_method[(metric, 'mean')])

QUANTILE_BASED_CI = True
METHODS = ['random_search', 'hill_search']
METRICS = ['x_diff', 'f_diff']
PLOT_N_BEST = 2
n_metrics = len(METRICS)

fig, axes = plt.subplots(ncols=n_metrics, nrows=1, figsize=(n_metrics * 4, 4))
for k, method in enumerate(METHODS):
df_plot = df_agg.loc[df_agg['method'] == method]
df_plot.sort_values(by=('x_diff', 'mean'), ascending=True, inplace=True)

if df_plot.empty:
continue

for i, metric in enumerate(METRICS):
for j in range(PLOT_N_BEST):
row = df_plot.iloc[j]
num_seeds = row[(metric, 'count')]
ci_factor = 1 / np.sqrt(num_seeds)

if QUANTILE_BASED_CI:
metric_median = row[(metric, 'median')]
axes[i].scatter(k, metric_median, label=f'{method}_{j}')
lower_err = - (row[(metric, 'lcb')] - metric_median) * ci_factor
upper_err = (row[(metric, 'ucb')] - metric_median) * ci_factor
axes[i].errorbar(k, metric_median, yerr=np.array([[lower_err, upper_err]]).T,
capsize=5)
else:
metric_mean = row[(metric, 'mean')]
metric_std = row[(metric, 'std')]
axes[i].scatter(k, metric_mean, label=f'{method}_{j}')
axes[i].errorbar(k, metric_mean, yerr=2 * metric_std * ci_factor)

if i == 0:
print(f'{method}_{j}', row['exp_result_folder'][0])
if k == 0:
axes[i].set_ylabel(metric)
axes[i].set_xticks(np.arange(len(METHODS)))
axes[i].set_xticklabels(METHODS)
axes[i].set_xlim((-0.5, len(METHODS) - 0.5))
axes[i].set_yscale('log')

plt.legend()
plt.tight_layout()
plt.show()


if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Inspect results of a regression experiment.')
parser.add_argument('--exp_name', type=str, default='sep12_str_std')
parser.add_argument('--data_source', type=str, default='real_racecar_new')
parser.add_argument('--exp_name', type=str, default='jan17_std_hf')
parser.add_argument('--data_source', type=str, default='real_racecar_v3')
args = parser.parse_args()
main(args)
54 changes: 34 additions & 20 deletions experiments/sim_real_transfer_exp/run_regression_exp.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ def regression_experiment(
data_source: str,
num_samples_train: int,
data_seed: int = 981648,
pred_diff: bool = False,
pred_diff: bool = True,

# logging parameters
use_wandb: bool = False,
Expand All @@ -48,7 +48,8 @@ def regression_experiment(
model_seed: int = 892616,
likelihood_std: Union[List[float], float] = 0.1,
data_batch_size: int = 8,
num_train_steps: int = 20000,
min_train_steps: int = 2500,
num_epochs: int = 60,
lr: float = 1e-3,
hidden_activation: str = 'leaky_relu',
num_layers: int = 3,
Expand All @@ -66,12 +67,14 @@ def regression_experiment(

# FSVGD parameters
bandwidth_gp_prior: float = 0.4,
outputscale_gp_prior: float = 1.0,
num_measurement_points: int = 32,

# FSVGD_Sim_Prior parameters
bandwidth_score_estim: float = None,
ssge_kernel_type: str = 'SE',
num_f_samples: int = 128,
use_hf_sim: bool = True,

switch_score_estimator_frac: float = 0.6667,
added_gp_lengthscale: float = 5.,
Expand All @@ -80,19 +83,24 @@ def regression_experiment(
# BNN_SVGD_DistillPrior
num_distill_steps: int = 500000,
):
# determine number of train steps
num_train_steps = num_epochs * num_samples_train // data_batch_size + min_train_steps

# provide data and sim
x_train, y_train, x_test, y_test, sim_lf = provide_data_and_sim(
x_train, y_train, x_test, y_test, sim = provide_data_and_sim(
data_source=data_source,
data_spec={'num_samples_train': num_samples_train,
'sampling': 'consecutive'},
data_spec={'num_samples_train': 6500, 'sampling': 'iid',
'use_hf_sim': bool(use_hf_sim), 'num_samples_test': 6000},
data_seed=data_seed)
x_train = x_train[:num_samples_train]
y_train = y_train[:num_samples_train]

# handle pred diff mode
if pred_diff:
assert x_train.shape[-1] == sim_lf.input_size and y_train.shape[-1] == sim_lf.output_size
y_train = y_train - x_train[..., :sim_lf.output_size]
y_test = y_test - x_test[..., :sim_lf.output_size]
sim_lf = PredictStateChangeWrapper(sim_lf)
assert x_train.shape[-1] == sim.input_size and y_train.shape[-1] == sim.output_size
y_train = y_train - x_train[..., :sim.output_size]
y_test = y_test - x_test[..., :sim.output_size]
sim = PredictStateChangeWrapper(sim)

if model.endswith('_no_add_gp'):
no_added_gp = True
Expand All @@ -105,12 +113,12 @@ def regression_experiment(

# create additive sim with a GP on top of the sim prior to model the fidelity gap
if no_added_gp:
sim = sim_lf
sim = sim
else:
sim = AdditiveSim(base_sims=[sim_lf,
sim = AdditiveSim(base_sims=[sim,
GaussianProcessSim(
sim_lf.input_size,
sim_lf.output_size,
sim.input_size,
sim.output_size,
output_scale=added_gp_outputscale,
length_scale=added_gp_lengthscale)])

Expand Down Expand Up @@ -144,6 +152,7 @@ def regression_experiment(
num_particles=num_particles,
bandwidth_svgd=bandwidth_svgd,
bandwidth_gp_prior=bandwidth_gp_prior,
outputscale_gp_prior=outputscale_gp_prior,
likelihood_reg=likelihood_reg,
num_measurement_points=num_measurement_points,
**standard_model_params)
Expand All @@ -166,6 +175,7 @@ def regression_experiment(
num_particles=num_particles,
bandwidth_svgd=bandwidth_svgd,
bandwidth_gp_prior=bandwidth_gp_prior,
outputscale_gp_prior=outputscale_gp_prior,
likelihood_reg=likelihood_reg,
num_measurement_points=num_measurement_points,
**standard_model_params
Expand All @@ -174,7 +184,7 @@ def regression_experiment(
base_bnn=base_bnn,
sim=sim,
use_base_bnn=(model == 'GreyBox'),
num_sim_model_train_steps=5_000,
num_sim_model_train_steps=20_000,
)
elif model == 'BNN_MMD_SimPrior':
model = BNN_MMD_SimPrior(domain=sim.domain,
Expand Down Expand Up @@ -299,23 +309,24 @@ def main(args):
# data parameters
parser.add_argument('--data_source', type=str, default='real_racecar_v3')
parser.add_argument('--pred_diff', type=int, default=1)
parser.add_argument('--num_samples_train', type=int, default=200)
parser.add_argument('--num_samples_train', type=int, default=50)
parser.add_argument('--data_seed', type=int, default=77698)

# standard BNN parameters
parser.add_argument('--model', type=str, default='BNN_FSVGD')
parser.add_argument('--model', type=str, default='SysID')
parser.add_argument('--model_seed', type=int, default=892616)
parser.add_argument('--likelihood_std', type=float, default=None)
parser.add_argument('--learn_likelihood_std', type=int, default=1)
parser.add_argument('--likelihood_reg', type=float, default=1.0)
parser.add_argument('--likelihood_reg', type=float, default=0.0)
parser.add_argument('--data_batch_size', type=int, default=8)
parser.add_argument('--num_train_steps', type=int, default=20000)
parser.add_argument('--min_train_steps', type=int, default=2500)
parser.add_argument('--num_epochs', type=int, default=60)
parser.add_argument('--lr', type=float, default=1e-3)
parser.add_argument('--hidden_activation', type=str, default='leaky_relu')
parser.add_argument('--num_layers', type=int, default=3)
parser.add_argument('--layer_size', type=int, default=64)
parser.add_argument('--normalize_likelihood_std', type=bool, default=True)
parser.add_argument('--likelihood_exponent', type=float, default=0.5)
parser.add_argument('--likelihood_exponent', type=float, default=1.0)

# SVGD parameters
parser.add_argument('--num_particles', type=int, default=20)
Expand All @@ -324,14 +335,17 @@ def main(args):
parser.add_argument('--bias_prior_std', type=float, default=1.0)

# FSVGD parameters
parser.add_argument('--bandwidth_gp_prior', type=float, default=0.4)
parser.add_argument('--bandwidth_gp_prior', type=float, default=0.1)
parser.add_argument('--outputscale_gp_prior', type=float, default=1.0)
parser.add_argument('--num_measurement_points', type=int, default=32)

# FSVGD_SimPrior parameters
parser.add_argument('--bandwidth_score_estim', type=float, default=None)
parser.add_argument('--ssge_kernel_type', type=str, default='IMQ')
parser.add_argument('--num_f_samples', type=int, default=128)
parser.add_argument('--switch_score_estimator_frac', type=float, default=0.6667)
parser.add_argument('--use_hf_sim', type=int, default=1)


# Additive SimPrior GP parameters
parser.add_argument('--added_gp_lengthscale', type=float, default=5.)
Expand Down
13 changes: 8 additions & 5 deletions experiments/sim_real_transfer_exp/sweep_regression_exp.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,23 +11,24 @@
MODEL_SPECIFIC_CONFIG = {
'BNN_SVGD': {
'bandwidth_svgd': {'distribution': 'log_uniform', 'min': -1., 'max': 4.},
'num_train_steps': {'values': [20000]},
'likelihood_reg': {'values': [0., 1.]},
'min_train_steps': {'values': [2000]},
'num_epochs': {'values': [60]},
'likelihood_reg': {'values': [1.]},
},
'BNN_FSVGD': {
'bandwidth_svgd': {'values': [10.]},
'bandwidth_gp_prior': {'distribution': 'log_uniform', 'min': -2., 'max': 0.},
'num_train_steps': {'values': [20000]},
'num_measurement_points': {'values': [32]},
'likelihood_reg': {'values': [0., 1.]},
'likelihood_reg': {'values': [1.]},
},
'BNN_FSVGD_SimPrior_gp': {
'bandwidth_svgd': {'values': [10.]},
'num_train_steps': {'values': [40000]},
'num_measurement_points': {'values': [32]},
'num_f_samples': {'values': [1024]},
'added_gp_lengthscale': {'distribution': 'uniform', 'min': 0., 'max': 20.}, # racecar: 4 - 8
'added_gp_outputscale': {'distribution': 'uniform', 'min': 0., 'max': 10.}, # racecar: 4 - 8
'added_gp_lengthscale': {'distribution': 'uniform', 'min': 10., 'max': 40.}, # racecar: 4 - 8
'added_gp_outputscale': {'distribution': 'uniform', 'min': 0., 'max': 40.}, # racecar: 4 - 8
},
'BNN_FSVGD_SimPrior_nu-method': {
'bandwidth_svgd': {'values': [0.2]},
Expand Down Expand Up @@ -77,6 +78,7 @@ def main(args):
'learn_likelihood_std': {'value': args.learn_likelihood_std},
'num_particles': {'value': 20},
'data_batch_size': {'value': 8},
'use_hf_sim': {'value': args.use_hf_sim},
}
# update with model specific sweep ranges
model_name = args.model.replace('_no_add_gp', '')
Expand Down Expand Up @@ -130,6 +132,7 @@ def main(args):
# # standard BNN parameters
parser.add_argument('--model', type=str, default='BNN_FSVGD')
parser.add_argument('--learn_likelihood_std', type=int, default=0)
parser.add_argument('--use_hf_sim', type=int, default=0)

args = parser.parse_args()
main(args)
Loading

0 comments on commit 3aa264b

Please sign in to comment.