From 315109094ba29b9388d983e898d06f62493d6afe Mon Sep 17 00:00:00 2001 From: Vasisht Tadigotla Date: Fri, 5 Jun 2015 11:32:25 -0400 Subject: [PATCH 1/3] enable use of spot instances for loadbalancing with -b/--spot_bid option to loadbalance --- starcluster/balancers/sge/__init__.py | 6 ++++-- starcluster/commands/loadbalance.py | 4 ++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/starcluster/balancers/sge/__init__.py b/starcluster/balancers/sge/__init__.py index ca6f31f17..024fc3e67 100644 --- a/starcluster/balancers/sge/__init__.py +++ b/starcluster/balancers/sge/__init__.py @@ -418,7 +418,8 @@ class SGELoadBalancer(LoadBalancer): def __init__(self, interval=60, max_nodes=None, wait_time=900, add_pi=1, kill_after=45, stab=180, lookback_win=3, min_nodes=None, kill_cluster=False, plot_stats=False, - plot_output_dir=None, dump_stats=False, stats_file=None): + plot_output_dir=None, dump_stats=False, stats_file=None, + spot_bid=None): self._cluster = None self._keep_polling = True self._visualizer = None @@ -437,6 +438,7 @@ def __init__(self, interval=60, max_nodes=None, wait_time=900, self.stats_file = stats_file self.plot_stats = plot_stats self.plot_output_dir = plot_output_dir + self.spot_bit = spot_bid if plot_stats: assert self.visualizer is not None @@ -724,7 +726,7 @@ def _eval_add_node(self): log.warn("Adding %d nodes at %s" % (need_to_add, str(utils.get_utc_now()))) try: - self._cluster.add_nodes(need_to_add) + self._cluster.add_nodes(need_to_add,spot_bid=self.spot_bid) self.__last_cluster_mod_time = utils.get_utc_now() log.info("Done adding nodes at %s" % str(self.__last_cluster_mod_time)) diff --git a/starcluster/commands/loadbalance.py b/starcluster/commands/loadbalance.py index 28e81d80f..47e8856bc 100644 --- a/starcluster/commands/loadbalance.py +++ b/starcluster/commands/loadbalance.py @@ -103,6 +103,10 @@ def addopts(self, parser): parser.add_option("-K", "--kill-cluster", dest="kill_cluster", action="store_true", default=False, help="Terminate the cluster when the queue is empty") + parser.add_option("-b", "--bid", dest="spot_bid", + action="callback", type="float", default=None, + callback=self._positive_int, + help="max bid for spot requests, not used by default") def execute(self, args): if not self.cfg.globals.enable_experimental: From 8801f59e5187ae30ac0d22d308fbaeba8a86a4af Mon Sep 17 00:00:00 2001 From: Vasisht Tadigotla Date: Fri, 5 Jun 2015 11:36:58 -0400 Subject: [PATCH 2/3] spaces vs tabs --- starcluster/balancers/sge/__init__.py | 2 +- starcluster/commands/loadbalance.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/starcluster/balancers/sge/__init__.py b/starcluster/balancers/sge/__init__.py index 024fc3e67..d8511f360 100644 --- a/starcluster/balancers/sge/__init__.py +++ b/starcluster/balancers/sge/__init__.py @@ -438,7 +438,7 @@ def __init__(self, interval=60, max_nodes=None, wait_time=900, self.stats_file = stats_file self.plot_stats = plot_stats self.plot_output_dir = plot_output_dir - self.spot_bit = spot_bid + self.spot_bit = spot_bid if plot_stats: assert self.visualizer is not None diff --git a/starcluster/commands/loadbalance.py b/starcluster/commands/loadbalance.py index 47e8856bc..672e1dc94 100644 --- a/starcluster/commands/loadbalance.py +++ b/starcluster/commands/loadbalance.py @@ -103,7 +103,7 @@ def addopts(self, parser): parser.add_option("-K", "--kill-cluster", dest="kill_cluster", action="store_true", default=False, help="Terminate the cluster when the queue is empty") - parser.add_option("-b", "--bid", dest="spot_bid", + parser.add_option("-b", "--bid", dest="spot_bid", action="callback", type="float", default=None, callback=self._positive_int, help="max bid for spot requests, not used by default") From f55fd4d2fbae2b47a31c300ad282b7e162fdc555 Mon Sep 17 00:00:00 2001 From: Vasisht Tadigotla Date: Fri, 5 Jun 2015 11:58:31 -0400 Subject: [PATCH 3/3] pep8 fixes to loadbalancer --- starcluster/balancers/sge/__init__.py | 2 +- starcluster/commands/loadbalance.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/starcluster/balancers/sge/__init__.py b/starcluster/balancers/sge/__init__.py index d8511f360..d414fe9c1 100644 --- a/starcluster/balancers/sge/__init__.py +++ b/starcluster/balancers/sge/__init__.py @@ -726,7 +726,7 @@ def _eval_add_node(self): log.warn("Adding %d nodes at %s" % (need_to_add, str(utils.get_utc_now()))) try: - self._cluster.add_nodes(need_to_add,spot_bid=self.spot_bid) + self._cluster.add_nodes(need_to_add, spot_bid=self.spot_bid) self.__last_cluster_mod_time = utils.get_utc_now() log.info("Done adding nodes at %s" % str(self.__last_cluster_mod_time)) diff --git a/starcluster/commands/loadbalance.py b/starcluster/commands/loadbalance.py index 672e1dc94..f374bee78 100644 --- a/starcluster/commands/loadbalance.py +++ b/starcluster/commands/loadbalance.py @@ -106,7 +106,7 @@ def addopts(self, parser): parser.add_option("-b", "--bid", dest="spot_bid", action="callback", type="float", default=None, callback=self._positive_int, - help="max bid for spot requests, not used by default") + help="max bid for spot nodes, not used by default") def execute(self, args): if not self.cfg.globals.enable_experimental: