-
Notifications
You must be signed in to change notification settings - Fork 27
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'main' into FixDaemonThreadsFromPreventingDeath2
- Loading branch information
Showing
15 changed files
with
432 additions
and
124 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
76 changes: 76 additions & 0 deletions
76
...erSolution/src/main/docker/migrationConsole/lib/console_link/console_link/logic/replay.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,76 @@ | ||
import json | ||
import logging | ||
from typing import Tuple | ||
from console_link.models.utils import ExitCode | ||
from console_link.models.replayer_base import Replayer | ||
import yaml | ||
|
||
|
||
logger = logging.getLogger(__name__) | ||
|
||
|
||
def describe(replayer: Replayer, as_json=False) -> str: | ||
response = replayer.describe() | ||
if as_json: | ||
return json.dumps(response) | ||
return yaml.safe_dump(response) | ||
|
||
|
||
def start(replayer: Replayer, *args, **kwargs) -> Tuple[ExitCode, str]: | ||
try: | ||
result = replayer.start(*args, **kwargs) | ||
except NotImplementedError: | ||
logger.error(f"Start is not implemented for replayer {type(replayer).__name__}") | ||
return ExitCode.FAILURE, f"Start is not implemented for replayer {type(replayer).__name__}" | ||
except Exception as e: | ||
logger.error(f"Failed to start replayer: {e}") | ||
return ExitCode.FAILURE, f"Failure when starting replayer: {type(e).__name__} {e}" | ||
|
||
if result.success: | ||
return ExitCode.SUCCESS, "Replayer started successfully." + "\n" + result.display() | ||
return ExitCode.FAILURE, "Replayer start failed." + "\n" + result.display() | ||
|
||
|
||
def stop(replayer: Replayer, *args, **kwargs) -> Tuple[ExitCode, str]: | ||
logger.info("Stopping replayer") | ||
try: | ||
result = replayer.stop(*args, **kwargs) | ||
except NotImplementedError: | ||
logger.error(f"Stop is not implemented for replayer {type(replayer).__name__}") | ||
return ExitCode.FAILURE, f"Stop is not implemented for replayer {type(replayer).__name__}" | ||
except Exception as e: | ||
logger.error(f"Failed to stop replayer: {e}") | ||
return ExitCode.FAILURE, f"Failure when stopping replayer: {type(e).__name__} {e}" | ||
if result.success: | ||
return ExitCode.SUCCESS, "Replayer stopped successfully." + "\n" + result.display() | ||
return ExitCode.FAILURE, "Replayer stop failed." + "\n" + result.display() | ||
|
||
|
||
def scale(replayer: Replayer, units: int, *args, **kwargs) -> Tuple[ExitCode, str]: | ||
logger.info(f"Scaling replayer to {units} units") | ||
try: | ||
result = replayer.scale(units, *args, **kwargs) | ||
except NotImplementedError: | ||
logger.error(f"Scale is not implemented for replayer {type(replayer).__name__}") | ||
return ExitCode.FAILURE, f"Scale is not implemented for replayer {type(replayer).__name__}" | ||
except Exception as e: | ||
logger.error(f"Failed to scale replayer: {e}") | ||
return ExitCode.FAILURE, f"Failure when scaling replayer: {type(e).__name__} {e}" | ||
if result.success: | ||
return ExitCode.SUCCESS, "Replayer scaled successfully." + "\n" + result.display() | ||
return ExitCode.FAILURE, "Replayer scale failed." + "\n" + result.display() | ||
|
||
|
||
def status(replayer: Replayer, *args, **kwargs) -> Tuple[ExitCode, str]: | ||
logger.info("Getting replayer status") | ||
try: | ||
status = replayer.get_status(*args, **kwargs) | ||
except NotImplementedError: | ||
logger.error(f"Status is not implemented for replayer {type(replayer).__name__}") | ||
return ExitCode.FAILURE, f"Status is not implemented for replayer: {type(replayer).__name__}" | ||
except Exception as e: | ||
logger.error(f"Failed to get status of replayer: {e}") | ||
return ExitCode.FAILURE, f"Failure when getting status of replayer: {type(e).__name__} {e}" | ||
if status: | ||
return ExitCode.SUCCESS, status.value | ||
return ExitCode.FAILURE, "Replayer status retrieval failed." + "\n" + status |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
82 changes: 0 additions & 82 deletions
82
...olution/src/main/docker/migrationConsole/lib/console_link/console_link/models/replayer.py
This file was deleted.
Oops, something went wrong.
74 changes: 74 additions & 0 deletions
74
...on/src/main/docker/migrationConsole/lib/console_link/console_link/models/replayer_base.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,74 @@ | ||
from enum import Enum | ||
from typing import Dict | ||
from abc import ABC, abstractmethod | ||
|
||
from console_link.models.schema_tools import contains_one_of | ||
from console_link.models.command_result import CommandResult | ||
|
||
from cerberus import Validator | ||
|
||
DOCKER_REPLAY_SCHEMA = { | ||
"type": "dict", | ||
"schema": { | ||
"socket": {"type": "string", "required": False} | ||
} | ||
} | ||
|
||
ECS_REPLAY_SCHEMA = { | ||
"type": "dict", | ||
"schema": { | ||
"cluster_name": {"type": "string", "required": True}, | ||
"service_name": {"type": "string", "required": True}, | ||
"aws_region": {"type": "string", "required": False} | ||
} | ||
} | ||
|
||
SCHEMA = { | ||
"replay": { | ||
"type": "dict", | ||
"schema": { | ||
"docker": DOCKER_REPLAY_SCHEMA, | ||
"ecs": ECS_REPLAY_SCHEMA, | ||
"scale": {"type": "integer", "required": False, "min": 1} | ||
}, | ||
"check_with": contains_one_of({"docker", "ecs"}) | ||
} | ||
} | ||
|
||
|
||
ReplayStatus = Enum("ReplayStatus", ["NOT_STARTED", "RUNNING", "STOPPED", "FAILED"]) | ||
|
||
|
||
class Replayer(ABC): | ||
""" | ||
Interface for replaying data from kafka to a target cluster. | ||
""" | ||
def __init__(self, config: Dict) -> None: | ||
v = Validator(SCHEMA) | ||
self.config = config | ||
if not v.validate({"replay": self.config}): | ||
raise ValueError("Invalid config file for replay", v.errors) | ||
self.default_scale = self.config.get("scale", 1) | ||
|
||
@abstractmethod | ||
def start(self, *args, **kwargs) -> CommandResult: | ||
"""Begin running the replayer. After running start, the user should be able to assume that--barring exceptions | ||
or failures--their data will begin playing against the target cluster.""" | ||
pass | ||
|
||
@abstractmethod | ||
def stop(self, *args, **kwargs) -> CommandResult: | ||
"""Stop or pause the replay. This does not make guarantees about resumeability.""" | ||
pass | ||
|
||
@abstractmethod | ||
def get_status(self, *args, **kwargs) -> ReplayStatus: | ||
"""Return a status""" | ||
pass | ||
|
||
@abstractmethod | ||
def scale(self, units: int, *args, **kwargs) -> CommandResult: | ||
pass | ||
|
||
def describe(self) -> Dict: | ||
return self.config |
31 changes: 31 additions & 0 deletions
31
...ion/src/main/docker/migrationConsole/lib/console_link/console_link/models/replayer_ecs.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,31 @@ | ||
from typing import Dict | ||
from console_link.models.command_result import CommandResult | ||
from console_link.models.ecs_service import ECSService | ||
from console_link.models.replayer_base import Replayer, ReplayStatus | ||
|
||
import logging | ||
|
||
logger = logging.getLogger(__name__) | ||
|
||
|
||
class ECSReplayer(Replayer): | ||
def __init__(self, config: Dict) -> None: | ||
super().__init__(config) | ||
self.ecs_config = self.config["ecs"] | ||
self.ecs_client = ECSService(self.ecs_config["cluster_name"], self.ecs_config["service_name"], | ||
self.ecs_config.get("aws_region", None)) | ||
|
||
def start(self, *args, **kwargs) -> CommandResult: | ||
logger.info(f"Starting ECS replayer by setting desired count to {self.default_scale} instances") | ||
return self.ecs_client.set_desired_count(self.default_scale) | ||
|
||
def stop(self, *args, **kwargs) -> CommandResult: | ||
logger.info("Stopping ECS replayer by setting desired count to 0 instances") | ||
return self.ecs_client.set_desired_count(0) | ||
|
||
def get_status(self, *args, **kwargs) -> ReplayStatus: | ||
raise NotImplementedError() | ||
|
||
def scale(self, units: int, *args, **kwargs) -> CommandResult: | ||
logger.info(f"Scaling ECS replayer by setting desired count to {units} instances") | ||
return self.ecs_client.set_desired_count(units) |
Oops, something went wrong.