From 406d91cc319f7df03d7c39c931fb8435a5a5bc70 Mon Sep 17 00:00:00 2001 From: GustaafL Date: Thu, 1 Feb 2024 16:31:00 +0100 Subject: [PATCH 1/4] refactor: move comments and variable setting Signed-off-by: GustaafL --- flexmeasures/app.py | 37 ++++++------ flexmeasures/utils/config_utils.py | 93 ++++++++++++++++++------------ 2 files changed, 76 insertions(+), 54 deletions(-) diff --git a/flexmeasures/app.py b/flexmeasures/app.py index 4d52f8bda..cd22fc897 100644 --- a/flexmeasures/app.py +++ b/flexmeasures/app.py @@ -16,6 +16,7 @@ from flask_sslify import SSLify from flask_json import FlaskJSON from flask_cors import CORS +from flexmeasures.utils.config_utils import use_documentation_config, use_testing_config from redis import Redis from rq import Queue @@ -37,7 +38,6 @@ def create( # noqa C901 Also, a list of plugins can be set. Usually this works as a config setting, but this is useful for automated testing. """ - from flexmeasures.utils import config_defaults from flexmeasures.utils.config_utils import read_config, configure_logging from flexmeasures.utils.app_utils import set_secret_key, init_sentry from flexmeasures.utils.error_utils import add_basic_error_handlers @@ -46,20 +46,29 @@ def create( # noqa C901 configure_logging() # do this first, see https://flask.palletsprojects.com/en/2.0.x/logging # we're loading dotenv files manually & early (can do Flask.run(load_dotenv=False)), - # as we need to know the ENV now (for it to be recognised by Flask()). + # as we need to know the ENV now (for it to be recognized by Flask()). load_dotenv() + + # Below sets app.config["ENV"] = os.environ.get("FLASK_ENV") or "production" app = Flask("flexmeasures") - if env is not None: # overwrite - app.config["FLEXMEASURES_ENV"] = env - if app.config.get("FLEXMEASURES_ENV") == "testing": - app.testing = True - if app.config.get("FLEXMEASURES_ENV") == "development": - app.debug = config_defaults.DevelopmentConfig.DEBUG + # Overwrite the setting of app.config["ENV"] to "production" instead the "FLASK_ENV" environment variable. + # Otherwise, the in the future deprecated "FLASK_ENV" can create issues. + if os.environ.get("FLASK_ENV"): + app.config["ENV"] = "production" + + if env == "documentation": + use_documentation_config(app, path_to_config=path_to_config) + elif env == "testing": + use_testing_config(app) + from fakeredis import FakeStrictRedis - # App configuration + redis_conn = FakeStrictRedis( + host="redis", port="1234" + ) # dummy connection details + else: + read_config(app, custom_path_to_config=path_to_config, env=env) - read_config(app, custom_path_to_config=path_to_config) if plugins: app.config["FLEXMEASURES_PLUGINS"] += plugins add_basic_error_handlers(app) @@ -74,13 +83,7 @@ def create( # noqa C901 CORS(app) # configure Redis (for redis queue) - if app.testing: - from fakeredis import FakeStrictRedis - - redis_conn = FakeStrictRedis( - host="redis", port="1234" - ) # dummy connection details - else: + if not app.testing: redis_conn = Redis( app.config["FLEXMEASURES_REDIS_URL"], port=app.config["FLEXMEASURES_REDIS_PORT"], diff --git a/flexmeasures/utils/config_utils.py b/flexmeasures/utils/config_utils.py index e73c8191a..097ad8bdb 100644 --- a/flexmeasures/utils/config_utils.py +++ b/flexmeasures/utils/config_utils.py @@ -57,9 +57,7 @@ def configure_logging(): def check_app_env(env: str | None): if env not in ( - "documentation", "development", - "testing", "staging", "production", ): @@ -69,13 +67,37 @@ def check_app_env(env: str | None): sys.exit(2) -def read_config(app: Flask, custom_path_to_config: str | None): +def use_documentation_config(app: Flask, path_to_config: str | None = None): + """Use the default documentation config""" + app.config["ENV"] = "documentation" + app.config["FLEXMEASURES_ENV"] = "documentation" + app.config.from_object( + "flexmeasures.utils.config_defaults.%sConfig" % camelize("documentation") + ) + read_custom_config(app, path_to_config) + + +def use_testing_config(app: Flask): + """Use default the testing config""" + app.config["ENV"] = "testing" + app.config["FLEXMEASURES_ENV"] = "testing" + app.testing = True + app.config.from_object( + "flexmeasures.utils.config_defaults.%sConfig" % camelize("testing") + ) + custom_test_db_uri = os.getenv("SQLALCHEMY_TEST_DATABASE_URI", None) + if custom_test_db_uri: + app.config["SQLALCHEMY_DATABASE_URI"] = custom_test_db_uri + + +def read_config( + app: Flask, + custom_path_to_config: str | None, + env: str = DefaultConfig.FLEXMEASURES_ENV_DEFAULT, +): """Read configuration from various expected sources, complain if not setup correctly.""" - flexmeasures_env = DefaultConfig.FLEXMEASURES_ENV_DEFAULT - if app.testing: - flexmeasures_env = "testing" - elif os.getenv("FLEXMEASURES_ENV", None): + if os.getenv("FLEXMEASURES_ENV", None): flexmeasures_env = os.getenv("FLEXMEASURES_ENV", None) elif os.getenv("FLASK_ENV", None): flexmeasures_env = os.getenv("FLASK_ENV", None) @@ -83,6 +105,8 @@ def read_config(app: Flask, custom_path_to_config: str | None): "'FLASK_ENV' is deprecated and replaced by FLEXMEASURES_ENV" " Change FLASK_ENV to FLEXMEASURES_ENV in the environment variables", ) + else: + flexmeasures_env = env check_app_env(flexmeasures_env) @@ -93,39 +117,30 @@ def read_config(app: Flask, custom_path_to_config: str | None): # Now, potentially overwrite those from config file or environment variables - # These two locations are possible (besides the custom path) - path_to_config_home = str(Path.home().joinpath(".flexmeasures.cfg")) - path_to_config_instance = os.path.join(app.instance_path, "flexmeasures.cfg") - # Custom config: do not use any when testing (that should run completely on defaults) - if not app.testing: - used_path_to_config = read_custom_config( - app, custom_path_to_config, path_to_config_home, path_to_config_instance - ) - read_env_vars(app) - else: # one exception: the ability to set where the test database is - custom_test_db_uri = os.getenv("SQLALCHEMY_TEST_DATABASE_URI", None) - if custom_test_db_uri: - app.config["SQLALCHEMY_DATABASE_URI"] = custom_test_db_uri + + used_path_to_config = read_custom_config(app, custom_path_to_config) + read_env_vars(app) # Check for missing values. # Documentation runs fine without them. - if not app.testing and flexmeasures_env != "documentation": - if not are_required_settings_complete(app): - if not os.path.exists(used_path_to_config): - print( - f"You can provide these settings ― as environment variables or in your config file (e.g. {path_to_config_home} or {path_to_config_instance})." - ) - else: - print( - f"Please provide these settings ― as environment variables or in your config file ({used_path_to_config})." - ) - sys.exit(2) - missing_fields, config_warnings = get_config_warnings(app) - if len(config_warnings) > 0: - for warning in config_warnings: - print(f"Warning: {warning}") - print(f"You might consider setting {', '.join(missing_fields)}.") + path_to_config_home = str(Path.home().joinpath(".flexmeasures.cfg")) + path_to_config_instance = os.path.join(app.instance_path, "flexmeasures.cfg") + if not are_required_settings_complete(app): + if not os.path.exists(used_path_to_config): + print( + f"You can provide these settings ― as environment variables or in your config file (e.g. {path_to_config_home} or {path_to_config_instance})." + ) + else: + print( + f"Please provide these settings ― as environment variables or in your config file ({used_path_to_config})." + ) + sys.exit(2) + missing_fields, config_warnings = get_config_warnings(app) + if len(config_warnings) > 0: + for warning in config_warnings: + print(f"Warning: {warning}") + print(f"You might consider setting {', '.join(missing_fields)}.") # Set the desired logging level on the root logger (controlling extension logging level) # and this app's logger. @@ -137,7 +152,8 @@ def read_config(app: Flask, custom_path_to_config: str | None): def read_custom_config( - app: Flask, suggested_path_to_config, path_to_config_home, path_to_config_instance + app: Flask, + suggested_path_to_config, ) -> str: """ Read in a custom config file and env vars. @@ -147,6 +163,9 @@ def read_custom_config( Return the path to the config file. """ + # These two locations are possible (besides the custom path) + path_to_config_home = str(Path.home().joinpath(".flexmeasures.cfg")) + path_to_config_instance = os.path.join(app.instance_path, "flexmeasures.cfg") if suggested_path_to_config is not None and not os.path.exists( suggested_path_to_config ): From 722e6c7907f5feb4213547429179f4c0f280ac40 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 1 Feb 2024 17:41:13 +0100 Subject: [PATCH 2/4] fix: try adding the test db uri to the DocumentationConfig Signed-off-by: F.N. Claessen --- flexmeasures/utils/config_defaults.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/flexmeasures/utils/config_defaults.py b/flexmeasures/utils/config_defaults.py index 512fe13ff..72732dc06 100644 --- a/flexmeasures/utils/config_defaults.py +++ b/flexmeasures/utils/config_defaults.py @@ -210,3 +210,6 @@ class TestingConfig(Config): class DocumentationConfig(Config): SECRET_KEY: str = "dummy-key-for-documentation" + SQLALCHEMY_DATABASE_URI: str = ( + "postgresql://flexmeasures_test:flexmeasures_test@localhost/flexmeasures_test" + ) From 25f474720606695faad28589f75434461e629562 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 1 Feb 2024 18:36:19 +0100 Subject: [PATCH 3/4] style: clarify use of dummy config setting Signed-off-by: F.N. Claessen --- flexmeasures/utils/config_defaults.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/flexmeasures/utils/config_defaults.py b/flexmeasures/utils/config_defaults.py index 72732dc06..286fa499d 100644 --- a/flexmeasures/utils/config_defaults.py +++ b/flexmeasures/utils/config_defaults.py @@ -210,6 +210,4 @@ class TestingConfig(Config): class DocumentationConfig(Config): SECRET_KEY: str = "dummy-key-for-documentation" - SQLALCHEMY_DATABASE_URI: str = ( - "postgresql://flexmeasures_test:flexmeasures_test@localhost/flexmeasures_test" - ) + SQLALCHEMY_DATABASE_URI: str = "dummy-uri-for-documentation" From 161751a002400be390bb155ec089f8b3961d1b0f Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Thu, 1 Feb 2024 18:45:26 +0100 Subject: [PATCH 4/4] fix: parseable dummy config setting Signed-off-by: F.N. Claessen --- flexmeasures/utils/config_defaults.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/utils/config_defaults.py b/flexmeasures/utils/config_defaults.py index 286fa499d..9f1117cb3 100644 --- a/flexmeasures/utils/config_defaults.py +++ b/flexmeasures/utils/config_defaults.py @@ -210,4 +210,4 @@ class TestingConfig(Config): class DocumentationConfig(Config): SECRET_KEY: str = "dummy-key-for-documentation" - SQLALCHEMY_DATABASE_URI: str = "dummy-uri-for-documentation" + SQLALCHEMY_DATABASE_URI: str = "postgresql://dummy:uri@for/documentation"