From 027f6b23d7edbeb164599bb065fed11162bb7efa Mon Sep 17 00:00:00 2001 From: Deniz Date: Thu, 19 Oct 2023 00:24:22 -0700 Subject: [PATCH 1/3] proper logger wrapper > Leaner format > Deduplicates messages --- .github/problem-matchers.json | 2 +- src/common/__init__.py | 1 - src/common/unique_logger.py | 40 ++++++++++++++++++++++++++++++++++ src/construct/boxes.py | 3 +-- src/construct/construct_all.py | 2 +- src/construct/coverage.py | 4 +--- src/construct/files.py | 4 +--- src/construct/hierarchy.py | 5 ++--- 8 files changed, 47 insertions(+), 14 deletions(-) create mode 100644 src/common/unique_logger.py diff --git a/.github/problem-matchers.json b/.github/problem-matchers.json index 2bdd41d..c853bf4 100644 --- a/.github/problem-matchers.json +++ b/.github/problem-matchers.json @@ -13,7 +13,7 @@ "owner": "loguru", "pattern": [ { - "regexp": "(ERROR|WARNING).*\\|.+(?:\\[3[13]m.\\[1m)(?:(.*).\\[0m)$", + "regexp": "^(ERROR|WARNING)\\s*:\\s(.+)$", "severity": 1, "message": 2 } diff --git a/src/common/__init__.py b/src/common/__init__.py index 36c07b4..04d0736 100644 --- a/src/common/__init__.py +++ b/src/common/__init__.py @@ -2,7 +2,6 @@ import json import requests from glob import glob -from loguru import logger from functools import cache MP4RA_BOXES_URL = "https://mp4ra.org/boxes.json" diff --git a/src/common/unique_logger.py b/src/common/unique_logger.py new file mode 100644 index 0000000..343b3af --- /dev/null +++ b/src/common/unique_logger.py @@ -0,0 +1,40 @@ +import os +import sys +from loguru import logger as _logger + + +class LoggerWrapper: + DEDUPLICATE = ["debug", "info", "success", "warning", "error", "critical"] + + def __init__(self): + self.messages = set() + + def _process(self, severity, message, **kwargs): + if message not in self.messages: + self.messages.add(message) + function = getattr(_logger, severity) + function(message, **kwargs) + + def __getattribute__(self, __name: str): + if __name in ["_process", "messages"]: + return object.__getattribute__(self, __name) + + if __name in LoggerWrapper.DEDUPLICATE: + return lambda message, **kwargs: self._process(__name, message, **kwargs) + return getattr(_logger, __name) + + +logger = LoggerWrapper() + +# If running in CI, don't add colors +if "CI" in os.environ: + fmt = "{level: <8}: {message}" +else: + fmt = "{level: <8}: {message}" + +# Remove default sink +logger.remove(0) + +# Add sinks +logger.add(sys.stderr, format=fmt) +logger.add("/tmp/construct.log", format=fmt, level="ERROR") diff --git a/src/construct/boxes.py b/src/construct/boxes.py index 8411e65..ff15102 100644 --- a/src/construct/boxes.py +++ b/src/construct/boxes.py @@ -3,10 +3,10 @@ import os import json from glob import glob -from loguru import logger from functools import cache from dataclasses import dataclass, field +from common.unique_logger import logger from common import get_mp4ra_boxes BOXES = {} @@ -160,7 +160,6 @@ def update_container(_spec, _box): def main(): - logger.add("/tmp/construct.log", level="ERROR") files = glob("../data/standard_features/**/*.json") # Ignore unrelated files diff --git a/src/construct/construct_all.py b/src/construct/construct_all.py index 9f611c8..65ffc1b 100644 --- a/src/construct/construct_all.py +++ b/src/construct/construct_all.py @@ -1,4 +1,4 @@ -from loguru import logger +from common.unique_logger import logger from construct.boxes import main as construct_boxes_main from construct.dictionary import main as construct_dictionary_main diff --git a/src/construct/coverage.py b/src/construct/coverage.py index 35ae05d..812bf47 100644 --- a/src/construct/coverage.py +++ b/src/construct/coverage.py @@ -1,13 +1,11 @@ import json from glob import glob -from loguru import logger +from common.unique_logger import logger from common import * def main(): - logger.add("/tmp/construct.log", level="ERROR") - with open("output/dictionary.json", "r", encoding="utf-8") as f: dictionary = json.load(f) diff --git a/src/construct/files.py b/src/construct/files.py index c7886e2..0ce1bdb 100644 --- a/src/construct/files.py +++ b/src/construct/files.py @@ -1,8 +1,8 @@ import os import json from glob import glob -from loguru import logger +from common.unique_logger import logger from common import get_ignored_files, get_mp4ra_boxes @@ -112,8 +112,6 @@ def crawl(root, path): def main(): - logger.add("/tmp/construct.log", level="ERROR") - # Check which boxes can be found anywhere (container.fourcc=* && container.type=*) can_be_found_anywhere = set() with open("output/boxes.json", "r", encoding="utf-8") as f: diff --git a/src/construct/hierarchy.py b/src/construct/hierarchy.py index 0129606..1e140a4 100644 --- a/src/construct/hierarchy.py +++ b/src/construct/hierarchy.py @@ -1,8 +1,9 @@ import json -from loguru import logger from dataclasses import dataclass from argparse import ArgumentParser +from common.unique_logger import logger + @dataclass class Box: @@ -317,8 +318,6 @@ def build_farm(_data, ruleset): def main(): - logger.add("/tmp/construct.log", level="ERROR") - parser = ArgumentParser() parser.add_argument( "-r", From e3ef417d68d3473584524efc0f8f15560e16e28a Mon Sep 17 00:00:00 2001 From: Deniz Date: Thu, 19 Oct 2023 00:24:27 -0700 Subject: [PATCH 2/3] only UnknownBox should raise error --- src/construct/coverage.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/construct/coverage.py b/src/construct/coverage.py index 812bf47..330ca09 100644 --- a/src/construct/coverage.py +++ b/src/construct/coverage.py @@ -76,9 +76,11 @@ def main(): extensions = list(set(extensions) - ignored) missing_extensions = set() + fourcc_in_extensions = set() for extension in extensions: with open(extension, "r", encoding="utf-8") as f: ext_data = json.load(f) + fourcc_in_extensions.update([e["box"]["@Type"] for e in ext_data["extensions"]]) missing_extensions.update( [f"{e['location']}.{e['box']['@Type']}" for e in ext_data["extensions"]] ) @@ -104,7 +106,10 @@ def main(): if not known_box: # Check if this was in under consideration files - if any(["under_consideration" in f for f in in_files]): + if ( + any(["under_consideration" in f for f in in_files]) + and box_fourcc in fourcc_in_extensions + ): extra = "" if box_fourcc in get_mp4ra_boxes(): extra = " It exists in MP4RA though." From c0d12bf43b02ead139396cab40a31201680a3cf2 Mon Sep 17 00:00:00 2001 From: Deniz Date: Thu, 19 Oct 2023 00:29:55 -0700 Subject: [PATCH 3/3] missing logger import --- src/common/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/common/__init__.py b/src/common/__init__.py index 04d0736..7d7cd06 100644 --- a/src/common/__init__.py +++ b/src/common/__init__.py @@ -4,6 +4,8 @@ from glob import glob from functools import cache +from .unique_logger import logger + MP4RA_BOXES_URL = "https://mp4ra.org/boxes.json" DOCUMENT_STATUS_URL = ( "https://github.com/MPEGGroup/FileFormat/blob/master/DocumentStatus.md"