Merge branch 'main' into m62054_exintrinsics

.github/problem-matchers.json

@@ -13,7 +13,7 @@
       "owner": "loguru",
       "pattern": [
         {
-          "regexp": "(ERROR|WARNING).*\\|.+(?:\\[3[13]m.\\[1m)(?:(.*).\\[0m)$",
+          "regexp": "^(ERROR|WARNING)\\s*:\\s(.+)$",
           "severity": 1,
           "message": 2
         }

src/common/__init__.py

@@ -2,9 +2,10 @@
 import json
 import requests
 from glob import glob
-from loguru import logger
 from functools import cache
 
+from .unique_logger import logger
+
 MP4RA_BOXES_URL = "https://mp4ra.org/boxes.json"
 DOCUMENT_STATUS_URL = (
     "https://github.com/MPEGGroup/FileFormat/blob/master/DocumentStatus.md"

src/common/unique_logger.py

@@ -0,0 +1,40 @@
+import os
+import sys
+from loguru import logger as _logger
+
+
+class LoggerWrapper:
+    DEDUPLICATE = ["debug", "info", "success", "warning", "error", "critical"]
+
+    def __init__(self):
+        self.messages = set()
+
+    def _process(self, severity, message, **kwargs):
+        if message not in self.messages:
+            self.messages.add(message)
+            function = getattr(_logger, severity)
+            function(message, **kwargs)
+
+    def __getattribute__(self, __name: str):
+        if __name in ["_process", "messages"]:
+            return object.__getattribute__(self, __name)
+
+        if __name in LoggerWrapper.DEDUPLICATE:
+            return lambda message, **kwargs: self._process(__name, message, **kwargs)
+        return getattr(_logger, __name)
+
+
+logger = LoggerWrapper()
+
+# If running in CI, don't add colors
+if "CI" in os.environ:
+    fmt = "{level: <8}: {message}"
+else:
+    fmt = "<level>{level: <8}: {message}</level>"
+
+# Remove default sink
+logger.remove(0)
+
+# Add sinks
+logger.add(sys.stderr, format=fmt)
+logger.add("/tmp/construct.log", format=fmt, level="ERROR")

src/construct/boxes.py

@@ -3,10 +3,10 @@
 import os
 import json
 from glob import glob
-from loguru import logger
 from functools import cache
 from dataclasses import dataclass, field
 
+from common.unique_logger import logger
 from common import get_mp4ra_boxes
 
 BOXES = {}
@@ -160,7 +160,6 @@ def update_container(_spec, _box):
 
 
 def main():
-    logger.add("/tmp/construct.log", level="ERROR")
     files = glob("../data/standard_features/**/*.json")
 
     # Ignore unrelated files

src/construct/construct_all.py

@@ -1,4 +1,4 @@
-from loguru import logger
+from common.unique_logger import logger
 
 from construct.boxes import main as construct_boxes_main
 from construct.dictionary import main as construct_dictionary_main

src/construct/coverage.py

@@ -1,13 +1,11 @@
 import json
 from glob import glob
-from loguru import logger
 
+from common.unique_logger import logger
 from common import *
 
 
 def main():
-    logger.add("/tmp/construct.log", level="ERROR")
-
     with open("output/dictionary.json", "r", encoding="utf-8") as f:
         dictionary = json.load(f)
 
@@ -78,9 +76,11 @@ def main():
     extensions = list(set(extensions) - ignored)
 
     missing_extensions = set()
+    fourcc_in_extensions = set()
     for extension in extensions:
         with open(extension, "r", encoding="utf-8") as f:
             ext_data = json.load(f)
+        fourcc_in_extensions.update([e["box"]["@Type"] for e in ext_data["extensions"]])
         missing_extensions.update(
             [f"{e['location']}.{e['box']['@Type']}" for e in ext_data["extensions"]]
         )
@@ -106,7 +106,10 @@ def main():
 
         if not known_box:
             # Check if this was in under consideration files
-            if any(["under_consideration" in f for f in in_files]):
+            if (
+                any(["under_consideration" in f for f in in_files])
+                and box_fourcc in fourcc_in_extensions
+            ):
                 extra = ""
                 if box_fourcc in get_mp4ra_boxes():
                     extra = " It exists in MP4RA though."

src/construct/files.py

@@ -1,8 +1,8 @@
 import os
 import json
 from glob import glob
-from loguru import logger
 
+from common.unique_logger import logger
 from common import get_ignored_files, get_mp4ra_boxes
 
 
@@ -112,8 +112,6 @@ def crawl(root, path):
 
 
 def main():
-    logger.add("/tmp/construct.log", level="ERROR")
-
     # Check which boxes can be found anywhere (container.fourcc=* && container.type=*)
     can_be_found_anywhere = set()
     with open("output/boxes.json", "r", encoding="utf-8") as f:

src/construct/hierarchy.py

@@ -1,8 +1,9 @@
 import json
-from loguru import logger
 from dataclasses import dataclass
 from argparse import ArgumentParser
 
+from common.unique_logger import logger
+
 
 @dataclass
 class Box:
@@ -317,8 +318,6 @@ def build_farm(_data, ruleset):
 
 
 def main():
-    logger.add("/tmp/construct.log", level="ERROR")
-
     parser = ArgumentParser()
     parser.add_argument(
         "-r",