From 276eba73c3b2cb14cf3c24ced7e9ccee64266f22 Mon Sep 17 00:00:00 2001 From: spielman Date: Tue, 17 Mar 2026 16:06:12 -0400 Subject: [PATCH 01/66] Extract reusable shot queue widget Add a reusable ShotQueueWidget and ShotQueueTreeView in labscript_utils.qtwidgets for queue editing and state restore. --- labscript_utils/qtwidgets/__init__.py | 4 +- labscript_utils/qtwidgets/shotqueue.py | 359 +++++++++++++++++++++++++ 2 files changed, 362 insertions(+), 1 deletion(-) create mode 100644 labscript_utils/qtwidgets/shotqueue.py diff --git a/labscript_utils/qtwidgets/__init__.py b/labscript_utils/qtwidgets/__init__.py index af02b0ad..5ec168d4 100644 --- a/labscript_utils/qtwidgets/__init__.py +++ b/labscript_utils/qtwidgets/__init__.py @@ -9,4 +9,6 @@ # BSD License. See the license.txt file in the root of the project # # for the full license. # # # -##################################################################### \ No newline at end of file +##################################################################### + +from .shotqueue import FILEPATH_COLUMN, ShotQueueTreeView, ShotQueueWidget diff --git a/labscript_utils/qtwidgets/shotqueue.py b/labscript_utils/qtwidgets/shotqueue.py new file mode 100644 index 00000000..5b2cacc3 --- /dev/null +++ b/labscript_utils/qtwidgets/shotqueue.py @@ -0,0 +1,359 @@ +##################################################################### +# # +# shotqueue.py # +# # +# Copyright 2026, Monash University # +# # +# This file is part of the labscript suite (see # +# http://labscriptsuite.org) and is licensed under the Simplified # +# BSD License. See the license.txt file in the root of the project # +# for the full license. # +# # +##################################################################### +import os + +from qtutils.qt.QtCore import * +from qtutils.qt.QtGui import * +from qtutils.qt.QtWidgets import * +from qtutils.qt.QtCore import pyqtSignal as Signal + + +FILEPATH_COLUMN = 0 +__all__ = ['FILEPATH_COLUMN', 'ShotQueueTreeView', 'ShotQueueWidget'] + + +class ShotQueueTreeView(QTreeView): + """Generic queue view with delete-key handling and shot-file drops.""" + + deleteRequested = Signal() + filesDropped = Signal(list) + + def __init__(self, parent=None, accepted_extensions=None): + QTreeView.__init__(self, parent) + self._accepted_extensions = self._normalise_extensions(accepted_extensions) + self.header().setStretchLastSection(True) + self.setAutoScroll(False) + self.setAcceptDrops(True) + self.setDragEnabled(True) + self.setDropIndicatorShown(True) + self.setDragDropMode(QAbstractItemView.DragDrop) + self.setDefaultDropAction(Qt.MoveAction) + self.setSelectionBehavior(QAbstractItemView.SelectRows) + self.setSelectionMode(QAbstractItemView.ExtendedSelection) + self.setRootIsDecorated(False) + self.setUniformRowHeights(True) + + def accepted_extensions(self): + return tuple(self._accepted_extensions) + + def set_accepted_extensions(self, accepted_extensions): + self._accepted_extensions = self._normalise_extensions(accepted_extensions) + + def keyPressEvent(self, event): + if event.key() == Qt.Key_Delete: + event.accept() + self.deleteRequested.emit() + return + QTreeView.keyPressEvent(self, event) + + def dragEnterEvent(self, event): + if self._event_has_acceptable_urls(event): + event.setDropAction(Qt.CopyAction) + event.accept() + return + QTreeView.dragEnterEvent(self, event) + + def dragMoveEvent(self, event): + if self._event_has_acceptable_urls(event): + event.setDropAction(Qt.CopyAction) + event.accept() + return + QTreeView.dragMoveEvent(self, event) + + def dropEvent(self, event): + if self._event_has_acceptable_urls(event): + paths = self._paths_from_urls(event.mimeData().urls()) + if paths: + event.setDropAction(Qt.CopyAction) + event.accept() + self.filesDropped.emit(paths) + return + QTreeView.dropEvent(self, event) + + def _event_has_acceptable_urls(self, event): + if not event.mimeData().hasUrls(): + return False + return bool(self._paths_from_urls(event.mimeData().urls())) + + def _paths_from_urls(self, urls): + paths = [] + for url in urls: + if not url.isLocalFile(): + continue + path = os.path.abspath(str(url.toLocalFile())) + if self._is_accepted_path(path): + paths.append(path) + return paths + + def _is_accepted_path(self, path): + return os.path.isfile(path) and path.lower().endswith(self._accepted_extensions) + + def _normalise_extensions(self, accepted_extensions): + if accepted_extensions is None: + accepted_extensions = ('.h5', '.hdf5') + return tuple(extension.lower() for extension in accepted_extensions) + + +class ShotQueueWidget(QWidget): + """Reusable single-column shot queue editor widget.""" + + queueChanged = Signal() + selectionChanged = Signal(list) + filesAdded = Signal(list) + + def __init__( + self, + parent=None, + accepted_extensions=None, + file_dialog_filter='Shot files (*.h5 *.hdf5)', + allow_duplicates=False, + column_title='Filepath', + ): + QWidget.__init__(self, parent) + self.accepted_extensions = self._normalise_extensions(accepted_extensions) + self.file_dialog_filter = file_dialog_filter + self.allow_duplicates = allow_duplicates + self.last_opened_shots_folder = '' + + self.queue_model = QStandardItemModel(self) + self.queue_model.setHorizontalHeaderItem(FILEPATH_COLUMN, QStandardItem(column_title)) + + self.queue_view = ShotQueueTreeView(self, accepted_extensions=self.accepted_extensions) + self.queue_view.setModel(self.queue_model) + + self.add_button = QToolButton(self) + self.add_button.setText('Add') + self.delete_button = QToolButton(self) + self.delete_button.setText('Delete') + self.clear_button = QToolButton(self) + self.clear_button.setText('Clear') + self.move_top_button = QToolButton(self) + self.move_top_button.setText('Top') + self.move_up_button = QToolButton(self) + self.move_up_button.setText('Up') + self.move_down_button = QToolButton(self) + self.move_down_button.setText('Down') + self.move_bottom_button = QToolButton(self) + self.move_bottom_button.setText('Bottom') + + button_layout = QHBoxLayout() + button_layout.setContentsMargins(0, 0, 0, 0) + button_layout.addWidget(self.add_button) + button_layout.addWidget(self.delete_button) + button_layout.addWidget(self.clear_button) + button_layout.addStretch(1) + button_layout.addWidget(self.move_top_button) + button_layout.addWidget(self.move_up_button) + button_layout.addWidget(self.move_down_button) + button_layout.addWidget(self.move_bottom_button) + + layout = QVBoxLayout(self) + layout.setContentsMargins(0, 0, 0, 0) + layout.addWidget(self.queue_view) + layout.addLayout(button_layout) + + self.add_button.clicked.connect(self.prompt_for_files) + self.delete_button.clicked.connect(self.remove_selected) + self.clear_button.clicked.connect(self.clear) + self.move_top_button.clicked.connect(self.move_top) + self.move_up_button.clicked.connect(self.move_up) + self.move_down_button.clicked.connect(self.move_down) + self.move_bottom_button.clicked.connect(self.move_bottom) + self.queue_view.deleteRequested.connect(self.remove_selected) + self.queue_view.filesDropped.connect(self.add_files) + self.queue_model.rowsInserted.connect(self._on_queue_changed) + self.queue_model.rowsRemoved.connect(self._on_queue_changed) + self.queue_model.modelReset.connect(self._on_queue_changed) + self.queue_view.selectionModel().selectionChanged.connect(self._on_selection_changed) + + self._update_button_states() + + def files(self): + return [ + self.queue_model.item(row, FILEPATH_COLUMN).text() + for row in range(self.queue_model.rowCount()) + ] + + def selected_files(self): + return [self.queue_model.item(row, FILEPATH_COLUMN).text() for row in self.selected_rows()] + + def selected_rows(self): + return sorted(index.row() for index in self.queue_view.selectionModel().selectedRows()) + + def select_paths(self, paths): + if isinstance(paths, str): + paths = [paths] + path_lookup = {os.path.abspath(str(path)) for path in paths} + rows = [] + for row in range(self.queue_model.rowCount()): + item = self.queue_model.item(row, FILEPATH_COLUMN) + if item.text() in path_lookup: + rows.append(row) + self._select_rows(rows) + + def set_files(self, paths): + self.queue_model.removeRows(0, self.queue_model.rowCount()) + self.append(paths) + + def append(self, paths): + paths = self._prepare_paths(paths) + if not paths: + return [] + for path in paths: + self.queue_model.appendRow(self._create_row(path)) + self.filesAdded.emit(paths) + return paths + + def prepend(self, path): + paths = self._prepare_paths([path]) + if not paths: + return [] + self.queue_model.insertRow(0, self._create_row(paths[0])) + self.filesAdded.emit(paths) + self._select_rows([0]) + return paths + + def add_files(self, paths): + added = self.append(paths) + if added: + self.last_opened_shots_folder = os.path.dirname(added[-1]) + first_new_row = self.queue_model.rowCount() - len(added) + self._select_rows(range(first_new_row, self.queue_model.rowCount())) + return added + + def prompt_for_files(self): + file_names = QFileDialog.getOpenFileNames( + self, + 'Select shot file(s) to add to the queue', + self.last_opened_shots_folder, + self.file_dialog_filter, + ) + if isinstance(file_names, tuple): + file_names, _ = file_names + return self.add_files(file_names) + + def remove_selected(self): + for row in reversed(self.selected_rows()): + self.queue_model.removeRow(row) + + def clear(self): + if self.queue_model.rowCount(): + self.queue_model.removeRows(0, self.queue_model.rowCount()) + + def move_up(self): + selected_rows = self.selected_rows() + if not selected_rows or selected_rows[0] == 0: + return + for row_index in selected_rows: + self.queue_model.insertRow(row_index - 1, self.queue_model.takeRow(row_index)) + self._select_rows([row - 1 for row in selected_rows]) + + def move_down(self): + selected_rows = self.selected_rows() + if not selected_rows or selected_rows[-1] == self.queue_model.rowCount() - 1: + return + for row_index in reversed(selected_rows): + self.queue_model.insertRow(row_index + 1, self.queue_model.takeRow(row_index)) + self._select_rows([row + 1 for row in selected_rows]) + + def move_top(self): + selected_rows = self.selected_rows() + if not selected_rows or selected_rows[0] == 0: + return + rows = [self.queue_model.takeRow(row_index) for row_index in reversed(selected_rows)] + for offset, row_items in enumerate(reversed(rows)): + self.queue_model.insertRow(offset, row_items) + self._select_rows(range(len(selected_rows))) + + def move_bottom(self): + selected_rows = self.selected_rows() + if not selected_rows or selected_rows[-1] == self.queue_model.rowCount() - 1: + return + rows = [self.queue_model.takeRow(row_index) for row_index in reversed(selected_rows)] + start_row = self.queue_model.rowCount() + for row_items in reversed(rows): + self.queue_model.appendRow(row_items) + self._select_rows(range(start_row, self.queue_model.rowCount())) + + def is_in_queue(self, path): + path = os.path.abspath(str(path)) + return bool(self.queue_model.findItems(path, column=FILEPATH_COLUMN)) + + def get_save_data(self): + return { + 'files_queued': self.files(), + 'last_opened_shots_folder': self.last_opened_shots_folder, + } + + def restore_save_data(self, data): + self.last_opened_shots_folder = data.get('last_opened_shots_folder', '') + self.set_files(data.get('files_queued', [])) + + def _create_row(self, path): + item = QStandardItem(path) + item.setToolTip(path) + item.setEditable(False) + return [item] + + def _prepare_paths(self, paths): + if isinstance(paths, str): + paths = [paths] + existing = set(self.files()) + pending = set() + prepared = [] + for path in paths: + path = os.path.abspath(str(path)) + if not self._is_accepted_path(path): + continue + if not self.allow_duplicates and (path in existing or path in pending): + continue + pending.add(path) + prepared.append(path) + return prepared + + def _is_accepted_path(self, path): + return os.path.isfile(path) and path.lower().endswith(self.accepted_extensions) + + def _on_queue_changed(self, *args): + self._update_button_states() + self.queueChanged.emit() + + def _on_selection_changed(self, *args): + self._update_button_states() + self.selectionChanged.emit(self.selected_files()) + + def _update_button_states(self): + row_count = self.queue_model.rowCount() + selected_rows = self.selected_rows() + has_selection = bool(selected_rows) + self.delete_button.setEnabled(has_selection) + self.clear_button.setEnabled(bool(row_count)) + self.move_top_button.setEnabled(has_selection and selected_rows[0] > 0) + self.move_up_button.setEnabled(has_selection and selected_rows[0] > 0) + self.move_bottom_button.setEnabled(has_selection and selected_rows[-1] < row_count - 1) + self.move_down_button.setEnabled(has_selection and selected_rows[-1] < row_count - 1) + + def _select_rows(self, rows): + rows = list(rows) + selection_model = self.queue_view.selectionModel() + selection_model.clearSelection() + for row in rows: + index = self.queue_model.index(row, FILEPATH_COLUMN) + selection_model.select(index, QItemSelectionModel.Select | QItemSelectionModel.Rows) + if rows: + self.queue_view.setCurrentIndex(self.queue_model.index(rows[0], FILEPATH_COLUMN)) + + def _normalise_extensions(self, accepted_extensions): + if accepted_extensions is None: + accepted_extensions = ('.h5', '.hdf5') + return tuple(extension.lower() for extension in accepted_extensions) From cbbcc4f0c36e0c1b9905212ff26fc8c877998f2c Mon Sep 17 00:00:00 2001 From: spielman Date: Tue, 17 Mar 2026 16:59:40 -0400 Subject: [PATCH 02/66] Fix shot queue widget edge cases Normalize accepted_extensions consistently so a single string like '.h5' is treated as one extension instead of being iterated character by character. Make labscript_utils.qtwidgets lazily expose the new shot queue classes so importing the package no longer eagerly imports Qt-dependent code. --- labscript_utils/qtwidgets/__init__.py | 21 ++++++++++++++++++++- labscript_utils/qtwidgets/shotqueue.py | 25 +++++++++++-------------- 2 files changed, 31 insertions(+), 15 deletions(-) diff --git a/labscript_utils/qtwidgets/__init__.py b/labscript_utils/qtwidgets/__init__.py index 5ec168d4..3f90f1c0 100644 --- a/labscript_utils/qtwidgets/__init__.py +++ b/labscript_utils/qtwidgets/__init__.py @@ -11,4 +11,23 @@ # # ##################################################################### -from .shotqueue import FILEPATH_COLUMN, ShotQueueTreeView, ShotQueueWidget +__all__ = ['FILEPATH_COLUMN', 'ShotQueueTreeView', 'ShotQueueWidget'] + + +def __getattr__(name): + if name not in __all__: + raise AttributeError("module %r has no attribute %r" % (__name__, name)) + + from .shotqueue import FILEPATH_COLUMN, ShotQueueTreeView, ShotQueueWidget + + exports = { + 'FILEPATH_COLUMN': FILEPATH_COLUMN, + 'ShotQueueTreeView': ShotQueueTreeView, + 'ShotQueueWidget': ShotQueueWidget, + } + globals().update(exports) + return exports[name] + + +def __dir__(): + return sorted(list(globals().keys()) + __all__) diff --git a/labscript_utils/qtwidgets/shotqueue.py b/labscript_utils/qtwidgets/shotqueue.py index 5b2cacc3..df95b23b 100644 --- a/labscript_utils/qtwidgets/shotqueue.py +++ b/labscript_utils/qtwidgets/shotqueue.py @@ -22,6 +22,14 @@ __all__ = ['FILEPATH_COLUMN', 'ShotQueueTreeView', 'ShotQueueWidget'] +def _normalise_extensions(accepted_extensions): + if accepted_extensions is None: + accepted_extensions = ('.h5', '.hdf5') + elif isinstance(accepted_extensions, str): + accepted_extensions = (accepted_extensions,) + return tuple(extension.lower() for extension in accepted_extensions) + + class ShotQueueTreeView(QTreeView): """Generic queue view with delete-key handling and shot-file drops.""" @@ -30,7 +38,7 @@ class ShotQueueTreeView(QTreeView): def __init__(self, parent=None, accepted_extensions=None): QTreeView.__init__(self, parent) - self._accepted_extensions = self._normalise_extensions(accepted_extensions) + self._accepted_extensions = _normalise_extensions(accepted_extensions) self.header().setStretchLastSection(True) self.setAutoScroll(False) self.setAcceptDrops(True) @@ -47,7 +55,7 @@ def accepted_extensions(self): return tuple(self._accepted_extensions) def set_accepted_extensions(self, accepted_extensions): - self._accepted_extensions = self._normalise_extensions(accepted_extensions) + self._accepted_extensions = _normalise_extensions(accepted_extensions) def keyPressEvent(self, event): if event.key() == Qt.Key_Delete: @@ -98,12 +106,6 @@ def _paths_from_urls(self, urls): def _is_accepted_path(self, path): return os.path.isfile(path) and path.lower().endswith(self._accepted_extensions) - def _normalise_extensions(self, accepted_extensions): - if accepted_extensions is None: - accepted_extensions = ('.h5', '.hdf5') - return tuple(extension.lower() for extension in accepted_extensions) - - class ShotQueueWidget(QWidget): """Reusable single-column shot queue editor widget.""" @@ -120,7 +122,7 @@ def __init__( column_title='Filepath', ): QWidget.__init__(self, parent) - self.accepted_extensions = self._normalise_extensions(accepted_extensions) + self.accepted_extensions = _normalise_extensions(accepted_extensions) self.file_dialog_filter = file_dialog_filter self.allow_duplicates = allow_duplicates self.last_opened_shots_folder = '' @@ -352,8 +354,3 @@ def _select_rows(self, rows): selection_model.select(index, QItemSelectionModel.Select | QItemSelectionModel.Rows) if rows: self.queue_view.setCurrentIndex(self.queue_model.index(rows[0], FILEPATH_COLUMN)) - - def _normalise_extensions(self, accepted_extensions): - if accepted_extensions is None: - accepted_extensions = ('.h5', '.hdf5') - return tuple(extension.lower() for extension in accepted_extensions) From e44fbe1c417a9cc0ed604e797534c7df7d112496 Mon Sep 17 00:00:00 2001 From: spielman Date: Tue, 17 Mar 2026 19:32:28 -0400 Subject: [PATCH 03/66] Make excepthook warning hook idempotent --- labscript_utils/excepthook/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/labscript_utils/excepthook/__init__.py b/labscript_utils/excepthook/__init__.py index 9ac5975d..c6551fc8 100644 --- a/labscript_utils/excepthook/__init__.py +++ b/labscript_utils/excepthook/__init__.py @@ -28,6 +28,7 @@ class l: logger = None child_processes = [] +_original_showwarning = warnings.showwarning def install_thread_excepthook(): @@ -83,12 +84,11 @@ def tkhandler(exceptclass, exception, exec_info, reraise=True): def logwarning(message, category, filename, lineno, file=None, line=None): logmessage = warnings.formatwarning(message, category, filename, lineno, line) l.logger.warn(logmessage) - warnings._showwarning(message, category, filename, lineno, file, line) + _original_showwarning(message, category, filename, lineno, file, line) def set_logger(logger): l.logger = logger - warnings._showwarning = warnings.showwarning warnings.showwarning = logwarning # Check for tkinter availability. Tkinter is frustratingly not available From b522ed3ea3e1a5208c6100101a34933630dcf193 Mon Sep 17 00:00:00 2001 From: spielman Date: Tue, 17 Mar 2026 19:34:14 -0400 Subject: [PATCH 04/66] Fix SecureSocket detection in ls_zprocess Context --- labscript_utils/ls_zprocess.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/labscript_utils/ls_zprocess.py b/labscript_utils/ls_zprocess.py index 4d9e9575..29c94d52 100644 --- a/labscript_utils/ls_zprocess.py +++ b/labscript_utils/ls_zprocess.py @@ -257,7 +257,7 @@ def socket(self, socket_type, socket_class=None, **kwargs): # be a SecureSocket. If caller has explicitly requested a different socket type # (e.g since pyzmq 25, ThreadAuthenticator sets up an internal socket by calling # `Context.socket(..., socket_class=zmq.Socket)), then don't.` - if socket_class is None or issubclass(socket_class, SecureContext): + if socket_class is None or issubclass(socket_class, SecureSocket): config = get_config() kwargs['allow_insecure'] = config['allow_insecure'] return SecureContext.socket(self, socket_type=socket_type, **kwargs) @@ -383,4 +383,3 @@ def ensure_connected_to_zlog(): else: client.ping() _connected_to_zlog = True - From 502563328a5d320c71b8efec2bee0008fa6240fa Mon Sep 17 00:00:00 2001 From: spielman Date: Tue, 17 Mar 2026 19:35:04 -0400 Subject: [PATCH 05/66] Support NumPy scalar JSON serialization --- labscript_utils/properties.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/labscript_utils/properties.py b/labscript_utils/properties.py index 8c2402fe..d31e7245 100644 --- a/labscript_utils/properties.py +++ b/labscript_utils/properties.py @@ -62,6 +62,10 @@ def _default(o): # Workaround for https://bugs.python.org/issue24313 if isinstance(o, np.integer): return int(o) + if isinstance(o, np.floating): + return float(o) + if isinstance(o, np.bool_): + return bool(o) raise TypeError From 5ccbfba447e006feecc21fa7d2ce796f0c7629b9 Mon Sep 17 00:00:00 2001 From: spielman Date: Wed, 18 Mar 2026 09:39:43 -0400 Subject: [PATCH 06/66] Configure shared Qt application defaults --- .../default_profile/labconfig/example.ini | 2 ++ labscript_utils/splash.py | 30 +++++++++++++++++-- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/labscript_profile/default_profile/labconfig/example.ini b/labscript_profile/default_profile/labconfig/example.ini index 7f219dc1..94a6c16f 100644 --- a/labscript_profile/default_profile/labconfig/example.ini +++ b/labscript_profile/default_profile/labconfig/example.ini @@ -16,6 +16,8 @@ connection_table_py = %(labscriptlib)s\connection_table.py [servers] zlock = localhost runmanager = localhost +blacs = localhost +lyse = localhost [ports] BLACS = 42517 diff --git a/labscript_utils/splash.py b/labscript_utils/splash.py index 51c56864..a80ab333 100644 --- a/labscript_utils/splash.py +++ b/labscript_utils/splash.py @@ -39,6 +39,32 @@ QtWidgets.QApplication.setAttribute(Qt.AA_UseHighDpiPixmaps, True) +def configure_qapplication(qapplication): + """Apply labscript-wide QApplication configuration.""" + if qapplication.property('_labscript_qapplication_configured'): + return qapplication + qapplication.setAttribute(Qt.AA_DontShowIconsInMenus, False) + if sys.platform == 'darwin': + # Native macOS widget styling makes many Qt controls look inconsistent + # with the rest of the suite. Use Qt's own style, but preserve the + # current palette so dark/light appearance still follows the active + # theme. + palette = QtGui.QPalette(qapplication.palette()) + style = QtWidgets.QStyleFactory.create('Fusion') + if style is not None: + qapplication.setStyle(style) + qapplication.setPalette(palette) + qapplication.setProperty('_labscript_qapplication_configured', True) + return qapplication + + +def get_qapplication(argv=None): + qapplication = QtWidgets.QApplication.instance() + if qapplication is None: + qapplication = QtWidgets.QApplication(sys.argv if argv is None else argv) + return configure_qapplication(qapplication) + + class Splash(QtWidgets.QFrame): w = 250 h = 230 @@ -50,9 +76,7 @@ class Splash(QtWidgets.QFrame): FG = '#000000' def __init__(self, imagepath): - self.qapplication = QtWidgets.QApplication.instance() - if self.qapplication is None: - self.qapplication = QtWidgets.QApplication(sys.argv) + self.qapplication = get_qapplication() super().__init__() self.icon = QtGui.QPixmap() self.icon.load(imagepath) From 805d94e5b1a57e2804bae8e3e694119035bf4884 Mon Sep 17 00:00:00 2001 From: spielman Date: Wed, 25 Mar 2026 13:55:30 -0400 Subject: [PATCH 07/66] Fix TOML globals profile bootstrap --- labscript_profile/create.py | 8 +++++--- labscript_profile/default_profile/labconfig/example.ini | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/labscript_profile/create.py b/labscript_profile/create.py index 38a67734..c36d4e4c 100644 --- a/labscript_profile/create.py +++ b/labscript_profile/create.py @@ -4,6 +4,7 @@ import configparser from pathlib import Path from subprocess import check_output +import h5py from labscript_profile import LABSCRIPT_SUITE_PROFILE, default_labconfig_path import argparse @@ -84,8 +85,9 @@ def compile_connection_table(): output_h5_path = os.path.expandvars(config['paths']['connection_table_h5']) # create output directory, if needed Path(output_h5_path).parent.mkdir(parents=True, exist_ok=True) - # compile the h5 file - runmanager.new_globals_file(output_h5_path) + # Create a fresh HDF5 target for the compile output. + with h5py.File(output_h5_path, 'w'): + pass def dummy_callback(success): pass @@ -164,4 +166,4 @@ def create_profile(apparatus_name = None, compile_table = False): if compile_table: print('Compiling the example connection table') - compile_connection_table() \ No newline at end of file + compile_connection_table() diff --git a/labscript_profile/default_profile/labconfig/example.ini b/labscript_profile/default_profile/labconfig/example.ini index 94a6c16f..c959a192 100644 --- a/labscript_profile/default_profile/labconfig/example.ini +++ b/labscript_profile/default_profile/labconfig/example.ini @@ -43,7 +43,7 @@ save_git_info = False [BLACS/plugins] connection_table = True -connection_table.hashable_types = ['.py', '.txt', '.ini', '.json'] +connection_table.hashable_types = ['.py', '.txt', '.toml', '.ini', '.json'] connection_table.polling_interval = 1.0 delete_repeated_shots = False From dee0033cf8b3b09d774bac4f0caf31fbb6baaac5 Mon Sep 17 00:00:00 2001 From: spielman Date: Thu, 19 Mar 2026 13:36:41 -0400 Subject: [PATCH 08/66] Migrate shared config handling to TOML --- NEWS.md | 9 +- README.md | 6 + docs/source/labconfig.rst | 16 +- labscript_profile/__init__.py | 130 ++++++++-- labscript_profile/create.py | 57 +++-- .../default_profile/labconfig/example.ini | 66 ------ .../default_profile/labconfig/example.toml | 64 +++++ labscript_profile/toml_config.py | 80 +++++++ labscript_utils/filewatcher.py | 2 +- labscript_utils/labconfig.py | 222 +++++++++++++----- pyproject.toml | 2 + 11 files changed, 484 insertions(+), 170 deletions(-) delete mode 100644 labscript_profile/default_profile/labconfig/example.ini create mode 100644 labscript_profile/default_profile/labconfig/example.toml create mode 100644 labscript_profile/toml_config.py diff --git a/NEWS.md b/NEWS.md index 7b5fa25b..353814b5 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,10 @@ +## Unreleased + +- TOML-backed labconfig and app-config writing now depends on `tomli-w`. The package + dependency is declared in `labscript-utils`, but existing editable or conda + environments may need that dependency installed explicitly when upgrading. + + ## [2.15.0] - 2019-12-04 This release includes one bugfix, one enhancement, one update for compatibility with a @@ -60,4 +67,4 @@ compatibility with a newer version of a library. ([PR #91](https://bitbucket.org/labscript_suite/labscript_utils/pull-requests/91)) - Compatibility with `importlib_metadata` >= 0.21. Contributed by Chris Billington - ([PR #92](https://bitbucket.org/labscript_suite/labscript_utils/pull-requests/88)) \ No newline at end of file + ([PR #92](https://bitbucket.org/labscript_suite/labscript_utils/pull-requests/88)) diff --git a/README.md b/README.md index eb133843..f63c5b26 100644 --- a/README.md +++ b/README.md @@ -19,3 +19,9 @@ Shared modules used by the [*labscript suite*](https://github.com/labscript-suit ## Installation labscript-utils is distributed as a Python package on [PyPI](https://pypi.org/user/labscript-suite) and [Anaconda Cloud](https://anaconda.org/labscript-suite), and should be installed with other components of the _labscript suite_. Please see the [installation guide](https://docs.labscriptsuite.org/en/latest/installation) for details. + +TOML-backed configuration support depends on `tomli-w` at runtime. This dependency is +declared in `labscript-utils` package metadata, so fresh installs should receive it +automatically. Existing editable or conda environments created before this dependency +was added may need `conda install tomli-w` or an updated `labscript-utils` +installation. diff --git a/docs/source/labconfig.rst b/docs/source/labconfig.rst index fa727640..f342a17b 100644 --- a/docs/source/labconfig.rst +++ b/docs/source/labconfig.rst @@ -1,19 +1,19 @@ -The *labconfig.ini* file +The *labconfig.toml* file ======================== -The `labconfig.ini` file is a global configuration file for your **labscript-suite** installation. +The `labconfig.toml` file is a global configuration file for your **labscript-suite** installation. It contains configurable settings that govern how the individual components of the suite operate. The name of this file must be the host computer's system name. -So if my system's name was `heisenberg`, the labconfig file name would be `heisenberg.ini`. +So if my system's name was `heisenberg`, the labconfig file name would be `heisenberg.toml`. This file should be located in the `labscript-suite` directory in the user space, in the `labconfig` subdirectory. When :doc:`installing the **labscript-suite** for the first time `, running the `labscript-profile-create` command will automatically generate the `labscript-suite` user space directory in the correct place -and generate a `labconfig.ini` file for use on your system. -By editing the `ini` file named after your system, you can update the configuration settings of your **labscript-suite** installation. +and generate a `labconfig.toml` file for use on your system. +By editing the TOML file named after your system, you can update the configuration settings of your **labscript-suite** installation. -The Default *labconfig.ini* ---------------------------- +The Default *labconfig.toml* +---------------------------- Below is a copy of the default lab configuration if you were to install the **labscript-suite** today. @@ -23,5 +23,5 @@ Below is a copy of the default lab configuration if you were to install the **la Instead, if keys are missing from your local profile, default behavior will be assumed. To implement the added functionality, you will need to manually add/change the keys in your local labconfig. -.. include:: ../../labscript_profile/default_profile/labconfig/example.ini +.. include:: ../../labscript_profile/default_profile/labconfig/example.toml :code: diff --git a/labscript_profile/__init__.py b/labscript_profile/__init__.py index 55571f78..a45a5744 100644 --- a/labscript_profile/__init__.py +++ b/labscript_profile/__init__.py @@ -1,12 +1,14 @@ import site import sys import os -from configparser import ConfigParser, NoSectionError, NoOptionError +import configparser +from ast import literal_eval from pathlib import Path from subprocess import check_output import socket from getpass import getuser +from .toml_config import TomlConfigParser, dump_toml_file, load_toml_file # The contents of this file are imported every time the Python interpreter starts up, # owing to our custom .pth file that runs the below two functions. This ensures that @@ -36,6 +38,13 @@ def hostname(): def default_labconfig_path(): + if LABSCRIPT_SUITE_PROFILE is None: + return None + return LABSCRIPT_SUITE_PROFILE / 'labconfig' / f'{hostname()}.toml' + + +def legacy_labconfig_path(): + """Temporary legacy INI labconfig location. Slated for removal soon.""" if LABSCRIPT_SUITE_PROFILE is None: return None return LABSCRIPT_SUITE_PROFILE / 'labconfig' / f'{hostname()}.ini' @@ -47,17 +56,110 @@ def add_userlib_and_pythonlib(): re-implements finding and reading the config file so as to not import labscript_utils, since we dont' want to import something like labscript_utils every time the interpreter starts up""" - labconfig = default_labconfig_path() - if labconfig is not None and labconfig.exists(): - # str() below is for py36 compat, where ConfigParser can't deal with Path objs - config = ConfigParser( - defaults={'labscript_suite': str(LABSCRIPT_SUITE_PROFILE)} - ) - config.read(labconfig) - for option in ['userlib', 'pythonlib']: - try: - paths = config.get('DEFAULT', option).split(',') - except (NoSectionError, NoOptionError): - paths = [] - for path in paths: + labconfig = ensure_labconfig() + if labconfig is None: + return + if not labconfig.exists(): + return + config = TomlConfigParser(defaults={'labscript_suite': str(LABSCRIPT_SUITE_PROFILE)}) + config.read_toml(labconfig) + for option in ['userlib', 'pythonlib']: + try: + paths = config.get('DEFAULT', option) + except (configparser.NoSectionError, configparser.NoOptionError): + paths = '' + if paths: + for path in paths.split(','): site.addsitedir(path) + + +def ensure_labconfig(): + """Return the TOML labconfig path, converting a legacy INI file if needed. + + The legacy conversion path is temporary and slated for removal soon. + """ + labconfig = default_labconfig_path() + if labconfig is None: + return None + if labconfig.exists(): + return labconfig + legacy = legacy_labconfig_path() + if legacy is None or not legacy.exists(): + return labconfig + _convert_legacy_labconfig(legacy, labconfig) + return labconfig + + +def _example_labconfig_data(): + if LABSCRIPT_SUITE_PROFILE is not None: + profile_example = LABSCRIPT_SUITE_PROFILE / 'labconfig' / 'example.toml' + if profile_example.exists(): + return load_toml_file(profile_example) + bundled_example = Path(__file__).resolve().parent / 'default_profile' / 'labconfig' / 'example.toml' + if bundled_example.exists(): + return load_toml_file(bundled_example) + return {} + + +def _coerce_legacy_value(value, template_value): + if template_value is None: + return value + if isinstance(template_value, bool): + if isinstance(value, bool): + return value + if isinstance(value, str): + lowered = value.strip().lower() + if lowered in configparser.ConfigParser.BOOLEAN_STATES: + return configparser.ConfigParser.BOOLEAN_STATES[lowered] + return value + if isinstance(template_value, int): + try: + return int(value) + except (TypeError, ValueError): + return value + if isinstance(template_value, float): + try: + return float(value) + except (TypeError, ValueError): + return value + if isinstance(template_value, list): + if not isinstance(value, str): + return value + try: + parsed = literal_eval(value) + except (SyntaxError, ValueError): + return value + if not isinstance(parsed, (list, tuple)): + return value + if not template_value: + return list(parsed) + template_item = template_value[0] + return [_coerce_legacy_value(item, template_item) for item in parsed] + return value + + +def _convert_legacy_labconfig(legacy_path, toml_path): + """Temporary legacy migration path from INI to TOML. Slated for removal soon.""" + config = configparser.ConfigParser(interpolation=None) + config.read(legacy_path) + template_data = _example_labconfig_data() + default_template = template_data.get('DEFAULT', {}) + data = { + 'DEFAULT': { + key: _coerce_legacy_value(value, default_template.get(key)) + for key, value in config.defaults().items() + } + } + for section in config.sections(): + section_items = dict(config._sections[section]) + section_items.pop('__name__', None) + autoload = section_items.get('autoload_config_file') + if isinstance(autoload, str) and autoload.lower().endswith('.ini'): + section_items['autoload_config_file'] = autoload[:-4] + '.toml' + section_template = template_data.get(section, {}) + data[section] = { + key: _coerce_legacy_value(value, section_template.get(key)) + for key, value in section_items.items() + } + toml_path.parent.mkdir(parents=True, exist_ok=True) + dump_toml_file(toml_path, data) diff --git a/labscript_profile/create.py b/labscript_profile/create.py index c36d4e4c..2ca7e46c 100644 --- a/labscript_profile/create.py +++ b/labscript_profile/create.py @@ -1,17 +1,31 @@ import sys import os import shutil -import configparser from pathlib import Path from subprocess import check_output import h5py -from labscript_profile import LABSCRIPT_SUITE_PROFILE, default_labconfig_path +from labscript_profile import ( + LABSCRIPT_SUITE_PROFILE, + default_labconfig_path, + legacy_labconfig_path, +) import argparse +from labscript_profile.toml_config import dump_toml_file, load_toml_file _here = os.path.dirname(os.path.abspath(__file__)) DEFAULT_PROFILE_CONTENTS = os.path.join(_here, 'default_profile') +def _replace_backslashes(value): + if isinstance(value, dict): + return {key: _replace_backslashes(child) for key, child in value.items()} + if isinstance(value, list): + return [_replace_backslashes(child) for child in value] + if isinstance(value, str): + return value.replace('\\', os.path.sep) + return value + + def make_shared_secret(directory): """Create a new zprocess shared secret file in the given directory and return its filepath""" @@ -32,37 +46,33 @@ def make_labconfig_file(apparatus_name = None): Overrides the default apparatus name with the provided one if not None """ - source_path = os.path.join(LABSCRIPT_SUITE_PROFILE, 'labconfig', 'example.ini') + source_path = os.path.join(LABSCRIPT_SUITE_PROFILE, 'labconfig', 'example.toml') target_path = default_labconfig_path() - if os.path.exists(target_path): + legacy_path = legacy_labconfig_path() + if os.path.exists(target_path) or (legacy_path is not None and os.path.exists(legacy_path)): raise FileExistsError(target_path) - with open(source_path) as infile, open(target_path, 'w') as outfile: - data = infile.read() - data = data.replace('\\', os.path.sep) - outfile.write(data) - - # Now change some things about it: - config = configparser.ConfigParser(interpolation=None) - config.read(target_path) + config = load_toml_file(source_path) + if os.path.sep != '\\': + config = _replace_backslashes(config) if sys.platform == 'linux': - config.set('programs', 'text_editor', 'gedit') + config['programs']['text_editor'] = 'gedit' elif sys.platform == 'darwin': - config.set('programs', 'text_editor', 'open') - config.set('programs', 'text_editor_arguments', '-a TextEdit {file}') + config['programs']['text_editor'] = 'open' + config['programs']['text_editor_arguments'] = '-a TextEdit {file}' if sys.platform != 'win32': - config.set('programs', 'hdf5_viewer', 'hdfview') - config.set('DEFAULT', 'shared_drive', '$HOME/labscript_shared') + config['programs']['hdf5_viewer'] = 'hdfview' + config['DEFAULT']['shared_drive'] = '$HOME/labscript_shared' shared_secret = make_shared_secret(target_path.parent) shared_secret_entry = Path( '%(labscript_suite)s', shared_secret.relative_to(LABSCRIPT_SUITE_PROFILE) ) - config.set('security', 'shared_secret', str(shared_secret_entry)) + config['security']['shared_secret'] = str(shared_secret_entry) if apparatus_name is not None: print(f'\tSetting apparatus name to \'{apparatus_name}\'') - config.set('DEFAULT', 'apparatus_name', apparatus_name) + config['DEFAULT']['apparatus_name'] = apparatus_name - with open(target_path, 'w') as f: - config.write(f) + target_path.parent.mkdir(parents=True, exist_ok=True) + dump_toml_file(target_path, config) def compile_connection_table(): """Compile the connection table defined in the labconfig file @@ -76,8 +86,9 @@ def compile_connection_table(): # if runmanager doesn't import, skip compilation return - config = configparser.ConfigParser(defaults = {'labscript_suite': str(LABSCRIPT_SUITE_PROFILE)}) - config.read(default_labconfig_path()) + from labscript_utils.labconfig import LabConfig + + config = LabConfig() # The path to the user's connection_table.py script script_path = os.path.expandvars(config['paths']['connection_table_py']) diff --git a/labscript_profile/default_profile/labconfig/example.ini b/labscript_profile/default_profile/labconfig/example.ini deleted file mode 100644 index c959a192..00000000 --- a/labscript_profile/default_profile/labconfig/example.ini +++ /dev/null @@ -1,66 +0,0 @@ -[DEFAULT] -apparatus_name = example_apparatus -shared_drive = C: -experiment_shot_storage = %(shared_drive)s\Experiments\%(apparatus_name)s -userlib= %(labscript_suite)s\userlib -pythonlib = %(userlib)s\pythonlib -labscriptlib = %(userlib)s\labscriptlib\%(apparatus_name)s -analysislib = %(userlib)s\analysislib\%(apparatus_name)s -app_saved_configs = %(labscript_suite)s\app_saved_configs\%(apparatus_name)s -user_devices = user_devices - -[paths] -connection_table_h5 = %(experiment_shot_storage)s\connection_table.h5 -connection_table_py = %(labscriptlib)s\connection_table.py - -[servers] -zlock = localhost -runmanager = localhost -blacs = localhost -lyse = localhost - -[ports] -BLACS = 42517 -lyse = 42519 -runviewer = 42521 -runmanager = 42523 -zlock = 7339 -zlog = 7340 -zprocess_remote = 7341 - -[timeouts] -communication_timeout = 60 - -[programs] -text_editor = %%PROGRAMFILES%%\Sublime Text 3\sublime_text.exe -text_editor_arguments = {file} -hdf5_viewer = %%LOCALAPPDATA%%\HDF_Group\HDFView\3.1.0\hdfview.bat -hdf5_viewer_arguments = {file} - -[labscript] -save_hg_info = False -save_git_info = False - -[BLACS/plugins] -connection_table = True -connection_table.hashable_types = ['.py', '.txt', '.toml', '.ini', '.json'] -connection_table.polling_interval = 1.0 - -delete_repeated_shots = False -general = True -memory = False -progress_bar = False -theme = True - - -[lyse] -autoload_config_file = %(app_saved_configs)s\lyse\lyse.ini -integer_indexing = False - -[runmanager] -autoload_config_file = %(app_saved_configs)s\runmanager\runmanager.ini -output_folder_format = %%Y\%%m\%%d\{sequence_index:04d} -filename_prefix_format = %%Y-%%m-%%d_{sequence_index:04d}_{script_basename} - -[security] -shared_secret = %(labscript_suite)s\labconfig\zpsecret-b810f83f.key diff --git a/labscript_profile/default_profile/labconfig/example.toml b/labscript_profile/default_profile/labconfig/example.toml new file mode 100644 index 00000000..8e368358 --- /dev/null +++ b/labscript_profile/default_profile/labconfig/example.toml @@ -0,0 +1,64 @@ +[DEFAULT] +apparatus_name = 'example_apparatus' +shared_drive = 'C:' +experiment_shot_storage = '%(shared_drive)s\Experiments\%(apparatus_name)s' +userlib = '%(labscript_suite)s\userlib' +pythonlib = '%(userlib)s\pythonlib' +labscriptlib = '%(userlib)s\labscriptlib\%(apparatus_name)s' +analysislib = '%(userlib)s\analysislib\%(apparatus_name)s' +app_saved_configs = '%(labscript_suite)s\app_saved_configs\%(apparatus_name)s' +user_devices = 'user_devices' + +[paths] +connection_table_h5 = '%(experiment_shot_storage)s\connection_table.h5' +connection_table_py = '%(labscriptlib)s\connection_table.py' + +[servers] +zlock = 'localhost' +runmanager = 'localhost' +blacs = 'localhost' +lyse = 'localhost' + +[ports] +BLACS = 42517 +lyse = 42519 +runviewer = 42521 +runmanager = 42523 +zlock = 7339 +zlog = 7340 +zprocess_remote = 7341 + +[timeouts] +communication_timeout = 60 + +[programs] +text_editor = '%%PROGRAMFILES%%\Sublime Text 3\sublime_text.exe' +text_editor_arguments = '{file}' +hdf5_viewer = '%%LOCALAPPDATA%%\HDF_Group\HDFView\3.1.0\hdfview.bat' +hdf5_viewer_arguments = '{file}' + +[labscript] +save_hg_info = false +save_git_info = false + +["BLACS/plugins"] +connection_table = true +"connection_table.hashable_types" = ['.py', '.txt', '.toml', '.json'] +"connection_table.polling_interval" = 1.0 +delete_repeated_shots = false +general = true +memory = false +progress_bar = false +theme = true + +[lyse] +autoload_config_file = '%(app_saved_configs)s\lyse\lyse.toml' +integer_indexing = false + +[runmanager] +autoload_config_file = '%(app_saved_configs)s\runmanager\runmanager.toml' +output_folder_format = '%%Y\%%m\%%d\{sequence_index:04d}' +filename_prefix_format = '%%Y-%%m-%%d_{sequence_index:04d}_{script_basename}' + +[security] +shared_secret = '%(labscript_suite)s\labconfig\zpsecret-b810f83f.key' diff --git a/labscript_profile/toml_config.py b/labscript_profile/toml_config.py new file mode 100644 index 00000000..67eb7f1e --- /dev/null +++ b/labscript_profile/toml_config.py @@ -0,0 +1,80 @@ +import configparser + +try: + import tomllib +except ImportError: + import tomli as tomllib + +import tomli_w + + +def load_toml_file(filename): + with open(filename, 'rb') as f: + return tomllib.load(f) + + +def dump_toml_file(filename, data): + with open(filename, 'wb') as f: + tomli_w.dump(data, f) + + +class TomlBasicInterpolation(configparser.BasicInterpolation): + """Basic interpolation that leaves non-string TOML values alone.""" + + def before_get(self, parser, section, option, value, defaults): + if not isinstance(value, str): + return value + return super().before_get(parser, section, option, value, defaults) + + +class TomlConfigParser(configparser.ConfigParser): + """ConfigParser variant that preserves native TOML values.""" + + def __init__(self, *args, **kwargs): + kwargs.setdefault('interpolation', TomlBasicInterpolation()) + super().__init__(*args, **kwargs) + + def set(self, section, option, value=None): + if not section or section == self.default_section: + sectdict = self._defaults + else: + try: + sectdict = self._sections[section] + except KeyError: + raise configparser.NoSectionError(section) from None + sectdict[self.optionxform(option)] = value + + def getboolean( + self, section, option, *, raw=False, vars=None, fallback=configparser._UNSET + ): + value = self.get(section, option, raw=raw, vars=vars, fallback=fallback) + if value is fallback: + return value + if isinstance(value, bool): + return value + return self._convert_to_boolean(value) + + def read_toml(self, filename): + data = load_toml_file(filename) + if not isinstance(data, dict): + raise TypeError("Top-level TOML document must be a mapping of sections") + for section_name, section in data.items(): + if not isinstance(section, dict): + raise TypeError(f"Section {section_name!r} must contain key/value pairs") + section_name = str(section_name) + if section_name != self.default_section and not self.has_section(section_name): + self.add_section(section_name) + for option, value in section.items(): + self.set(section_name, str(option), value) + return [str(filename)] + + def to_toml_dict(self): + data = {} + if self.defaults(): + data[self.default_section] = dict(self.defaults()) + for section in self.sections(): + data[section] = dict(self._sections[section]) + return data + + def write_toml(self, filename): + dump_toml_file(filename, self.to_toml_dict()) diff --git a/labscript_utils/filewatcher.py b/labscript_utils/filewatcher.py index 4b6f84b2..edc081a8 100644 --- a/labscript_utils/filewatcher.py +++ b/labscript_utils/filewatcher.py @@ -268,7 +268,7 @@ def callback(name, modified, event=None): callback, files=test_files[0], folders=test_folder, - hashable_types=['.py', '.ini', '.txt'], + hashable_types=['.py', '.toml', '.txt'], interval=2, ) diff --git a/labscript_utils/labconfig.py b/labscript_utils/labconfig.py index d110fe82..be4e60e9 100644 --- a/labscript_utils/labconfig.py +++ b/labscript_utils/labconfig.py @@ -10,31 +10,45 @@ # for the full license. # # # ##################################################################### -import os import configparser +import os +import warnings from ast import literal_eval -from pprint import pformat from pathlib import Path -import warnings from labscript_utils import dedent -from labscript_profile import default_labconfig_path, LABSCRIPT_SUITE_PROFILE +from labscript_profile import ( + LABSCRIPT_SUITE_PROFILE, + default_labconfig_path, + ensure_labconfig, +) +from labscript_profile.toml_config import ( + TomlBasicInterpolation, + TomlConfigParser, + dump_toml_file, + load_toml_file, +) + default_config_path = default_labconfig_path() -class EnvInterpolation(configparser.BasicInterpolation): - """Interpolation which expands environment variables in values, - by post-filtering BasicInterpolation.before_get()""" +class EnvInterpolation(TomlBasicInterpolation): + """Interpolation that expands environment variables after TOML/basic interpolation.""" def before_get(self, *args): value = super(EnvInterpolation, self).before_get(*args) - return os.path.expandvars(value) + if isinstance(value, str): + return os.path.expandvars(value) + return value + +class LabConfig(TomlConfigParser): + """TOML labconfig reader with the small ConfigParser-like API used by the suite.""" -class LabConfig(configparser.ConfigParser): NoOptionError = configparser.NoOptionError NoSectionError = configparser.NoSectionError + DuplicateSectionError = configparser.DuplicateSectionError def __init__( self, config_path=default_config_path, required_params=None, defaults=None, @@ -43,12 +57,20 @@ def __init__( required_params = {} if defaults is None: defaults = {} - # str() below is for py36 compat, where ConfigParser can't deal with Path objs + else: + defaults = dict(defaults) defaults['labscript_suite'] = str(LABSCRIPT_SUITE_PROFILE) if isinstance(config_path, list): - self.config_path = config_path[0] + config_paths = list(config_path) + if config_paths: + if config_paths[0] == default_config_path: + config_paths[0] = ensure_labconfig() + self.config_path = config_paths[0] + else: + self.config_path = None else: - self.config_path = config_path + self.config_path = ensure_labconfig() if config_path == default_config_path else config_path + config_paths = [self.config_path] self.file_format = "" for section, options in required_params.items(): @@ -56,15 +78,12 @@ def __init__( for option in options: self.file_format += "%s = \n" % option - # Load the config file - configparser.ConfigParser.__init__( + TomlConfigParser.__init__( self, defaults=defaults, interpolation=EnvInterpolation() ) - # read all files in the config path if it is a list (self.config_path only - # contains one string): - self.read(config_path) + for path in config_paths: + self._read_path(path) - # Rename experiment_name to apparatus_name and raise a DeprectionWarning experiment_name = self.get("DEFAULT", "experiment_name", fallback=None) if experiment_name: msg = """The experiment_name keyword has been renamed apparatus_name in @@ -76,57 +95,146 @@ def __init__( your labconfig. Please omit the deprecate experiment_name keyword.""" raise Exception(dedent(msg)) - else: - self.set("DEFAULT", "apparatus_name", experiment_name) + self.set("DEFAULT", "apparatus_name", experiment_name) try: for section, options in required_params.items(): for option in options: self.get(section, option) - except configparser.NoOptionError: + except (configparser.NoOptionError, configparser.NoSectionError): msg = f"""The experiment configuration file located at {config_path} does not have the required keys. Make sure the config file contains the following structure:\n{self.file_format}""" raise Exception(dedent(msg)) + def _read_path(self, path): + """Read a TOML labconfig, or temporarily fall back to legacy INI.""" + if path is None: + return + path = Path(path) + if not path.exists(): + return + if path.suffix.lower() == '.toml': + self.read_toml(path) + return + if path.suffix.lower() == '.ini': + msg = """INI labconfig support is deprecated and slated for removal soon. + Convert this file to TOML.""" + warnings.warn(dedent(msg), FutureWarning) + self.read(path) + return + msg = f"Unsupported labconfig format for {path}. Expected a .toml or .ini file." + raise RuntimeError(msg) + + +def _resolve_appconfig_load_path(filename): + path = Path(filename) + suffix = path.suffix.lower() + if suffix == '.ini': + if path.exists(): + return path + toml_path = path.with_suffix('.toml') + if toml_path.exists(): + return toml_path + return path + if suffix == '.toml': + if path.exists(): + return path + legacy_path = path.with_suffix('.ini') + if legacy_path.exists(): + return legacy_path + return path + toml_path = path.with_suffix('.toml') + if toml_path.exists(): + return toml_path + legacy_path = path.with_suffix('.ini') + if legacy_path.exists(): + return legacy_path + return toml_path + + +def _resolve_appconfig_save_path(filename): + return Path(filename).with_suffix('.toml') + + +def _to_toml_compatible(value, location='value'): + if isinstance(value, dict): + converted = {} + for key, child in value.items(): + if not isinstance(key, str): + msg = f"{location} contains a non-string dict key {key!r}" + raise TypeError(msg) + converted[key] = _to_toml_compatible(child, f"{location}.{key}") + return converted + if isinstance(value, (list, tuple)): + return [ + _to_toml_compatible(child, f"{location}[{index}]") + for index, child in enumerate(value) + ] + if isinstance(value, (str, bool, int, float)): + return value + msg = f"{location} value {value!r} is not representable in TOML app config" + raise TypeError(msg) + + +def _load_legacy_appconfig_value(value): + """Temporary legacy INI app-config decoder. Slated for removal soon.""" + if not isinstance(value, str): + return value + try: + return literal_eval(value) + except (ValueError, SyntaxError): + return value + def save_appconfig(filename, data): - """Save a dictionary as an ini file. The keys of the dictionary comprise the section - names, and the values must themselves be dictionaries for the names and values - within each section. All section values will be converted to strings with - pprint.pformat().""" - # Error checking - for section_name, section in data.items(): - for name, value in section.items(): - try: - valid = value == literal_eval(pformat(value)) - except (ValueError, SyntaxError): - valid = False - if not valid: - msg = f"{section_name}/{name} value {value} not a Python built-in type" - raise TypeError(msg) + """Save a dictionary as a TOML app config.""" data = { - section_name: {name: pformat(value) for name, value in section.items()} + section_name: { + name: _to_toml_compatible(value, f"{section_name}/{name}") + for name, value in section.items() + } for section_name, section in data.items() } - c = configparser.ConfigParser(interpolation=None) - c.optionxform = str # preserve case - c.read_dict(data) - Path(filename).parent.mkdir(parents=True, exist_ok=True) - with open(filename, 'w') as f: - c.write(f) - - -def load_appconfig(filename): - """Load an .ini file and return a dictionary of its contents. All values will be - converted to Python objects with ast.literal_eval(). All keys will be lowercase - regardless of the written contents on the .ini file.""" - c = configparser.ConfigParser(interpolation=None) - c.optionxform = str # preserve case - # No file? No config - don't crash. - if Path(filename).exists(): - c.read(filename) - return { - section_name: {name: literal_eval(value) for name, value in section.items()} - for section_name, section in c.items() - } + filename = _resolve_appconfig_save_path(filename) + filename.parent.mkdir(parents=True, exist_ok=True) + dump_toml_file(filename, data) + return str(filename) + + +def load_appconfig(filename, return_save_path=False): + """Load a TOML app config, or a legacy INI app config if required. + + If a legacy INI file is loaded, a sibling TOML file is immediately written and can be + returned as the canonical save path via ``return_save_path=True``. Legacy INI support + here is temporary and slated for removal soon. + """ + requested_path = Path(filename) + filename = _resolve_appconfig_load_path(filename) + save_path = _resolve_appconfig_save_path(requested_path) + if filename.suffix.lower() == '.ini': + c = configparser.ConfigParser(interpolation=None) + c.optionxform = str + if filename.exists(): + c.read(filename) + data = { + section_name: { + name: _load_legacy_appconfig_value(value) + for name, value in c.items(section_name, raw=True) + } + for section_name in c.sections() + } + if filename.exists(): + save_path = Path(save_appconfig(save_path, data)) + else: + raw = load_toml_file(filename) if filename.exists() else {} + data = { + section_name: dict(section.items()) + for section_name, section in raw.items() + if section_name != 'DEFAULT' and isinstance(section, dict) + } + if filename.suffix.lower() == '.toml': + save_path = filename + if return_save_path: + return data, str(save_path) + return data diff --git a/pyproject.toml b/pyproject.toml index 53ec1e59..fed85e39 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,8 @@ dependencies = [ "scipy", "zprocess>=2.18.0", "setuptools_scm>=4.1.0", + "tomli>=1.1.0; python_version < '3.11'", + "tomli-w>=1.2.0", ] dynamic = ["version"] From 806fab58ee817d7b43dfe9c7550fa564c522409b Mon Sep 17 00:00:00 2001 From: spielman Date: Thu, 19 Mar 2026 13:58:46 -0400 Subject: [PATCH 09/66] Normalize labconfig TOML section names --- labscript_profile/__init__.py | 9 ++- labscript_profile/create.py | 4 +- .../default_profile/labconfig/example.toml | 6 +- labscript_profile/toml_config.py | 75 ++++++++++++++++++- labscript_utils/labconfig.py | 4 +- 5 files changed, 87 insertions(+), 11 deletions(-) diff --git a/labscript_profile/__init__.py b/labscript_profile/__init__.py index a45a5744..07def1ec 100644 --- a/labscript_profile/__init__.py +++ b/labscript_profile/__init__.py @@ -143,21 +143,22 @@ def _convert_legacy_labconfig(legacy_path, toml_path): config = configparser.ConfigParser(interpolation=None) config.read(legacy_path) template_data = _example_labconfig_data() - default_template = template_data.get('DEFAULT', {}) + default_template = template_data.get('default', {}) data = { - 'DEFAULT': { + 'default': { key: _coerce_legacy_value(value, default_template.get(key)) for key, value in config.defaults().items() } } for section in config.sections(): + canonical_section = section.lower() section_items = dict(config._sections[section]) section_items.pop('__name__', None) autoload = section_items.get('autoload_config_file') if isinstance(autoload, str) and autoload.lower().endswith('.ini'): section_items['autoload_config_file'] = autoload[:-4] + '.toml' - section_template = template_data.get(section, {}) - data[section] = { + section_template = template_data.get(canonical_section, {}) + data[canonical_section] = { key: _coerce_legacy_value(value, section_template.get(key)) for key, value in section_items.items() } diff --git a/labscript_profile/create.py b/labscript_profile/create.py index 2ca7e46c..b1b3bff1 100644 --- a/labscript_profile/create.py +++ b/labscript_profile/create.py @@ -61,7 +61,7 @@ def make_labconfig_file(apparatus_name = None): config['programs']['text_editor_arguments'] = '-a TextEdit {file}' if sys.platform != 'win32': config['programs']['hdf5_viewer'] = 'hdfview' - config['DEFAULT']['shared_drive'] = '$HOME/labscript_shared' + config['default']['shared_drive'] = '$HOME/labscript_shared' shared_secret = make_shared_secret(target_path.parent) shared_secret_entry = Path( '%(labscript_suite)s', shared_secret.relative_to(LABSCRIPT_SUITE_PROFILE) @@ -69,7 +69,7 @@ def make_labconfig_file(apparatus_name = None): config['security']['shared_secret'] = str(shared_secret_entry) if apparatus_name is not None: print(f'\tSetting apparatus name to \'{apparatus_name}\'') - config['DEFAULT']['apparatus_name'] = apparatus_name + config['default']['apparatus_name'] = apparatus_name target_path.parent.mkdir(parents=True, exist_ok=True) dump_toml_file(target_path, config) diff --git a/labscript_profile/default_profile/labconfig/example.toml b/labscript_profile/default_profile/labconfig/example.toml index 8e368358..2c5c47b8 100644 --- a/labscript_profile/default_profile/labconfig/example.toml +++ b/labscript_profile/default_profile/labconfig/example.toml @@ -1,4 +1,4 @@ -[DEFAULT] +[default] apparatus_name = 'example_apparatus' shared_drive = 'C:' experiment_shot_storage = '%(shared_drive)s\Experiments\%(apparatus_name)s' @@ -20,7 +20,7 @@ blacs = 'localhost' lyse = 'localhost' [ports] -BLACS = 42517 +blacs = 42517 lyse = 42519 runviewer = 42521 runmanager = 42523 @@ -41,7 +41,7 @@ hdf5_viewer_arguments = '{file}' save_hg_info = false save_git_info = false -["BLACS/plugins"] +["blacs/plugins"] connection_table = true "connection_table.hashable_types" = ['.py', '.txt', '.toml', '.json'] "connection_table.polling_interval" = 1.0 diff --git a/labscript_profile/toml_config.py b/labscript_profile/toml_config.py index 67eb7f1e..e5c6d756 100644 --- a/labscript_profile/toml_config.py +++ b/labscript_profile/toml_config.py @@ -31,10 +31,64 @@ class TomlConfigParser(configparser.ConfigParser): """ConfigParser variant that preserves native TOML values.""" def __init__(self, *args, **kwargs): + self.lowercase_keys_and_sections = kwargs.pop('lowercase_keys_and_sections', True) + if self.lowercase_keys_and_sections: + kwargs.setdefault('default_section', 'default') kwargs.setdefault('interpolation', TomlBasicInterpolation()) super().__init__(*args, **kwargs) + if not self.lowercase_keys_and_sections: + self.optionxform = str + + def _canonical_section_name(self, section): + if section is None: + return None + if not self.lowercase_keys_and_sections: + return section + return str(section).lower() + + def _normalize_loaded_sections(self): + if not self.lowercase_keys_and_sections: + return + renamed_sections = {} + migrated_defaults = {} + for name, section_data in self._sections.items(): + canonical = self._canonical_section_name(name) + if canonical == self.default_section: + migrated_defaults.update(section_data) + continue + existing = renamed_sections.get(canonical) + if existing is not None and existing is not section_data: + raise configparser.DuplicateSectionError(canonical) + renamed_sections[canonical] = section_data + self._defaults.update(migrated_defaults) + self._sections = renamed_sections + self._proxies = {self.default_section: configparser.SectionProxy(self, self.default_section)} + for canonical in renamed_sections: + self._proxies[canonical] = configparser.SectionProxy(self, canonical) + + def add_section(self, section): + return super().add_section(self._canonical_section_name(section)) + + def has_section(self, section): + return super().has_section(self._canonical_section_name(section)) + + def options(self, section): + return super().options(self._canonical_section_name(section)) + + def has_option(self, section, option): + return super().has_option(self._canonical_section_name(section), option) + + def remove_section(self, section): + return super().remove_section(self._canonical_section_name(section)) + + def remove_option(self, section, option): + return super().remove_option(self._canonical_section_name(section), option) + + def __getitem__(self, key): + return super().__getitem__(self._canonical_section_name(key)) def set(self, section, option, value=None): + section = self._canonical_section_name(section) if not section or section == self.default_section: sectdict = self._defaults else: @@ -54,6 +108,25 @@ def getboolean( return value return self._convert_to_boolean(value) + def get(self, section, option, *, raw=False, vars=None, fallback=configparser._UNSET): + return super().get( + self._canonical_section_name(section), + option, + raw=raw, + vars=vars, + fallback=fallback, + ) + + def items(self, section=configparser._UNSET, raw=False, vars=None): + if section is configparser._UNSET: + return super().items(section=section, raw=raw, vars=vars) + return super().items(self._canonical_section_name(section), raw=raw, vars=vars) + + def read(self, filenames, encoding=None): + read_ok = super().read(filenames, encoding=encoding) + self._normalize_loaded_sections() + return read_ok + def read_toml(self, filename): data = load_toml_file(filename) if not isinstance(data, dict): @@ -61,7 +134,7 @@ def read_toml(self, filename): for section_name, section in data.items(): if not isinstance(section, dict): raise TypeError(f"Section {section_name!r} must contain key/value pairs") - section_name = str(section_name) + section_name = self._canonical_section_name(section_name) if section_name != self.default_section and not self.has_section(section_name): self.add_section(section_name) for option, value in section.items(): diff --git a/labscript_utils/labconfig.py b/labscript_utils/labconfig.py index be4e60e9..6587eb11 100644 --- a/labscript_utils/labconfig.py +++ b/labscript_utils/labconfig.py @@ -79,7 +79,9 @@ def __init__( self.file_format += "%s = \n" % option TomlConfigParser.__init__( - self, defaults=defaults, interpolation=EnvInterpolation() + self, + defaults=defaults, + interpolation=EnvInterpolation(), ) for path in config_paths: self._read_path(path) From c24a4fed85986c66dc858d4665f3793485dd6c1a Mon Sep 17 00:00:00 2001 From: spielman Date: Thu, 19 Mar 2026 15:19:22 -0400 Subject: [PATCH 10/66] Updated example.toml. --- labscript_profile/default_profile/labconfig/example.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/labscript_profile/default_profile/labconfig/example.toml b/labscript_profile/default_profile/labconfig/example.toml index 2c5c47b8..f655e501 100644 --- a/labscript_profile/default_profile/labconfig/example.toml +++ b/labscript_profile/default_profile/labconfig/example.toml @@ -43,7 +43,7 @@ save_git_info = false ["blacs/plugins"] connection_table = true -"connection_table.hashable_types" = ['.py', '.txt', '.toml', '.json'] +"connection_table.hashable_types" = ['.py', '.txt', '.ini', '.toml', '.json'] "connection_table.polling_interval" = 1.0 delete_repeated_shots = false general = true From 7a623bab4857b078db2b415683df9aea8c9d4a92 Mon Sep 17 00:00:00 2001 From: spielman Date: Fri, 20 Mar 2026 10:23:49 -0400 Subject: [PATCH 11/66] Refine TOML labconfig normalization --- labscript_profile/__init__.py | 2 +- labscript_utils/device_registry/_device_registry.py | 2 +- labscript_utils/labconfig.py | 9 +++++---- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/labscript_profile/__init__.py b/labscript_profile/__init__.py index 07def1ec..1d05b9a4 100644 --- a/labscript_profile/__init__.py +++ b/labscript_profile/__init__.py @@ -65,7 +65,7 @@ def add_userlib_and_pythonlib(): config.read_toml(labconfig) for option in ['userlib', 'pythonlib']: try: - paths = config.get('DEFAULT', option) + paths = config.get('default', option) except (configparser.NoSectionError, configparser.NoOptionError): paths = '' if paths: diff --git a/labscript_utils/device_registry/_device_registry.py b/labscript_utils/device_registry/_device_registry.py index e5a122a8..0e2348a0 100644 --- a/labscript_utils/device_registry/_device_registry.py +++ b/labscript_utils/device_registry/_device_registry.py @@ -74,7 +74,7 @@ def _get_device_dirs(): """Return the directory of labscript_devices, and the folders containing submodules of any packages listed in the user_devices labconfig setting""" try: - user_devices = LabConfig().get('DEFAULT', 'user_devices') + user_devices = LabConfig().get('default', 'user_devices') except (LabConfig.NoOptionError, LabConfig.NoSectionError): user_devices = 'user_devices' # Split on commas, remove whitespace: diff --git a/labscript_utils/labconfig.py b/labscript_utils/labconfig.py index 6587eb11..86bb7101 100644 --- a/labscript_utils/labconfig.py +++ b/labscript_utils/labconfig.py @@ -86,21 +86,22 @@ def __init__( for path in config_paths: self._read_path(path) - experiment_name = self.get("DEFAULT", "experiment_name", fallback=None) + experiment_name = self.get("default", "experiment_name", fallback=None) if experiment_name: msg = """The experiment_name keyword has been renamed apparatus_name in labscript_utils 3.0, and will be removed in a future version. Please update your labconfig to use the apparatus_name keyword.""" warnings.warn(dedent(msg), FutureWarning) - if self.get("DEFAULT", "apparatus_name", fallback=None): + if self.get("default", "apparatus_name", fallback=None): msg = """You have defined both experiment_name and apparatus_name in your labconfig. Please omit the deprecate experiment_name keyword.""" raise Exception(dedent(msg)) - self.set("DEFAULT", "apparatus_name", experiment_name) + self.set("default", "apparatus_name", experiment_name) try: for section, options in required_params.items(): + section = self._canonical_section_name(section) for option in options: self.get(section, option) except (configparser.NoOptionError, configparser.NoSectionError): @@ -233,7 +234,7 @@ def load_appconfig(filename, return_save_path=False): data = { section_name: dict(section.items()) for section_name, section in raw.items() - if section_name != 'DEFAULT' and isinstance(section, dict) + if section_name != 'default' and isinstance(section, dict) } if filename.suffix.lower() == '.toml': save_path = filename From 11e77a8a5957d000c0a7caea37f3e525b4d586b5 Mon Sep 17 00:00:00 2001 From: spielman Date: Fri, 20 Mar 2026 10:50:34 -0400 Subject: [PATCH 12/66] Fix zlock launch with TOML port values --- labscript_utils/zlock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/labscript_utils/zlock.py b/labscript_utils/zlock.py index 5878997f..baac341b 100644 --- a/labscript_utils/zlock.py +++ b/labscript_utils/zlock.py @@ -40,7 +40,7 @@ def main(): '-m', 'zprocess.zlock', '--port', - config['zlock_port'], + str(config['zlock_port']), '-l', LOG_PATH, ] From c172cae8d3fb9928dd90dda7624ca9757ef7ee03 Mon Sep 17 00:00:00 2001 From: spielman Date: Sat, 21 Mar 2026 19:01:02 -0400 Subject: [PATCH 13/66] Prefer canonical TOML over stale legacy app configs --- labscript_profile/__init__.py | 8 +++++--- labscript_profile/create.py | 1 + labscript_utils/labconfig.py | 34 +++++++++++++++++++++++++++++----- 3 files changed, 35 insertions(+), 8 deletions(-) diff --git a/labscript_profile/__init__.py b/labscript_profile/__init__.py index 1d05b9a4..f2b12580 100644 --- a/labscript_profile/__init__.py +++ b/labscript_profile/__init__.py @@ -42,7 +42,7 @@ def default_labconfig_path(): return None return LABSCRIPT_SUITE_PROFILE / 'labconfig' / f'{hostname()}.toml' - +# LEGACY INI COMPATIBILITY. DEPRECATED CODE, WILL BE REMOVED. def legacy_labconfig_path(): """Temporary legacy INI labconfig location. Slated for removal soon.""" if LABSCRIPT_SUITE_PROFILE is None: @@ -83,6 +83,7 @@ def ensure_labconfig(): return None if labconfig.exists(): return labconfig + # LEGACY INI COMPATIBILITY. DEPRECATED CODE, WILL BE REMOVED. legacy = legacy_labconfig_path() if legacy is None or not legacy.exists(): return labconfig @@ -100,7 +101,7 @@ def _example_labconfig_data(): return load_toml_file(bundled_example) return {} - +# LEGACY INI COMPATIBILITY. DEPRECATED CODE, WILL BE REMOVED. def _coerce_legacy_value(value, template_value): if template_value is None: return value @@ -137,7 +138,7 @@ def _coerce_legacy_value(value, template_value): return [_coerce_legacy_value(item, template_item) for item in parsed] return value - +# LEGACY INI COMPATIBILITY. DEPRECATED CODE, WILL BE REMOVED. def _convert_legacy_labconfig(legacy_path, toml_path): """Temporary legacy migration path from INI to TOML. Slated for removal soon.""" config = configparser.ConfigParser(interpolation=None) @@ -155,6 +156,7 @@ def _convert_legacy_labconfig(legacy_path, toml_path): section_items = dict(config._sections[section]) section_items.pop('__name__', None) autoload = section_items.get('autoload_config_file') + # LEGACY INI COMPATIBILITY. DEPRECATED CODE, WILL BE REMOVED. if isinstance(autoload, str) and autoload.lower().endswith('.ini'): section_items['autoload_config_file'] = autoload[:-4] + '.toml' section_template = template_data.get(canonical_section, {}) diff --git a/labscript_profile/create.py b/labscript_profile/create.py index b1b3bff1..dcea2d11 100644 --- a/labscript_profile/create.py +++ b/labscript_profile/create.py @@ -48,6 +48,7 @@ def make_labconfig_file(apparatus_name = None): source_path = os.path.join(LABSCRIPT_SUITE_PROFILE, 'labconfig', 'example.toml') target_path = default_labconfig_path() + # LEGACY INI COMPATIBILITY. DEPRECATED CODE, WILL BE REMOVED. legacy_path = legacy_labconfig_path() if os.path.exists(target_path) or (legacy_path is not None and os.path.exists(legacy_path)): raise FileExistsError(target_path) diff --git a/labscript_utils/labconfig.py b/labscript_utils/labconfig.py index 86bb7101..971bd4cd 100644 --- a/labscript_utils/labconfig.py +++ b/labscript_utils/labconfig.py @@ -120,6 +120,7 @@ def _read_path(self, path): if path.suffix.lower() == '.toml': self.read_toml(path) return + # LEGACY INI COMPATIBILITY. DEPRECATED CODE, WILL BE REMOVED. if path.suffix.lower() == '.ini': msg = """INI labconfig support is deprecated and slated for removal soon. Convert this file to TOML.""" @@ -133,16 +134,18 @@ def _read_path(self, path): def _resolve_appconfig_load_path(filename): path = Path(filename) suffix = path.suffix.lower() + # LEGACY INI COMPATIBILITY. DEPRECATED CODE, WILL BE REMOVED. if suffix == '.ini': - if path.exists(): - return path toml_path = path.with_suffix('.toml') if toml_path.exists(): return toml_path + if path.exists(): + return path return path if suffix == '.toml': if path.exists(): return path + # LEGACY INI COMPATIBILITY. DEPRECATED CODE, WILL BE REMOVED. legacy_path = path.with_suffix('.ini') if legacy_path.exists(): return legacy_path @@ -150,6 +153,7 @@ def _resolve_appconfig_load_path(filename): toml_path = path.with_suffix('.toml') if toml_path.exists(): return toml_path + # LEGACY INI COMPATIBILITY. DEPRECATED CODE, WILL BE REMOVED. legacy_path = path.with_suffix('.ini') if legacy_path.exists(): return legacy_path @@ -159,6 +163,23 @@ def _resolve_appconfig_load_path(filename): def _resolve_appconfig_save_path(filename): return Path(filename).with_suffix('.toml') +# LEGACY INI COMPATIBILITY. DEPRECATED CODE, WILL BE REMOVED. +def backup_legacy_config(path): + """Rename a migrated legacy config file to ``.old`` without clobbering backups.""" + path = Path(path) + if not path.exists(): + return None + backup_path = Path(str(path) + '.old') + index = 1 + while backup_path.exists(): + backup_path = Path(str(path) + '.old' + str(index)) + index += 1 + try: + path.rename(backup_path) + except OSError: + return None + return backup_path + def _to_toml_compatible(value, location='value'): if isinstance(value, dict): @@ -179,7 +200,7 @@ def _to_toml_compatible(value, location='value'): msg = f"{location} value {value!r} is not representable in TOML app config" raise TypeError(msg) - +# LEGACY INI COMPATIBILITY. DEPRECATED CODE, WILL BE REMOVED. def _load_legacy_appconfig_value(value): """Temporary legacy INI app-config decoder. Slated for removal soon.""" if not isinstance(value, str): @@ -209,12 +230,14 @@ def load_appconfig(filename, return_save_path=False): """Load a TOML app config, or a legacy INI app config if required. If a legacy INI file is loaded, a sibling TOML file is immediately written and can be - returned as the canonical save path via ``return_save_path=True``. Legacy INI support - here is temporary and slated for removal soon. + returned as the canonical save path via ``return_save_path=True``. After successful + conversion, the legacy INI file is moved aside to ``.old`` so future loads prefer the + canonical TOML file. Legacy INI support here is temporary and slated for removal soon. """ requested_path = Path(filename) filename = _resolve_appconfig_load_path(filename) save_path = _resolve_appconfig_save_path(requested_path) + # LEGACY INI COMPATIBILITY. DEPRECATED CODE, WILL BE REMOVED. if filename.suffix.lower() == '.ini': c = configparser.ConfigParser(interpolation=None) c.optionxform = str @@ -229,6 +252,7 @@ def load_appconfig(filename, return_save_path=False): } if filename.exists(): save_path = Path(save_appconfig(save_path, data)) + backup_legacy_config(filename) else: raw = load_toml_file(filename) if filename.exists() else {} data = { From 5725e3e306832fb7597245cb24ad1dc274059a8c Mon Sep 17 00:00:00 2001 From: spielman Date: Thu, 26 Mar 2026 09:14:10 -0400 Subject: [PATCH 14/66] Set macOS app icons from splash path --- labscript_utils/splash.py | 39 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/labscript_utils/splash.py b/labscript_utils/splash.py index 51c56864..94b4e960 100644 --- a/labscript_utils/splash.py +++ b/labscript_utils/splash.py @@ -39,6 +39,39 @@ QtWidgets.QApplication.setAttribute(Qt.AA_UseHighDpiPixmaps, True) +def configure_qapplication(qapplication): + """Apply labscript-wide QApplication configuration.""" + qapplication.setAttribute(Qt.AA_DontShowIconsInMenus, False) + if sys.platform == 'darwin': + icon_path = qapplication.property('_labscript_icon_path') + if icon_path: + icon = QtGui.QIcon(icon_path) + if not icon.isNull(): + qapplication.setWindowIcon(icon) + if qapplication.property('_labscript_qapplication_configured'): + return qapplication + # Native macOS widget styling makes many Qt controls look inconsistent + # with the rest of the suite. Use Qt's own style, but preserve the + # current palette so dark/light appearance still follows the active + # theme. + palette = QtGui.QPalette(qapplication.palette()) + style = QtWidgets.QStyleFactory.create('Fusion') + if style is not None: + qapplication.setStyle(style) + qapplication.setPalette(palette) + elif qapplication.property('_labscript_qapplication_configured'): + return qapplication + qapplication.setProperty('_labscript_qapplication_configured', True) + return qapplication + + +def get_qapplication(argv=None): + qapplication = QtWidgets.QApplication.instance() + if qapplication is None: + qapplication = QtWidgets.QApplication(sys.argv if argv is None else argv) + return configure_qapplication(qapplication) + + class Splash(QtWidgets.QFrame): w = 250 h = 230 @@ -50,9 +83,9 @@ class Splash(QtWidgets.QFrame): FG = '#000000' def __init__(self, imagepath): - self.qapplication = QtWidgets.QApplication.instance() - if self.qapplication is None: - self.qapplication = QtWidgets.QApplication(sys.argv) + self.qapplication = get_qapplication() + self.qapplication.setProperty('_labscript_icon_path', imagepath) + configure_qapplication(self.qapplication) super().__init__() self.icon = QtGui.QPixmap() self.icon.load(imagepath) From 89fdbdec69467514bc0142abdf9ed48beeafbbb7 Mon Sep 17 00:00:00 2001 From: spielman Date: Thu, 26 Mar 2026 09:22:56 -0400 Subject: [PATCH 15/66] Inline redundant splash QApplication helper --- labscript_utils/splash.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/labscript_utils/splash.py b/labscript_utils/splash.py index 94b4e960..00890695 100644 --- a/labscript_utils/splash.py +++ b/labscript_utils/splash.py @@ -65,13 +65,6 @@ def configure_qapplication(qapplication): return qapplication -def get_qapplication(argv=None): - qapplication = QtWidgets.QApplication.instance() - if qapplication is None: - qapplication = QtWidgets.QApplication(sys.argv if argv is None else argv) - return configure_qapplication(qapplication) - - class Splash(QtWidgets.QFrame): w = 250 h = 230 @@ -83,7 +76,9 @@ class Splash(QtWidgets.QFrame): FG = '#000000' def __init__(self, imagepath): - self.qapplication = get_qapplication() + self.qapplication = QtWidgets.QApplication.instance() + if self.qapplication is None: + self.qapplication = QtWidgets.QApplication(sys.argv) self.qapplication.setProperty('_labscript_icon_path', imagepath) configure_qapplication(self.qapplication) super().__init__() From 454570c12748266641f4f3ea86523b05e9644cb4 Mon Sep 17 00:00:00 2001 From: spielman Date: Thu, 26 Mar 2026 17:02:09 -0400 Subject: [PATCH 16/66] Add shared indexed filepath helper --- labscript_utils/file_utils.py | 41 +++++++++++++++++++++++++++++++++++ labscript_utils/shot_utils.py | 3 ++- 2 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 labscript_utils/file_utils.py diff --git a/labscript_utils/file_utils.py b/labscript_utils/file_utils.py new file mode 100644 index 00000000..57a2ef85 --- /dev/null +++ b/labscript_utils/file_utils.py @@ -0,0 +1,41 @@ +##################################################################### +# # +# file_utils.py # +# # +# Copyright 2013, Monash University # +# # +# This file is part of the labscript suite (see # +# http://labscriptsuite.org) and is licensed under the Simplified # +# BSD License. See the license.txt file in the root of the project # +# for the full license. # +# # +##################################################################### + +import os +import re + + +def next_available_indexed_filepath(filepath, suffix_format, start=0): + """Return the first non-existent filepath created by appending a formatted suffix. + + ``suffix_format`` is formatted with a single ``index`` keyword argument and is + inserted between the filepath stem and its extension. If filepath already ends + with a suffix matching the pattern, that suffix is stripped and the search begins + at the next higher index.""" + basename, ext = os.path.splitext(filepath) + suffix_pattern = re.escape(suffix_format) + suffix_pattern = re.sub( + r'\\\{index(?::[^}]*)?\\\}', + lambda match: r'(?P\d+)', + suffix_pattern, + ) + match = re.search(suffix_pattern + r'$', basename) + if match is not None: + basename = basename[:match.start()] + start = max(start, int(match.group('index')) + 1) + index = start + while True: + candidate = basename + suffix_format.format(index=index) + ext + if not os.path.exists(candidate): + return candidate, index + index += 1 diff --git a/labscript_utils/shot_utils.py b/labscript_utils/shot_utils.py index aaccced9..987abed7 100644 --- a/labscript_utils/shot_utils.py +++ b/labscript_utils/shot_utils.py @@ -14,6 +14,7 @@ import h5py import numpy as np + def get_shot_globals(filepath): """Returns the evaluated globals for a shot, for use by labscript or lyse. Simple dictionary access as in dict(h5py.File(filepath).attrs) would be fine @@ -34,4 +35,4 @@ def get_shot_globals(filepath): if isinstance(value, bytes): value = value.decode() params[name] = value - return params \ No newline at end of file + return params From 8bf4faab0a1d1db57adf859a8f5c1c7486e5190e Mon Sep 17 00:00:00 2001 From: spielman Date: Mon, 6 Apr 2026 11:23:59 -0400 Subject: [PATCH 17/66] Make indexed filepath start explicit --- labscript_utils/file_utils.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/labscript_utils/file_utils.py b/labscript_utils/file_utils.py index 57a2ef85..ee032183 100644 --- a/labscript_utils/file_utils.py +++ b/labscript_utils/file_utils.py @@ -15,13 +15,14 @@ import re -def next_available_indexed_filepath(filepath, suffix_format, start=0): +def next_available_indexed_filepath(filepath, suffix_format, start=None): """Return the first non-existent filepath created by appending a formatted suffix. ``suffix_format`` is formatted with a single ``index`` keyword argument and is inserted between the filepath stem and its extension. If filepath already ends - with a suffix matching the pattern, that suffix is stripped and the search begins - at the next higher index.""" + with a suffix matching the pattern, that suffix is stripped. If ``start`` is + ``None``, the search begins at the next higher index from that suffix. Otherwise + ``start`` is used exactly as provided.""" basename, ext = os.path.splitext(filepath) suffix_pattern = re.escape(suffix_format) suffix_pattern = re.sub( @@ -32,7 +33,10 @@ def next_available_indexed_filepath(filepath, suffix_format, start=0): match = re.search(suffix_pattern + r'$', basename) if match is not None: basename = basename[:match.start()] - start = max(start, int(match.group('index')) + 1) + if start is None: + start = int(match.group('index')) + 1 + if start is None: + start = 0 index = start while True: candidate = basename + suffix_format.format(index=index) + ext From d87ad88588ed7c375d84a960bebcb5c27ab40926 Mon Sep 17 00:00:00 2001 From: spielman Date: Mon, 6 Apr 2026 11:55:10 -0400 Subject: [PATCH 18/66] Simplify shared shot queue controls --- labscript_utils/qtwidgets/shotqueue.py | 21 +-------------------- 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/labscript_utils/qtwidgets/shotqueue.py b/labscript_utils/qtwidgets/shotqueue.py index df95b23b..30c8c54c 100644 --- a/labscript_utils/qtwidgets/shotqueue.py +++ b/labscript_utils/qtwidgets/shotqueue.py @@ -132,6 +132,7 @@ def __init__( self.queue_view = ShotQueueTreeView(self, accepted_extensions=self.accepted_extensions) self.queue_view.setModel(self.queue_model) + self.queue_view.setVerticalScrollBarPolicy(Qt.ScrollBarAlwaysOn) self.add_button = QToolButton(self) self.add_button.setText('Add') @@ -139,14 +140,6 @@ def __init__( self.delete_button.setText('Delete') self.clear_button = QToolButton(self) self.clear_button.setText('Clear') - self.move_top_button = QToolButton(self) - self.move_top_button.setText('Top') - self.move_up_button = QToolButton(self) - self.move_up_button.setText('Up') - self.move_down_button = QToolButton(self) - self.move_down_button.setText('Down') - self.move_bottom_button = QToolButton(self) - self.move_bottom_button.setText('Bottom') button_layout = QHBoxLayout() button_layout.setContentsMargins(0, 0, 0, 0) @@ -154,10 +147,6 @@ def __init__( button_layout.addWidget(self.delete_button) button_layout.addWidget(self.clear_button) button_layout.addStretch(1) - button_layout.addWidget(self.move_top_button) - button_layout.addWidget(self.move_up_button) - button_layout.addWidget(self.move_down_button) - button_layout.addWidget(self.move_bottom_button) layout = QVBoxLayout(self) layout.setContentsMargins(0, 0, 0, 0) @@ -167,10 +156,6 @@ def __init__( self.add_button.clicked.connect(self.prompt_for_files) self.delete_button.clicked.connect(self.remove_selected) self.clear_button.clicked.connect(self.clear) - self.move_top_button.clicked.connect(self.move_top) - self.move_up_button.clicked.connect(self.move_up) - self.move_down_button.clicked.connect(self.move_down) - self.move_bottom_button.clicked.connect(self.move_bottom) self.queue_view.deleteRequested.connect(self.remove_selected) self.queue_view.filesDropped.connect(self.add_files) self.queue_model.rowsInserted.connect(self._on_queue_changed) @@ -340,10 +325,6 @@ def _update_button_states(self): has_selection = bool(selected_rows) self.delete_button.setEnabled(has_selection) self.clear_button.setEnabled(bool(row_count)) - self.move_top_button.setEnabled(has_selection and selected_rows[0] > 0) - self.move_up_button.setEnabled(has_selection and selected_rows[0] > 0) - self.move_bottom_button.setEnabled(has_selection and selected_rows[-1] < row_count - 1) - self.move_down_button.setEnabled(has_selection and selected_rows[-1] < row_count - 1) def _select_rows(self, rows): rows = list(rows) From 7c13f18ac876f7d788e2ac5787d3e9cd18f42bf5 Mon Sep 17 00:00:00 2001 From: spielman Date: Mon, 6 Apr 2026 12:00:28 -0400 Subject: [PATCH 19/66] Use context-menu deletion in shot queue widget --- labscript_utils/qtwidgets/shotqueue.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/labscript_utils/qtwidgets/shotqueue.py b/labscript_utils/qtwidgets/shotqueue.py index 30c8c54c..46e4c87b 100644 --- a/labscript_utils/qtwidgets/shotqueue.py +++ b/labscript_utils/qtwidgets/shotqueue.py @@ -64,6 +64,17 @@ def keyPressEvent(self, event): return QTreeView.keyPressEvent(self, event) + def contextMenuEvent(self, event): + menu = QMenu(self) + delete_action = menu.addAction('Delete selected rows') + delete_action.setEnabled(bool(self.selectionModel().selectedRows())) + chosen_action = menu.exec_(event.globalPos()) + if chosen_action is delete_action: + self.deleteRequested.emit() + event.accept() + return + QTreeView.contextMenuEvent(self, event) + def dragEnterEvent(self, event): if self._event_has_acceptable_urls(event): event.setDropAction(Qt.CopyAction) @@ -136,16 +147,10 @@ def __init__( self.add_button = QToolButton(self) self.add_button.setText('Add') - self.delete_button = QToolButton(self) - self.delete_button.setText('Delete') - self.clear_button = QToolButton(self) - self.clear_button.setText('Clear') button_layout = QHBoxLayout() button_layout.setContentsMargins(0, 0, 0, 0) button_layout.addWidget(self.add_button) - button_layout.addWidget(self.delete_button) - button_layout.addWidget(self.clear_button) button_layout.addStretch(1) layout = QVBoxLayout(self) @@ -154,8 +159,6 @@ def __init__( layout.addLayout(button_layout) self.add_button.clicked.connect(self.prompt_for_files) - self.delete_button.clicked.connect(self.remove_selected) - self.clear_button.clicked.connect(self.clear) self.queue_view.deleteRequested.connect(self.remove_selected) self.queue_view.filesDropped.connect(self.add_files) self.queue_model.rowsInserted.connect(self._on_queue_changed) @@ -320,11 +323,7 @@ def _on_selection_changed(self, *args): self.selectionChanged.emit(self.selected_files()) def _update_button_states(self): - row_count = self.queue_model.rowCount() - selected_rows = self.selected_rows() - has_selection = bool(selected_rows) - self.delete_button.setEnabled(has_selection) - self.clear_button.setEnabled(bool(row_count)) + pass def _select_rows(self, rows): rows = list(rows) From 29999395b9cf3a2a0f2d1bd962650ce0ee75adea Mon Sep 17 00:00:00 2001 From: spielman Date: Mon, 6 Apr 2026 12:02:45 -0400 Subject: [PATCH 20/66] Handle macOS delete key in shot queue --- labscript_utils/qtwidgets/shotqueue.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/labscript_utils/qtwidgets/shotqueue.py b/labscript_utils/qtwidgets/shotqueue.py index 46e4c87b..68a48f3b 100644 --- a/labscript_utils/qtwidgets/shotqueue.py +++ b/labscript_utils/qtwidgets/shotqueue.py @@ -58,7 +58,7 @@ def set_accepted_extensions(self, accepted_extensions): self._accepted_extensions = _normalise_extensions(accepted_extensions) def keyPressEvent(self, event): - if event.key() == Qt.Key_Delete: + if event.key() in (Qt.Key_Delete, Qt.Key_Backspace): event.accept() self.deleteRequested.emit() return From 5876c9b87760719752aaa5384c87e7f061e7fcf4 Mon Sep 17 00:00:00 2001 From: spielman Date: Mon, 6 Apr 2026 12:10:21 -0400 Subject: [PATCH 21/66] Move shot queue column support into shared widget --- labscript_utils/qtwidgets/shotqueue.py | 89 +++++++++++++++++++++++--- 1 file changed, 81 insertions(+), 8 deletions(-) diff --git a/labscript_utils/qtwidgets/shotqueue.py b/labscript_utils/qtwidgets/shotqueue.py index 68a48f3b..08d92775 100644 --- a/labscript_utils/qtwidgets/shotqueue.py +++ b/labscript_utils/qtwidgets/shotqueue.py @@ -118,7 +118,7 @@ def _is_accepted_path(self, path): return os.path.isfile(path) and path.lower().endswith(self._accepted_extensions) class ShotQueueWidget(QWidget): - """Reusable single-column shot queue editor widget.""" + """Reusable shot queue editor widget.""" queueChanged = Signal() selectionChanged = Signal(list) @@ -131,19 +131,31 @@ def __init__( file_dialog_filter='Shot files (*.h5 *.hdf5)', allow_duplicates=False, column_title='Filepath', + column_titles=None, ): QWidget.__init__(self, parent) self.accepted_extensions = _normalise_extensions(accepted_extensions) self.file_dialog_filter = file_dialog_filter self.allow_duplicates = allow_duplicates self.last_opened_shots_folder = '' + if column_titles is None: + column_titles = [column_title] + self.column_titles = list(column_titles) self.queue_model = QStandardItemModel(self) - self.queue_model.setHorizontalHeaderItem(FILEPATH_COLUMN, QStandardItem(column_title)) + self.queue_model.setColumnCount(len(self.column_titles)) + for column, title in enumerate(self.column_titles): + self.queue_model.setHorizontalHeaderItem(column, QStandardItem(title)) self.queue_view = ShotQueueTreeView(self, accepted_extensions=self.accepted_extensions) self.queue_view.setModel(self.queue_model) self.queue_view.setVerticalScrollBarPolicy(Qt.ScrollBarAlwaysOn) + self.queue_view.header().setStretchLastSection(False) + self.queue_view.header().setSectionResizeMode(FILEPATH_COLUMN, QHeaderView.Stretch) + for column in range(1, len(self.column_titles)): + self.queue_view.header().setSectionResizeMode( + column, QHeaderView.ResizeToContents + ) self.add_button = QToolButton(self) self.add_button.setText('Add') @@ -170,12 +182,12 @@ def __init__( def files(self): return [ - self.queue_model.item(row, FILEPATH_COLUMN).text() + self._queue_path_for_row(row) for row in range(self.queue_model.rowCount()) ] def selected_files(self): - return [self.queue_model.item(row, FILEPATH_COLUMN).text() for row in self.selected_rows()] + return [self._queue_path_for_row(row) for row in self.selected_rows()] def selected_rows(self): return sorted(index.row() for index in self.queue_view.selectionModel().selectedRows()) @@ -186,8 +198,7 @@ def select_paths(self, paths): path_lookup = {os.path.abspath(str(path)) for path in paths} rows = [] for row in range(self.queue_model.rowCount()): - item = self.queue_model.item(row, FILEPATH_COLUMN) - if item.text() in path_lookup: + if self._queue_path_for_row(row) in path_lookup: rows.append(row) self._select_rows(rows) @@ -277,7 +288,10 @@ def move_bottom(self): def is_in_queue(self, path): path = os.path.abspath(str(path)) - return bool(self.queue_model.findItems(path, column=FILEPATH_COLUMN)) + return any( + self._queue_path_for_row(row) == path + for row in range(self.queue_model.rowCount()) + ) def get_save_data(self): return { @@ -289,11 +303,70 @@ def restore_save_data(self, data): self.last_opened_shots_folder = data.get('last_opened_shots_folder', '') self.set_files(data.get('files_queued', [])) + def set_row_infos(self, row_infos): + self.queue_model.removeRows(0, self.queue_model.rowCount()) + if not row_infos: + return + for row_info in row_infos: + self.queue_model.appendRow(self._create_row_from_info(row_info)) + def _create_row(self, path): item = QStandardItem(path) item.setToolTip(path) item.setEditable(False) - return [item] + item.setData(path, Qt.UserRole) + return [item] + self._create_padding_items(self.queue_model.columnCount() - 1) + + def _create_row_from_info(self, row_info): + if not isinstance(row_info, dict): + return self._create_row(str(row_info)) + path = os.path.abspath(str(row_info['path'])) + label = row_info.get('label', os.path.basename(path)) + tooltip = row_info.get('tooltip', path) + columns = list(row_info.get('columns', [])) + + item = QStandardItem(label) + item.setEditable(False) + item.setToolTip(tooltip) + item.setData(path, Qt.UserRole) + row_items = [item] + + for column_info in columns[: self.queue_model.columnCount() - 1]: + if isinstance(column_info, dict): + text = str(column_info.get('text', '')) + column_tooltip = column_info.get('tooltip', '') + alignment = column_info.get( + 'alignment', Qt.AlignLeft | Qt.AlignVCenter + ) + else: + text = str(column_info) + column_tooltip = '' + alignment = Qt.AlignLeft | Qt.AlignVCenter + column_item = QStandardItem(text) + column_item.setEditable(False) + column_item.setToolTip(column_tooltip) + column_item.setTextAlignment(alignment) + row_items.append(column_item) + + row_items.extend( + self._create_padding_items(self.queue_model.columnCount() - len(row_items)) + ) + return row_items + + def _create_padding_items(self, count): + items = [] + for _ in range(max(0, count)): + item = QStandardItem('') + item.setEditable(False) + items.append(item) + return items + + def _queue_path_for_row(self, row): + item = self.queue_model.item(row, FILEPATH_COLUMN) + path = item.data(Qt.UserRole) + if path is None: + path = item.text() + return os.path.abspath(str(path)) def _prepare_paths(self, paths): if isinstance(paths, str): From a66065477b7a5122cb36e99f4379462333d2998d Mon Sep 17 00:00:00 2001 From: spielman Date: Mon, 6 Apr 2026 12:12:43 -0400 Subject: [PATCH 22/66] Support path column selection in shot queue widget --- labscript_utils/qtwidgets/shotqueue.py | 87 ++++++++++++++------------ 1 file changed, 47 insertions(+), 40 deletions(-) diff --git a/labscript_utils/qtwidgets/shotqueue.py b/labscript_utils/qtwidgets/shotqueue.py index 08d92775..0183b187 100644 --- a/labscript_utils/qtwidgets/shotqueue.py +++ b/labscript_utils/qtwidgets/shotqueue.py @@ -132,6 +132,7 @@ def __init__( allow_duplicates=False, column_title='Filepath', column_titles=None, + path_column=FILEPATH_COLUMN, ): QWidget.__init__(self, parent) self.accepted_extensions = _normalise_extensions(accepted_extensions) @@ -141,6 +142,7 @@ def __init__( if column_titles is None: column_titles = [column_title] self.column_titles = list(column_titles) + self.path_column = int(path_column) self.queue_model = QStandardItemModel(self) self.queue_model.setColumnCount(len(self.column_titles)) @@ -149,13 +151,15 @@ def __init__( self.queue_view = ShotQueueTreeView(self, accepted_extensions=self.accepted_extensions) self.queue_view.setModel(self.queue_model) - self.queue_view.setVerticalScrollBarPolicy(Qt.ScrollBarAlwaysOn) + self.queue_view.setVerticalScrollBarPolicy(Qt.ScrollBarAsNeeded) self.queue_view.header().setStretchLastSection(False) - self.queue_view.header().setSectionResizeMode(FILEPATH_COLUMN, QHeaderView.Stretch) - for column in range(1, len(self.column_titles)): - self.queue_view.header().setSectionResizeMode( - column, QHeaderView.ResizeToContents + for column in range(len(self.column_titles)): + resize_mode = ( + QHeaderView.Stretch + if column == self.path_column + else QHeaderView.ResizeToContents ) + self.queue_view.header().setSectionResizeMode(column, resize_mode) self.add_button = QToolButton(self) self.add_button.setText('Add') @@ -311,11 +315,13 @@ def set_row_infos(self, row_infos): self.queue_model.appendRow(self._create_row_from_info(row_info)) def _create_row(self, path): - item = QStandardItem(path) - item.setToolTip(path) - item.setEditable(False) - item.setData(path, Qt.UserRole) - return [item] + self._create_padding_items(self.queue_model.columnCount() - 1) + row_items = self._create_padding_items(self.queue_model.columnCount()) + row_items[self.path_column] = self._create_display_item( + path, + tooltip=path, + path=path, + ) + return row_items def _create_row_from_info(self, row_info): if not isinstance(row_info, dict): @@ -324,45 +330,46 @@ def _create_row_from_info(self, row_info): label = row_info.get('label', os.path.basename(path)) tooltip = row_info.get('tooltip', path) columns = list(row_info.get('columns', [])) - - item = QStandardItem(label) - item.setEditable(False) - item.setToolTip(tooltip) - item.setData(path, Qt.UserRole) - row_items = [item] - - for column_info in columns[: self.queue_model.columnCount() - 1]: - if isinstance(column_info, dict): - text = str(column_info.get('text', '')) - column_tooltip = column_info.get('tooltip', '') - alignment = column_info.get( - 'alignment', Qt.AlignLeft | Qt.AlignVCenter - ) - else: - text = str(column_info) - column_tooltip = '' - alignment = Qt.AlignLeft | Qt.AlignVCenter - column_item = QStandardItem(text) - column_item.setEditable(False) - column_item.setToolTip(column_tooltip) - column_item.setTextAlignment(alignment) - row_items.append(column_item) - - row_items.extend( - self._create_padding_items(self.queue_model.columnCount() - len(row_items)) + row_items = self._create_padding_items(self.queue_model.columnCount()) + row_items[self.path_column] = self._create_display_item( + label, + tooltip=tooltip, + path=path, ) + extra_columns = [i for i in range(self.queue_model.columnCount()) if i != self.path_column] + for column_index, column_info in zip(extra_columns, columns): + row_items[column_index] = self._create_display_item_from_info(column_info) return row_items def _create_padding_items(self, count): items = [] for _ in range(max(0, count)): - item = QStandardItem('') - item.setEditable(False) - items.append(item) + items.append(self._create_display_item('')) return items + def _create_display_item(self, text, tooltip='', alignment=None, path=None): + item = QStandardItem(str(text)) + item.setEditable(False) + item.setToolTip(tooltip) + if alignment is not None: + item.setTextAlignment(alignment) + if path is not None: + item.setData(os.path.abspath(str(path)), Qt.UserRole) + return item + + def _create_display_item_from_info(self, column_info): + if isinstance(column_info, dict): + text = column_info.get('text', '') + tooltip = column_info.get('tooltip', '') + alignment = column_info.get('alignment', Qt.AlignLeft | Qt.AlignVCenter) + else: + text = column_info + tooltip = '' + alignment = Qt.AlignLeft | Qt.AlignVCenter + return self._create_display_item(text, tooltip=tooltip, alignment=alignment) + def _queue_path_for_row(self, row): - item = self.queue_model.item(row, FILEPATH_COLUMN) + item = self.queue_model.item(row, self.path_column) path = item.data(Qt.UserRole) if path is None: path = item.text() From 9e581dbc1892cb6399c75f40bc2400a7c5696585 Mon Sep 17 00:00:00 2001 From: spielman Date: Mon, 6 Apr 2026 12:16:40 -0400 Subject: [PATCH 23/66] Use external scrollbar for shot queue widget --- labscript_utils/qtwidgets/shotqueue.py | 43 +++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/labscript_utils/qtwidgets/shotqueue.py b/labscript_utils/qtwidgets/shotqueue.py index 0183b187..99d553c8 100644 --- a/labscript_utils/qtwidgets/shotqueue.py +++ b/labscript_utils/qtwidgets/shotqueue.py @@ -151,7 +151,7 @@ def __init__( self.queue_view = ShotQueueTreeView(self, accepted_extensions=self.accepted_extensions) self.queue_view.setModel(self.queue_model) - self.queue_view.setVerticalScrollBarPolicy(Qt.ScrollBarAsNeeded) + self.queue_view.setVerticalScrollBarPolicy(Qt.ScrollBarAlwaysOff) self.queue_view.header().setStretchLastSection(False) for column in range(len(self.column_titles)): resize_mode = ( @@ -160,6 +160,7 @@ def __init__( else QHeaderView.ResizeToContents ) self.queue_view.header().setSectionResizeMode(column, resize_mode) + self.queue_scrollbar = QScrollBar(Qt.Vertical, self) self.add_button = QToolButton(self) self.add_button.setText('Add') @@ -169,9 +170,15 @@ def __init__( button_layout.addWidget(self.add_button) button_layout.addStretch(1) + queue_layout = QHBoxLayout() + queue_layout.setContentsMargins(0, 0, 0, 0) + queue_layout.setSpacing(0) + queue_layout.addWidget(self.queue_view) + queue_layout.addWidget(self.queue_scrollbar) + layout = QVBoxLayout(self) layout.setContentsMargins(0, 0, 0, 0) - layout.addWidget(self.queue_view) + layout.addLayout(queue_layout) layout.addLayout(button_layout) self.add_button.clicked.connect(self.prompt_for_files) @@ -181,7 +188,12 @@ def __init__( self.queue_model.rowsRemoved.connect(self._on_queue_changed) self.queue_model.modelReset.connect(self._on_queue_changed) self.queue_view.selectionModel().selectionChanged.connect(self._on_selection_changed) + self.queue_view.verticalScrollBar().rangeChanged.connect(self._sync_vertical_scrollbar) + self.queue_view.verticalScrollBar().valueChanged.connect(self.queue_scrollbar.setValue) + self.queue_scrollbar.valueChanged.connect(self.queue_view.verticalScrollBar().setValue) + self._sync_vertical_scrollbar() + self._resize_auxiliary_columns() self._update_button_states() def files(self): @@ -216,6 +228,7 @@ def append(self, paths): return [] for path in paths: self.queue_model.appendRow(self._create_row(path)) + self._resize_auxiliary_columns() self.filesAdded.emit(paths) return paths @@ -224,6 +237,7 @@ def prepend(self, path): if not paths: return [] self.queue_model.insertRow(0, self._create_row(paths[0])) + self._resize_auxiliary_columns() self.filesAdded.emit(paths) self._select_rows([0]) return paths @@ -310,9 +324,11 @@ def restore_save_data(self, data): def set_row_infos(self, row_infos): self.queue_model.removeRows(0, self.queue_model.rowCount()) if not row_infos: + self._resize_auxiliary_columns() return for row_info in row_infos: self.queue_model.appendRow(self._create_row_from_info(row_info)) + self._resize_auxiliary_columns() def _create_row(self, path): row_items = self._create_padding_items(self.queue_model.columnCount()) @@ -375,6 +391,22 @@ def _queue_path_for_row(self, row): path = item.text() return os.path.abspath(str(path)) + def _resize_auxiliary_columns(self): + header = self.queue_view.header() + for column, title in enumerate(self.column_titles): + if column == self.path_column: + continue + title_width = self.fontMetrics().horizontalAdvance(title) + 24 + content_width = max(0, self.queue_view.sizeHintForColumn(column)) + 12 + header.resizeSection(column, max(title_width, content_width)) + + def _sync_vertical_scrollbar(self, minimum=0, maximum=0): + tree_scrollbar = self.queue_view.verticalScrollBar() + self.queue_scrollbar.setRange(minimum, maximum) + self.queue_scrollbar.setPageStep(tree_scrollbar.pageStep()) + self.queue_scrollbar.setSingleStep(tree_scrollbar.singleStep()) + self.queue_scrollbar.setValue(tree_scrollbar.value()) + def _prepare_paths(self, paths): if isinstance(paths, str): paths = [paths] @@ -395,6 +427,9 @@ def _is_accepted_path(self, path): return os.path.isfile(path) and path.lower().endswith(self.accepted_extensions) def _on_queue_changed(self, *args): + self._resize_auxiliary_columns() + tree_scrollbar = self.queue_view.verticalScrollBar() + self._sync_vertical_scrollbar(tree_scrollbar.minimum(), tree_scrollbar.maximum()) self._update_button_states() self.queueChanged.emit() @@ -410,7 +445,7 @@ def _select_rows(self, rows): selection_model = self.queue_view.selectionModel() selection_model.clearSelection() for row in rows: - index = self.queue_model.index(row, FILEPATH_COLUMN) + index = self.queue_model.index(row, self.path_column) selection_model.select(index, QItemSelectionModel.Select | QItemSelectionModel.Rows) if rows: - self.queue_view.setCurrentIndex(self.queue_model.index(rows[0], FILEPATH_COLUMN)) + self.queue_view.setCurrentIndex(self.queue_model.index(rows[0], self.path_column)) From fe14e619000cd080e65558d9390466df99f31719 Mon Sep 17 00:00:00 2001 From: spielman Date: Mon, 6 Apr 2026 12:19:15 -0400 Subject: [PATCH 24/66] Reserve corner space for shot queue scrollbar --- labscript_utils/qtwidgets/shotqueue.py | 42 +++----------------------- 1 file changed, 5 insertions(+), 37 deletions(-) diff --git a/labscript_utils/qtwidgets/shotqueue.py b/labscript_utils/qtwidgets/shotqueue.py index 99d553c8..140e3339 100644 --- a/labscript_utils/qtwidgets/shotqueue.py +++ b/labscript_utils/qtwidgets/shotqueue.py @@ -151,7 +151,7 @@ def __init__( self.queue_view = ShotQueueTreeView(self, accepted_extensions=self.accepted_extensions) self.queue_view.setModel(self.queue_model) - self.queue_view.setVerticalScrollBarPolicy(Qt.ScrollBarAlwaysOff) + self.queue_view.setVerticalScrollBarPolicy(Qt.ScrollBarAlwaysOn) self.queue_view.header().setStretchLastSection(False) for column in range(len(self.column_titles)): resize_mode = ( @@ -160,7 +160,9 @@ def __init__( else QHeaderView.ResizeToContents ) self.queue_view.header().setSectionResizeMode(column, resize_mode) - self.queue_scrollbar = QScrollBar(Qt.Vertical, self) + self.scrollbar_corner = QWidget(self.queue_view) + self.scrollbar_corner.setFixedWidth(self.queue_view.verticalScrollBar().sizeHint().width()) + self.queue_view.setCornerWidget(self.scrollbar_corner) self.add_button = QToolButton(self) self.add_button.setText('Add') @@ -170,15 +172,9 @@ def __init__( button_layout.addWidget(self.add_button) button_layout.addStretch(1) - queue_layout = QHBoxLayout() - queue_layout.setContentsMargins(0, 0, 0, 0) - queue_layout.setSpacing(0) - queue_layout.addWidget(self.queue_view) - queue_layout.addWidget(self.queue_scrollbar) - layout = QVBoxLayout(self) layout.setContentsMargins(0, 0, 0, 0) - layout.addLayout(queue_layout) + layout.addWidget(self.queue_view) layout.addLayout(button_layout) self.add_button.clicked.connect(self.prompt_for_files) @@ -188,12 +184,7 @@ def __init__( self.queue_model.rowsRemoved.connect(self._on_queue_changed) self.queue_model.modelReset.connect(self._on_queue_changed) self.queue_view.selectionModel().selectionChanged.connect(self._on_selection_changed) - self.queue_view.verticalScrollBar().rangeChanged.connect(self._sync_vertical_scrollbar) - self.queue_view.verticalScrollBar().valueChanged.connect(self.queue_scrollbar.setValue) - self.queue_scrollbar.valueChanged.connect(self.queue_view.verticalScrollBar().setValue) - self._sync_vertical_scrollbar() - self._resize_auxiliary_columns() self._update_button_states() def files(self): @@ -228,7 +219,6 @@ def append(self, paths): return [] for path in paths: self.queue_model.appendRow(self._create_row(path)) - self._resize_auxiliary_columns() self.filesAdded.emit(paths) return paths @@ -237,7 +227,6 @@ def prepend(self, path): if not paths: return [] self.queue_model.insertRow(0, self._create_row(paths[0])) - self._resize_auxiliary_columns() self.filesAdded.emit(paths) self._select_rows([0]) return paths @@ -324,11 +313,9 @@ def restore_save_data(self, data): def set_row_infos(self, row_infos): self.queue_model.removeRows(0, self.queue_model.rowCount()) if not row_infos: - self._resize_auxiliary_columns() return for row_info in row_infos: self.queue_model.appendRow(self._create_row_from_info(row_info)) - self._resize_auxiliary_columns() def _create_row(self, path): row_items = self._create_padding_items(self.queue_model.columnCount()) @@ -391,22 +378,6 @@ def _queue_path_for_row(self, row): path = item.text() return os.path.abspath(str(path)) - def _resize_auxiliary_columns(self): - header = self.queue_view.header() - for column, title in enumerate(self.column_titles): - if column == self.path_column: - continue - title_width = self.fontMetrics().horizontalAdvance(title) + 24 - content_width = max(0, self.queue_view.sizeHintForColumn(column)) + 12 - header.resizeSection(column, max(title_width, content_width)) - - def _sync_vertical_scrollbar(self, minimum=0, maximum=0): - tree_scrollbar = self.queue_view.verticalScrollBar() - self.queue_scrollbar.setRange(minimum, maximum) - self.queue_scrollbar.setPageStep(tree_scrollbar.pageStep()) - self.queue_scrollbar.setSingleStep(tree_scrollbar.singleStep()) - self.queue_scrollbar.setValue(tree_scrollbar.value()) - def _prepare_paths(self, paths): if isinstance(paths, str): paths = [paths] @@ -427,9 +398,6 @@ def _is_accepted_path(self, path): return os.path.isfile(path) and path.lower().endswith(self.accepted_extensions) def _on_queue_changed(self, *args): - self._resize_auxiliary_columns() - tree_scrollbar = self.queue_view.verticalScrollBar() - self._sync_vertical_scrollbar(tree_scrollbar.minimum(), tree_scrollbar.maximum()) self._update_button_states() self.queueChanged.emit() From 94d9d2e90b5d6adc49ba86aa843cad36d029a213 Mon Sep 17 00:00:00 2001 From: spielman Date: Mon, 6 Apr 2026 12:21:44 -0400 Subject: [PATCH 25/66] Remove shot queue scrollbar corner workaround --- labscript_utils/qtwidgets/shotqueue.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/labscript_utils/qtwidgets/shotqueue.py b/labscript_utils/qtwidgets/shotqueue.py index 140e3339..f2f01c0b 100644 --- a/labscript_utils/qtwidgets/shotqueue.py +++ b/labscript_utils/qtwidgets/shotqueue.py @@ -160,9 +160,6 @@ def __init__( else QHeaderView.ResizeToContents ) self.queue_view.header().setSectionResizeMode(column, resize_mode) - self.scrollbar_corner = QWidget(self.queue_view) - self.scrollbar_corner.setFixedWidth(self.queue_view.verticalScrollBar().sizeHint().width()) - self.queue_view.setCornerWidget(self.scrollbar_corner) self.add_button = QToolButton(self) self.add_button.setText('Add') From c515061415e3d3f1b268d243b5d94dc34b26967c Mon Sep 17 00:00:00 2001 From: spielman Date: Mon, 6 Apr 2026 12:24:46 -0400 Subject: [PATCH 26/66] Make shot queue default hookups optional --- labscript_utils/qtwidgets/shotqueue.py | 47 +++++--------------------- 1 file changed, 9 insertions(+), 38 deletions(-) diff --git a/labscript_utils/qtwidgets/shotqueue.py b/labscript_utils/qtwidgets/shotqueue.py index f2f01c0b..239dbfa1 100644 --- a/labscript_utils/qtwidgets/shotqueue.py +++ b/labscript_utils/qtwidgets/shotqueue.py @@ -133,6 +133,9 @@ def __init__( column_title='Filepath', column_titles=None, path_column=FILEPATH_COLUMN, + connect_add_button=True, + connect_delete_requested=True, + connect_files_dropped=True, ): QWidget.__init__(self, parent) self.accepted_extensions = _normalise_extensions(accepted_extensions) @@ -174,9 +177,12 @@ def __init__( layout.addWidget(self.queue_view) layout.addLayout(button_layout) - self.add_button.clicked.connect(self.prompt_for_files) - self.queue_view.deleteRequested.connect(self.remove_selected) - self.queue_view.filesDropped.connect(self.add_files) + if connect_add_button: + self.add_button.clicked.connect(self.prompt_for_files) + if connect_delete_requested: + self.queue_view.deleteRequested.connect(self.remove_selected) + if connect_files_dropped: + self.queue_view.filesDropped.connect(self.add_files) self.queue_model.rowsInserted.connect(self._on_queue_changed) self.queue_model.rowsRemoved.connect(self._on_queue_changed) self.queue_model.modelReset.connect(self._on_queue_changed) @@ -255,41 +261,6 @@ def clear(self): if self.queue_model.rowCount(): self.queue_model.removeRows(0, self.queue_model.rowCount()) - def move_up(self): - selected_rows = self.selected_rows() - if not selected_rows or selected_rows[0] == 0: - return - for row_index in selected_rows: - self.queue_model.insertRow(row_index - 1, self.queue_model.takeRow(row_index)) - self._select_rows([row - 1 for row in selected_rows]) - - def move_down(self): - selected_rows = self.selected_rows() - if not selected_rows or selected_rows[-1] == self.queue_model.rowCount() - 1: - return - for row_index in reversed(selected_rows): - self.queue_model.insertRow(row_index + 1, self.queue_model.takeRow(row_index)) - self._select_rows([row + 1 for row in selected_rows]) - - def move_top(self): - selected_rows = self.selected_rows() - if not selected_rows or selected_rows[0] == 0: - return - rows = [self.queue_model.takeRow(row_index) for row_index in reversed(selected_rows)] - for offset, row_items in enumerate(reversed(rows)): - self.queue_model.insertRow(offset, row_items) - self._select_rows(range(len(selected_rows))) - - def move_bottom(self): - selected_rows = self.selected_rows() - if not selected_rows or selected_rows[-1] == self.queue_model.rowCount() - 1: - return - rows = [self.queue_model.takeRow(row_index) for row_index in reversed(selected_rows)] - start_row = self.queue_model.rowCount() - for row_items in reversed(rows): - self.queue_model.appendRow(row_items) - self._select_rows(range(start_row, self.queue_model.rowCount())) - def is_in_queue(self, path): path = os.path.abspath(str(path)) return any( From 65f8ccde64110dd1e5c59a56e76e3e6d91c1173b Mon Sep 17 00:00:00 2001 From: spielman Date: Tue, 7 Apr 2026 11:32:34 -0400 Subject: [PATCH 27/66] Add lookup formatter for runmanager templates --- labscript_utils/lookup_format.py | 147 +++++++++++++++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 labscript_utils/lookup_format.py diff --git a/labscript_utils/lookup_format.py b/labscript_utils/lookup_format.py new file mode 100644 index 00000000..a649cc28 --- /dev/null +++ b/labscript_utils/lookup_format.py @@ -0,0 +1,147 @@ +##################################################################### +# # +# lookup_format.py # +# # +# Copyright 2026, Monash University # +# # +# This file is part of the labscript suite (see # +# http://labscriptsuite.org) and is licensed under the Simplified # +# BSD License. See the license.txt file in the root of the project # +# for the full license. # +# # +##################################################################### +import ast +import string + + +class InvalidLookupFormatField(ValueError): + """Raised when a lookup-format field is not valid lookup-only syntax.""" + + +def format_lookup_string(template, context, dt=None, preserve_unresolved=False): + """Format a template using optional ``strftime`` plus lookup-only fields. + + Args: + template (str): Format template. + context (dict): Top-level lookup context. + dt (datetime, optional): Datetime used for a first ``strftime`` pass. + preserve_unresolved (bool): If True, unresolved lookups are left intact. + """ + if dt is not None: + template = dt.strftime(template) + formatter = _LookupFormatter(context, preserve_unresolved=preserve_unresolved) + return formatter.format(template) + + +class _LookupFormatter(object): + def __init__(self, context, preserve_unresolved=False): + self.context = context + self.preserve_unresolved = preserve_unresolved + self.formatter = string.Formatter() + + def format(self, template): + parts = [] + for literal_text, field_name, format_spec, conversion in self.formatter.parse(template): + parts.append(literal_text) + if field_name is None: + continue + placeholder = self._reconstruct_placeholder( + field_name, conversion, format_spec + ) + try: + value = self._resolve_field(field_name) + if conversion: + value = self.formatter.convert_field(value, conversion) + if format_spec: + format_spec = self.format(format_spec) + parts.append(self.formatter.format_field(value, format_spec)) + except KeyError: + if self.preserve_unresolved: + parts.append(placeholder) + else: + raise + return ''.join(parts) + + def _resolve_field(self, field_name): + root_name, lookups = _parse_lookup_field(field_name) + try: + value = self.context[root_name] + except KeyError: + raise KeyError(root_name) + for lookup in lookups: + try: + value = value[lookup] + except KeyError: + raise + return value + + @staticmethod + def _reconstruct_placeholder(field_name, conversion, format_spec): + placeholder = '{' + field_name + if conversion: + placeholder += '!' + conversion + if format_spec: + placeholder += ':' + format_spec + placeholder += '}' + return placeholder + + +def _parse_lookup_field(field_name): + field_name = str(field_name) + length = len(field_name) + index = 0 + + root_name, index = _parse_identifier(field_name, index) + lookups = [] + + while index < length: + if field_name[index] != '[': + msg = "Invalid lookup-format field %r" % field_name + raise InvalidLookupFormatField(msg) + closing_index = field_name.find(']', index + 1) + if closing_index == -1: + msg = "Invalid lookup-format field %r" % field_name + raise InvalidLookupFormatField(msg) + lookup_text = field_name[index + 1 : closing_index].strip() + if not lookup_text: + msg = "Invalid lookup-format field %r" % field_name + raise InvalidLookupFormatField(msg) + lookups.append(_parse_lookup_key(lookup_text)) + index = closing_index + 1 + + return root_name, lookups + + +def _parse_identifier(field_name, index): + if index >= len(field_name): + msg = "Invalid lookup-format field %r" % field_name + raise InvalidLookupFormatField(msg) + + start = index + first = field_name[index] + if not (first.isalpha() or first == '_'): + msg = "Invalid lookup-format field %r" % field_name + raise InvalidLookupFormatField(msg) + index += 1 + + while index < len(field_name): + char = field_name[index] + if char.isalnum() or char == '_': + index += 1 + continue + break + + return field_name[start:index], index + + +def _parse_lookup_key(lookup_text): + if lookup_text[0] in ('"', "'"): + try: + return ast.literal_eval(lookup_text) + except (SyntaxError, ValueError) as exc: + msg = "Invalid lookup-format key %r" % lookup_text + raise InvalidLookupFormatField(msg) from exc + try: + return int(lookup_text) + except ValueError: + return lookup_text From d0175eb984d3129c575159c7de10d18e4f69c5ca Mon Sep 17 00:00:00 2001 From: spielman Date: Tue, 7 Apr 2026 11:58:15 -0400 Subject: [PATCH 28/66] Tighten lookup format unresolved handling --- labscript_utils/lookup_format.py | 42 ++++++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/labscript_utils/lookup_format.py b/labscript_utils/lookup_format.py index a649cc28..51d1cb73 100644 --- a/labscript_utils/lookup_format.py +++ b/labscript_utils/lookup_format.py @@ -18,7 +18,21 @@ class InvalidLookupFormatField(ValueError): """Raised when a lookup-format field is not valid lookup-only syntax.""" -def format_lookup_string(template, context, dt=None, preserve_unresolved=False): +class _UnresolvedLookupError(LookupError): + """Raised when a lookup could not be resolved from the provided context.""" + + def __init__(self, root_name): + LookupError.__init__(self, root_name) + self.root_name = root_name + + +def format_lookup_string( + template, + context, + dt=None, + preserve_unresolved=False, + preserve_unresolved_roots=(), +): """Format a template using optional ``strftime`` plus lookup-only fields. Args: @@ -26,17 +40,29 @@ def format_lookup_string(template, context, dt=None, preserve_unresolved=False): context (dict): Top-level lookup context. dt (datetime, optional): Datetime used for a first ``strftime`` pass. preserve_unresolved (bool): If True, unresolved lookups are left intact. + preserve_unresolved_roots (iterable): Root names that may be preserved when a + lookup below them is unresolved. """ if dt is not None: template = dt.strftime(template) - formatter = _LookupFormatter(context, preserve_unresolved=preserve_unresolved) + formatter = _LookupFormatter( + context, + preserve_unresolved=preserve_unresolved, + preserve_unresolved_roots=preserve_unresolved_roots, + ) return formatter.format(template) class _LookupFormatter(object): - def __init__(self, context, preserve_unresolved=False): + def __init__( + self, + context, + preserve_unresolved=False, + preserve_unresolved_roots=(), + ): self.context = context self.preserve_unresolved = preserve_unresolved + self.preserve_unresolved_roots = set(preserve_unresolved_roots) self.formatter = string.Formatter() def format(self, template): @@ -55,8 +81,8 @@ def format(self, template): if format_spec: format_spec = self.format(format_spec) parts.append(self.formatter.format_field(value, format_spec)) - except KeyError: - if self.preserve_unresolved: + except _UnresolvedLookupError as exc: + if self.preserve_unresolved or exc.root_name in self.preserve_unresolved_roots: parts.append(placeholder) else: raise @@ -67,12 +93,12 @@ def _resolve_field(self, field_name): try: value = self.context[root_name] except KeyError: - raise KeyError(root_name) + raise _UnresolvedLookupError(root_name) for lookup in lookups: try: value = value[lookup] - except KeyError: - raise + except (IndexError, KeyError, TypeError): + raise _UnresolvedLookupError(root_name) return value @staticmethod From f1d4a9885e731be5a27fca83ba3119d74f1b7796 Mon Sep 17 00:00:00 2001 From: spielman Date: Tue, 7 Apr 2026 13:02:31 -0400 Subject: [PATCH 29/66] Show active config path in window title --- labscript_utils/labconfig.py | 71 ++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/labscript_utils/labconfig.py b/labscript_utils/labconfig.py index d110fe82..72f61eab 100644 --- a/labscript_utils/labconfig.py +++ b/labscript_utils/labconfig.py @@ -13,6 +13,7 @@ import os import configparser from ast import literal_eval +from getpass import getuser from pprint import pformat from pathlib import Path import warnings @@ -23,6 +24,76 @@ default_config_path = default_labconfig_path() +def format_path_for_display(path): + """Return an absolute path with the user's home abbreviated for display.""" + absolute_path = os.path.abspath(os.fspath(path)) + try: + home_path = str(Path("~" + getuser()).expanduser()) + except Exception: + home_path = str(Path.home()) + + normalized_path = os.path.normcase(os.path.normpath(absolute_path)) + normalized_home = os.path.normcase(os.path.normpath(home_path)) + if normalized_path == normalized_home: + return '%USERPROFILE%' if os.name == 'nt' else '~' + + home_prefix = normalized_home + os.path.sep + if normalized_path.startswith(home_prefix): + relative_path = os.path.relpath(absolute_path, home_path) + prefix = '%USERPROFILE%' if os.name == 'nt' else '~' + separator = '\\' if os.name == 'nt' else '/' + return prefix + separator + relative_path.replace(os.path.sep, separator) + + return absolute_path + + +def get_default_appconfig_file( + exp_config, app_name, config_filename, ensure_directory=False +): + try: + default_path = os.path.join(exp_config.get('DEFAULT', 'app_saved_configs'), app_name) + except (LabConfig.NoOptionError, LabConfig.NoSectionError): + exp_config.set( + 'DEFAULT', + 'app_saved_configs', + os.path.join( + '%(labscript_suite)s', 'userlib', 'app_saved_configs', + '%(apparatus_name)s' + ), + ) + default_path = os.path.join( + exp_config.get('DEFAULT', 'app_saved_configs'), app_name + ) + if ensure_directory and not os.path.exists(default_path): + os.makedirs(default_path) + return os.path.join(default_path, config_filename) + + +class LabscriptApplication(object): + app_name = None + default_config_filename = None + + def init_config_window_title(self): + self.base_window_title = self.ui.windowTitle().split(' - ', 1)[0] + + def get_default_config_file(self, ensure_directory=False): + if self.app_name is None or self.default_config_filename is None: + raise NotImplementedError( + 'LabscriptApplication requires app_name and default_config_filename' + ) + return get_default_appconfig_file( + self.exp_config, + self.app_name, + self.default_config_filename, + ensure_directory=ensure_directory, + ) + + def set_config_window_title(self, filename): + self.ui.setWindowTitle( + '{} - {}'.format(self.base_window_title, format_path_for_display(filename)) + ) + + class EnvInterpolation(configparser.BasicInterpolation): """Interpolation which expands environment variables in values, by post-filtering BasicInterpolation.before_get()""" From 465943d1eaae217394043a0f1d44c52d6e56f714 Mon Sep 17 00:00:00 2001 From: spielman Date: Fri, 24 Apr 2026 22:05:46 -0400 Subject: [PATCH 30/66] Add shared plugin framework --- labscript_utils/plugins.py | 333 +++++++++++++++++++++++++++++++++++++ tests/test_plugins.py | 228 +++++++++++++++++++++++++ 2 files changed, 561 insertions(+) create mode 100644 labscript_utils/plugins.py create mode 100644 tests/test_plugins.py diff --git a/labscript_utils/plugins.py b/labscript_utils/plugins.py new file mode 100644 index 00000000..4321ccb0 --- /dev/null +++ b/labscript_utils/plugins.py @@ -0,0 +1,333 @@ +##################################################################### +# # +# plugins.py # +# # +# Copyright 2013, Monash University # +# # +# This file is part of the labscript suite (see # +# http://labscriptsuite.org) and is licensed under the Simplified # +# BSD License. See the license.txt file in the root of the project # +# for the full license. # +# # +##################################################################### +import importlib +import logging +import os +from types import MethodType + + +DEFAULT_PRIORITY = 10 + +__all__ = [ + 'DEFAULT_PRIORITY', + 'Callback', + 'callback', + 'BasePlugin', + 'MenuBuilder', + 'PluginManager', +] + + +class Callback(object): + """Class wrapping a callable. At present only differs from a regular + function in that it has a "priority" attribute - lower numbers means + higher priority. If there are multiple callbacks triggered by the same + event, they will be returned in order of priority by get_callbacks""" + def __init__(self, func, priority=DEFAULT_PRIORITY): + self.priority = priority + self.func = func + + def __get__(self, instance, class_): + """Make sure our callable binds like an instance method. Otherwise + __call__ doesn't get the instance argument.""" + if instance is None: + return self + else: + return MethodType(self, instance) + + def __call__(self, *args, **kwargs): + return self.func(*args, **kwargs) + + +class callback(object): + """Decorator to turn a function into a Callback object. Presently + optional, and only required if the callback needs to have a non-default + priority set""" + # Instantiate the decorator: + def __init__(self, priority=DEFAULT_PRIORITY): + self.priority = priority + + # Call the decorator + def __call__(self, func): + return Callback(func, self.priority) + + +class BasePlugin(object): + """Default no-op implementation of the existing plugin interface.""" + def __init__(self, initial_settings): + self.initial_settings = initial_settings + self.menu = None + self.notifications = {} + + def get_menu_class(self): + return None + + def get_notification_classes(self): + return [] + + def get_setting_classes(self): + return [] + + def get_callbacks(self): + return None + + def get_tab_classes(self): + return {} + + def tabs_created(self, tabs): + pass + + def set_menu_instance(self, menu): + self.menu = menu + + def set_notification_instances(self, notifications): + self.notifications = notifications + + def plugin_setup_complete(self, data=None): + pass + + def get_save_data(self): + return {} + + def close(self): + pass + + +class MenuBuilder(object): + """Build menus from the nested dictionary format used by BLACS plugins.""" + def __init__(self, icon_factory=None): + self.icon_factory = icon_factory + + def create_menu(self, parent, menu_parameters): + if 'name' in menu_parameters: + if 'menu_items' in menu_parameters: + child = parent.addMenu(menu_parameters['name']) + for child_menu_params in menu_parameters['menu_items']: + self.create_menu(child, child_menu_params) + else: + if 'icon' in menu_parameters and self.icon_factory is not None: + child = parent.addAction( + self.icon_factory(menu_parameters['icon']), + menu_parameters['name'], + ) + else: + child = parent.addAction(menu_parameters['name']) + + if 'action' in menu_parameters: + child.triggered.connect(menu_parameters['action']) + + elif 'separator' in menu_parameters: + parent.addSeparator() + + +class PluginManager(object): + """Manage discovery and setup of plugins using the existing plugin API.""" + def __init__( + self, + plugin_package, + plugins_dir, + config, + config_section, + default_plugins=(), + logger=None, + ): + self.plugin_package = plugin_package + self.plugins_dir = plugins_dir + self.config = config + self.config_section = config_section + self.default_plugins = set(default_plugins) + self.logger = logger or logging.getLogger(__name__) + self.modules = {} + self.plugins = {} + + def discover_modules(self): + if not self.config.has_section(self.config_section): + self.config.add_section(self.config_section) + + configured_plugins = set( + name for name, val in self.config.items(self.config_section) + ) + + modules = {} + for module_name in os.listdir(self.plugins_dir): + module_path = os.path.join(self.plugins_dir, module_name) + if not os.path.isdir(module_path) or module_name == '__pycache__': + continue + + # If this is a new plugin, add it to the config. + if module_name not in configured_plugins: + self.config.set( + self.config_section, + module_name, + str(module_name in self.default_plugins), + ) + configured_plugins.add(module_name) + + # Only load activated plugins. + if self.config.getboolean(self.config_section, module_name): + try: + module = importlib.import_module( + self.plugin_package + '.' + module_name + ) + except Exception: + self.logger.exception( + "Could not import plugin '%s'. Skipping." % module_name + ) + else: + modules[module_name] = module + + self.modules = modules + return modules + + def instantiate_plugins(self, plugin_settings=None): + if plugin_settings is None: + plugin_settings = {} + + plugins = {} + for module_name, module in self.modules.items(): + try: + plugins[module_name] = module.Plugin( + plugin_settings[module_name] + if module_name in plugin_settings else {} + ) + except Exception: + self.logger.exception( + "Could not instantiate plugin '%s'. Skipping" % module_name + ) + + self.plugins = plugins + return plugins + + def create_tabs(self, tab_widget, settings_dict, tablist, settings, tab_data): + for module_name, plugin in self.plugins.items(): + try: + if hasattr(plugin, 'get_tab_classes'): + tab_dict = {} + + for tab_name, TabClass in plugin.get_tab_classes().items(): + settings_key = "{}: {}".format(module_name, tab_name) + settings_dict.setdefault(settings_key, {"tab_name": tab_name}) + settings_dict[settings_key]["front_panel_settings"] = ( + settings[settings_key] if settings_key in settings else {} + ) + settings_dict[settings_key]["saved_data"] = ( + tab_data[settings_key]['data'] + if settings_key in tab_data else {} + ) + + tablist[settings_key] = TabClass( + tab_widget, + settings_dict[settings_key], + ) + tab_dict[tab_name] = tablist[settings_key] + + if hasattr(plugin, 'tabs_created'): + plugin.tabs_created(tab_dict) + + except Exception: + self.logger.exception( + "Could not instantiate tab for plugin '%s'. Skipping" + % module_name + ) + + def setup_plugins(self, data, notifications, menu_builder, menubar): + settings_pages = [] + settings_callbacks = [] + + for module_name, plugin in self.plugins.items(): + try: + # Setup settings page. + settings_pages.extend(plugin.get_setting_classes()) + + # Setup menu. + if plugin.get_menu_class(): + # Must store a reference or else the methods called when + # the menu actions are triggered will be garbage collected. + menu = plugin.get_menu_class()(data) + menu_builder.create_menu(menubar, menu.get_menu_items()) + plugin.set_menu_instance(menu) + + # Setup notifications. + plugin_notifications = {} + for notification_class in plugin.get_notification_classes(): + notifications.add_notification(notification_class) + plugin_notifications[notification_class] = ( + notifications.get_instance(notification_class) + ) + plugin.set_notification_instances(plugin_notifications) + + # Register callbacks. + callbacks = plugin.get_callbacks() + # Save the settings_changed callback in a separate list for + # setting up later. + if isinstance(callbacks, dict) and 'settings_changed' in callbacks: + settings_callbacks.append(callbacks['settings_changed']) + + except Exception: + self.logger.exception( + "Plugin '%s' error. Plugin may not be functional." % module_name + ) + + return settings_pages, settings_callbacks + + def setup_complete(self, data): + for module_name, plugin in self.plugins.items(): + try: + plugin.plugin_setup_complete(data) + except Exception: + self.logger.exception( + "Error in plugin_setup_complete() for plugin '%s'. " + "Trying again with old call signature..." % module_name + ) + # Backwards compatibility for old plugins. + try: + plugin.plugin_setup_complete() + self.logger.warning( + "Plugin '%s' using old API. Please update " + "Plugin.plugin_setup_complete method to accept a " + "dictionary of blacs_data as the only argument." + % module_name + ) + except Exception: + self.logger.exception( + "Plugin '%s' error. Plugin may not be functional." + % module_name + ) + + def get_callbacks(self, name): + """Return all callbacks for a particular name, in priority order.""" + callbacks = [] + + for plugin in self.plugins.values(): + try: + plugin_callbacks = plugin.get_callbacks() + if plugin_callbacks is not None: + if name in plugin_callbacks: + callbacks.append(plugin_callbacks[name]) + except Exception: + self.logger.exception('Error getting callbacks from %s.' % str(plugin)) + + callbacks.sort( + key=lambda callback: getattr(callback, 'priority', DEFAULT_PRIORITY) + ) + return callbacks + + def close_plugins(self): + for module_name, plugin in self.plugins.items(): + try: + plugin.close() + except Exception as e: + self.logger.error( + 'Could not close plugin %s. Error was: %s' + % (module_name, str(e)) + ) diff --git a/tests/test_plugins.py b/tests/test_plugins.py new file mode 100644 index 00000000..026057b1 --- /dev/null +++ b/tests/test_plugins.py @@ -0,0 +1,228 @@ +import logging +import sys +import uuid +from pathlib import Path +from types import SimpleNamespace + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from labscript_utils.plugins import ( + BasePlugin, + Callback, + MenuBuilder, + PluginManager, + callback, +) + + +class FakeConfig(object): + def __init__(self): + self.sections = {} + + def has_section(self, name): + return name in self.sections + + def add_section(self, name): + self.sections[name] = {} + + def items(self, name): + return list(self.sections[name].items()) + + def set(self, section, option, value): + self.sections[section][option] = value + + def getboolean(self, section, option): + return self.sections[section][option].lower() == 'true' + + +def test_callback_binds_as_method_and_keeps_priority(): + class Example(object): + @callback(priority=5) + def method(self, value): + return self, value + + example = Example() + + assert isinstance(Example.__dict__['method'], Callback) + assert Example.__dict__['method'].priority == 5 + assert example.method('value') == (example, 'value') + + +def test_base_plugin_defaults_are_no_ops(): + plugin = BasePlugin({'x': 1}) + + assert plugin.initial_settings == {'x': 1} + assert plugin.get_menu_class() is None + assert plugin.get_notification_classes() == [] + assert plugin.get_setting_classes() == [] + assert plugin.get_callbacks() is None + assert plugin.get_tab_classes() == {} + assert plugin.get_save_data() == {} + + +def make_plugin_package(tmp_path, modules): + package = 'plugin_test_' + uuid.uuid4().hex + package_dir = tmp_path / package + plugins_dir = package_dir / 'plugins' + plugins_dir.mkdir(parents=True) + (package_dir / '__init__.py').write_text('') + (plugins_dir / '__init__.py').write_text('') + for name, source in modules.items(): + module_dir = plugins_dir / name + module_dir.mkdir() + (module_dir / '__init__.py').write_text(source) + sys.path.insert(0, str(tmp_path)) + return package + '.plugins', plugins_dir + + +def test_discovery_defaults_config_and_imports_only_enabled_plugins(tmp_path): + plugin_package, plugins_dir = make_plugin_package( + tmp_path, + { + 'enabled': 'class Plugin(object):\n pass\n', + 'disabled': 'raise RuntimeError("should not import")\n', + }, + ) + config = FakeConfig() + manager = PluginManager( + plugin_package, + str(plugins_dir), + config, + 'app/plugins', + ['enabled'], + ) + + modules = manager.discover_modules() + + assert set(modules) == {'enabled'} + assert config.sections['app/plugins']['enabled'] == 'True' + assert config.sections['app/plugins']['disabled'] == 'False' + + +def test_instantiate_plugins_uses_saved_settings(): + class Plugin(object): + def __init__(self, initial_settings): + self.initial_settings = initial_settings + + manager = PluginManager( + 'package.plugins', + 'plugins', + FakeConfig(), + 'app/plugins', + ) + manager.modules = {'plugin': SimpleNamespace(Plugin=Plugin)} + + plugins = manager.instantiate_plugins({'plugin': {'answer': 42}}) + + assert plugins['plugin'].initial_settings == {'answer': 42} + + +def test_get_callbacks_sorts_by_priority_and_logs_errors(caplog): + class Slow(object): + def get_callbacks(self): + return {'event': self.on_event} + + @callback(priority=20) + def on_event(self): + pass + + class Fast(object): + def get_callbacks(self): + return {'event': self.on_event} + + @callback(priority=5) + def on_event(self): + pass + + class Broken(object): + def get_callbacks(self): + raise RuntimeError('broken') + + logger = logging.getLogger('test.plugins') + manager = PluginManager( + 'package.plugins', + 'plugins', + FakeConfig(), + 'app/plugins', + logger=logger, + ) + manager.plugins = { + 'slow': Slow(), + 'broken': Broken(), + 'fast': Fast(), + } + + with caplog.at_level(logging.ERROR, logger='test.plugins'): + callbacks = manager.get_callbacks('event') + + assert [callback.priority for callback in callbacks] == [5, 20] + assert 'Error getting callbacks from' in caplog.text + + +class FakeSignal(object): + def __init__(self): + self.callbacks = [] + + def connect(self, callback): + self.callbacks.append(callback) + + +class FakeAction(object): + def __init__(self, name, icon=None): + self.name = name + self.icon = icon + self.triggered = FakeSignal() + + +class FakeMenu(object): + def __init__(self, name='root'): + self.name = name + self.items = [] + + def addMenu(self, name): + menu = FakeMenu(name) + self.items.append(('menu', menu)) + return menu + + def addAction(self, *args): + if len(args) == 1: + action = FakeAction(args[0]) + else: + action = FakeAction(args[1], icon=args[0]) + self.items.append(('action', action)) + return action + + def addSeparator(self): + self.items.append(('separator', None)) + + +def test_menu_builder_creates_nested_menus_icons_actions_and_separators(): + root = FakeMenu() + called = [] + + def action(): + called.append(True) + + MenuBuilder(icon_factory=lambda name: 'icon:' + name).create_menu( + root, + { + 'name': 'Top', + 'menu_items': [ + {'name': 'Plain', 'action': action}, + {'separator': True}, + {'name': 'Icon', 'icon': 'resource', 'action': action}, + ], + }, + ) + + top = root.items[0][1] + plain = top.items[0][1] + icon = top.items[2][1] + + assert root.items[0][0] == 'menu' + assert plain.name == 'Plain' + assert plain.triggered.callbacks == [action] + assert top.items[1] == ('separator', None) + assert icon.name == 'Icon' + assert icon.icon == 'icon:resource' + assert icon.triggered.callbacks == [action] From 3cef7b3da44d406b0534e6aefbab978535ab4315 Mon Sep 17 00:00:00 2001 From: spielman Date: Sat, 25 Apr 2026 12:10:49 -0400 Subject: [PATCH 31/66] More generalized plugins. --- labscript_utils/plugins.py | 1030 ++++++++++++++++++++++++++++++++++-- tests/test_plugins.py | 216 +++++++- 2 files changed, 1195 insertions(+), 51 deletions(-) diff --git a/labscript_utils/plugins.py b/labscript_utils/plugins.py index 4321ccb0..982b32e2 100644 --- a/labscript_utils/plugins.py +++ b/labscript_utils/plugins.py @@ -10,6 +10,581 @@ # for the full license. # # # ##################################################################### + +"""Plugin discovery, instantiation, and callback plumbing for labscript apps. + +This module is the small framework layer that labscript-suite applications use +to discover plugins, instantiate them, collect app-neutral contributions, and +ask them for callbacks at runtime. The application owns the actual settings +storage, notification manager, menu objects, UI containers, and event firing. +``PluginManager`` only calls plugin hooks and routes contributions to contexts +the application registered explicitly. + +The usual package layout is:: + + myapp/ + plugins/ + __init__.py + example/ + __init__.py # defines class Plugin + +The module name becomes the plugin name. ``PluginManager`` looks for a module +attribute named ``Plugin`` and passes the saved per-plugin settings dictionary +to ``Plugin(initial_settings)``. Existing plugins do not have to inherit +``BasePlugin``; they only need to provide the same methods by duck typing. + +Configuration behavior is intentionally conservative: + +* Missing plugin entries are added to the config when discovered. +* Names listed in ``default_plugins`` are written as enabled by default. +* Only enabled modules are imported. +* Import or instantiation failures are logged and skipped so one broken plugin + does not stop the application from starting. + +``Callback`` and ``callback`` wrap event handlers with priority metadata. +Lower priority numbers run first. ``Callback`` behaves like a descriptor, so +decorated methods still bind to plugin instances normally when accessed through +an instance. + +Two menu paths are supported, intentionally with different contracts: + +* ``MenuBuilder`` is the legacy nested-dictionary renderer used by existing + BLACS-style plugins. It consumes dictionaries with ``name``, ``menu_items``, + ``icon``, ``action``, and ``separator`` keys and renders them immediately + into application menu objects. +* ``MenuContext`` is the shared contribution path for future applications. It + collects actions with explicit ``location``, ``path``, ``group``, ``order``, + ``name``, and ``action`` metadata, then renders all plugins together after + contribution routing is complete. + +``labscript_utils.plugins`` does not know whether a plugin UI belongs in a +tab, MDI subwindow, dock widget, dialog, fixed frame, or plugin-owned modal +container. Plugins declare a named ``context`` and applications register +objects that implement ``add(plugin_name, contribution, data)``. That is the +full shared UI contract. + +Minimal application integration example +--------------------------------------- + +The application owns plugin settings storage, concrete UI objects, and context +implementations. A compact integration with legacy settings/notifications plus +new contribution routing looks like this:: + + from labscript_utils.plugins import MenuBuilder, MenuContext, PluginManager + + + class App(object): + def __init__(self, config, logger): + self.config = config + self.logger = logger + self.plugin_manager = PluginManager( + plugin_package='myapp.plugins', + plugins_dir='/path/to/myapp/plugins', + config=self.config, + config_section='myapp/plugins', + default_plugins=('example',), + logger=self.logger, + ) + + self.plugin_manager.discover_modules() + + plugin_settings = self.load_plugin_settings() + self.plugin_manager.instantiate_plugins(plugin_settings) + + menu_builder = MenuBuilder(icon_factory=self.icon_factory) + settings_pages, settings_callbacks = self.plugin_manager.setup_plugins( + data=self.app_data, + notifications=self.notification_manager, + menu_builder=menu_builder, + menubar=self.menubar, + ) + + menus = MenuContext(icon_factory=self.icon_factory, logger=self.logger) + menus.register_location('file', self.file_menu) + menus.register_location('window', self.window_menu) + + self.plugin_manager.register_context('menus', menus) + self.plugin_manager.register_context('mdi', MDIContext(self.mdi_area)) + self.plugin_manager.register_context( + 'dialogs', + DialogContext(parent=self.main_window), + ) + self.plugin_manager.setup_contexts(self.app_data) + menus.render() + + self.settings_widget = self.build_settings_ui(settings_pages) + for callback in settings_callbacks: + self.register_settings_callback(callback) + + self.plugin_manager.setup_complete(self.app_data) + + def load_plugin_settings(self): + # App-owned storage. Return a mapping from plugin name to dict. + return { + 'example': {'enabled': True}, + } + + def build_settings_ui(self, settings_pages): + # App-owned settings UI. Each item in settings_pages is a plugin + # contributed settings page class. + return settings_pages + + def register_settings_callback(self, callback): + # App-owned settings wiring. + self.settings_changed_callbacks.append(callback) + + def fire_event(self, name): + # App-owned event firing. PluginManager only returns callbacks. + for callback in self.plugin_manager.get_callbacks(name): + callback() + + def close_plugins(self): + self.plugin_manager.close_plugins() + + +Minimal plugin package example +------------------------------ + +This example shows the shared plugin surface. The plugin contributes a legacy +menu, a contribution-based menu action, a settings page, a notification, an MDI +window request, callbacks, ``setup_complete()`` handling, save data, and +shutdown cleanup. Methods can return empty lists or dictionaries when a feature +is not used:: + + from labscript_utils.plugins import BasePlugin, callback + + + class SettingsPage(object): + pass + + + class Notification(object): + pass + + + class DataBrowser(object): + def __init__(self, parent, services): + self.parent = parent + self.services = services + + + class Menu(object): + def __init__(self, data): + self.data = data + + def get_menu_items(self): + return { + 'name': 'Example', + 'menu_items': [ + {'name': 'Legacy Open', 'action': self.open_action}, + {'separator': True}, + { + 'name': 'Legacy Settings', + 'icon': 'gear', + 'action': self.settings_action, + }, + ], + } + + def open_action(self): + pass + + def settings_action(self): + pass + + + class Plugin(BasePlugin): + def __init__(self, initial_settings): + super(Plugin, self).__init__(initial_settings) + self.saved_state = initial_settings + self.app_data = None + + def get_menu_class(self): + return Menu + + def get_menu_contributions(self): + return [ + { + 'location': 'file', + 'path': ('Examples',), + 'group': 'open', + 'order': 20, + 'name': 'Open Data Browser', + 'shortcut': 'Ctrl+D', + 'icon': 'folder-open', + 'action': self.open_data_browser, + }, + ] + + def get_ui_contributions(self): + return [ + { + 'context': 'mdi', + 'key': 'data_browser', + 'title': 'Data Browser', + 'factory': DataBrowser, + }, + ] + + def get_notification_classes(self): + return [Notification] + + def get_setting_classes(self): + return [SettingsPage] + + def get_callbacks(self): + return { + 'settings_changed': self.on_settings_changed, + 'shot_complete': self.on_shot_complete, + } + + def plugin_setup_complete(self, data): + self.app_data = data + + def get_save_data(self): + return {'saved_state': self.saved_state} + + def close(self): + pass + + def open_data_browser(self): + pass + + @callback(priority=5) + def on_settings_changed(self): + pass + + @callback(priority=20) + def on_shot_complete(self): + pass + + +Feature excerpts and expected UI +-------------------------------- + +BasePlugin subclass with saved ``initial_settings`` and ``get_save_data()`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The constructor receives the per-plugin saved settings dictionary. Store it if +you need to restore state later. ``get_save_data()`` should return the +serializable data you want written back for the next application start:: + + class Plugin(BasePlugin): + def __init__(self, initial_settings): + super(Plugin, self).__init__(initial_settings) + self.initial_settings = initial_settings + + def get_save_data(self): + return { + 'window_geometry': self.window_geometry, + 'last_path': self.last_path, + } + +Settings page contribution skeleton +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``get_setting_classes()`` returns the page classes the application should add +to its settings UI. The application owns the widget construction and storage:: + + class Plugin(BasePlugin): + def get_setting_classes(self): + return [SettingsPage] + + class SettingsPage(object): + pass + +Expected UI result: the application shows one settings page for each returned +class, using the app's own settings shell and persistence rules. + +Legacy menu skeleton with MenuBuilder +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``get_menu_class()`` returns a class whose instance exposes +``get_menu_items()``. The nested dictionaries are translated into the app's +menu objects by ``MenuBuilder``. It expects menus and actions to support +``addMenu()``, ``addAction()``, and ``addSeparator()``:: + + class Plugin(BasePlugin): + def get_menu_class(self): + return Menu + + class Menu(object): + def __init__(self, data): + self.data = data + + def get_menu_items(self): + return { + 'name': 'Example', + 'menu_items': [ + {'name': 'Open', 'action': self.open_action}, + {'separator': True}, + { + 'name': 'Settings', + 'icon': 'gear', + 'action': self.settings_action, + }, + ], + } + + def open_action(self): + pass + + def settings_action(self): + pass + +Expected UI result: the app user sees a top-level menu labeled ``Example``, +with an ``Open`` action, a separator, and a ``Settings`` action. If the app +supplies ``icon_factory``, the ``Settings`` action can also show an icon. + +Contribution menu skeleton with MenuContext +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``get_menu_contributions()`` returns dictionaries routed to the application +``menus`` context. ``MenuContext`` understands these keys: + +* ``location``: app-registered top-level menu id, such as ``file`` or + ``window`` +* ``path``: optional submenu path below that location; defaults to ``()`` +* ``group``: separator group within the same location/path; defaults to + ``None`` +* ``order``: numeric order within the group; defaults to ``DEFAULT_PRIORITY`` +* ``name``: action text +* ``action``: callable connected to the action's ``triggered`` signal +* ``shortcut``, ``icon``, ``checkable``, and ``enabled``: optional action + properties applied when the concrete action object supports them + +File-dialog-style plugin example:: + + class FileDialogsPlugin(BasePlugin): + def __init__(self, initial_settings): + super(FileDialogsPlugin, self).__init__(initial_settings) + self.services = None + + def plugin_setup_complete(self, data): + self.services = data['project_services'] + + def get_menu_contributions(self): + return [ + { + 'location': 'file', + 'group': 'project', + 'order': 10, + 'name': 'New Project', + 'shortcut': 'Ctrl+N', + 'action': self.new_project, + }, + { + 'location': 'file', + 'group': 'project', + 'order': 20, + 'name': 'Open Project...', + 'shortcut': 'Ctrl+O', + 'action': self.open_project, + }, + { + 'location': 'file', + 'group': 'save', + 'order': 10, + 'name': 'Save', + 'shortcut': 'Ctrl+S', + 'action': self.save_project, + }, + { + 'location': 'file', + 'group': 'save', + 'order': 20, + 'name': 'Save As...', + 'shortcut': 'Ctrl+Shift+S', + 'action': self.save_project_as, + }, + { + 'location': 'file', + 'group': 'application', + 'order': 100, + 'name': 'Quit', + 'shortcut': 'Ctrl+Q', + 'action': self.quit_application, + }, + ] + + def new_project(self): + self.services.new_project_dialog.open() + + def open_project(self): + self.services.load_project_dialog.open() + + def save_project(self): + self.services.save_project_command.execute() + + def save_project_as(self): + self.services.save_as_project_dialog.open() + + def quit_application(self): + self.services.quit_command.execute() + +Expected UI result: the application renders the actions in its registered +``file`` menu, creates separators between the ``project``, ``save``, and +``application`` groups, and applies shortcuts without the shared framework +knowing what a project is. + +App-neutral UI homing with contexts +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``get_ui_contributions()`` returns dictionaries with a required ``context``. +The rest of the dictionary is application-defined. This is enough to support +known containers, deferred containers, and plugin-provided containers without +adding type-specific methods such as ``create_tabs()`` or +``create_mdi_windows()`` to ``PluginManager``:: + + class KnownContainerContext(object): + def __init__(self, tab_widget): + self.tab_widget = tab_widget + + def add(self, plugin_name, contribution, data): + widget = contribution['factory']( + self.tab_widget, + data['settings'][contribution['key']], + ) + self.tab_widget.addTab(widget, contribution['title']) + + + class DeferredMDIContext(object): + def __init__(self, mdi_area): + self.mdi_area = mdi_area + self.openers = {} + + def add(self, plugin_name, contribution, data): + key = contribution['key'] + + def open_window(): + widget = contribution['factory'](parent=self.mdi_area, data=data) + self.mdi_area.addSubWindow(widget) + widget.show() + return widget + + self.openers[key] = open_window + + + class DialogContext(object): + def __init__(self, parent): + self.parent = parent + self.dialogs = {} + + def add(self, plugin_name, contribution, data): + self.dialogs[contribution['key']] = contribution['factory']( + parent=self.parent, + services=data['services'], + ) + + + class Plugin(BasePlugin): + def get_ui_contributions(self): + return [ + { + 'context': 'tabs', + 'key': 'overview', + 'title': 'Overview', + 'factory': OverviewWidget, + }, + { + 'context': 'mdi', + 'key': 'data_browser', + 'title': 'Data Browser', + 'factory': DataBrowser, + }, + { + 'context': 'dialogs', + 'key': 'open_project', + 'title': 'Open Project', + 'factory': OpenProjectDialog, + }, + ] + +Expected UI result: each application-owned context decides how to instantiate, +store, parent, restore, defer, activate, or display its contribution. Unknown +contexts are logged and skipped by ``PluginManager.setup_contexts()``. + +Notification contribution skeleton +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``get_notification_classes()`` returns notification classes. The application +owns the notification manager and decides how the resulting widgets are shown:: + + class Plugin(BasePlugin): + def get_notification_classes(self): + return [Notification] + + class Notification(object): + pass + +Expected UI result: the application creates whatever notification widgets or +panels it normally uses for those classes and inserts them into its own +notification area. + +Callbacks with priorities +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``@callback(priority=...)`` wraps a method in ``Callback`` metadata. Lower +numbers run first when the application asks ``PluginManager.get_callbacks()``:: + + class SlowPlugin(BasePlugin): + def get_callbacks(self): + return { + 'shot_complete': self.on_slow, + } + + @callback(priority=20) + def on_slow(self): + pass + + class FastPlugin(BasePlugin): + def get_callbacks(self): + return { + 'shot_complete': self.on_fast, + } + + @callback(priority=5) + def on_fast(self): + pass + +Expected runtime order: when the application fires ``shot_complete`` and calls +``plugin_manager.get_callbacks('shot_complete')``, the priority 5 callback +runs before the priority 20 callback, even if the plugins were discovered in a +different order. + +``setup_complete()`` for app data +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``setup_complete()`` is called after the application has finished startup. +Use it to access app-owned state that was not available during construction, +such as the main window, settings object, registered services, or concrete +context objects:: + + class Plugin(BasePlugin): + def plugin_setup_complete(self, data): + self.app = data['app'] + self.settings = data['settings'] + self.services = data['services'] + +The ``data`` dictionary is app-defined. Old plugins may still define +``plugin_setup_complete(self)`` without the ``data`` argument, and the manager +keeps that compatibility path. + +Shutdown cleanup with ``close()`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``close()`` is the plugin shutdown hook. Use it to stop timers, threads, +listeners, or any other plugin-owned resources:: + + class Plugin(BasePlugin): + def close(self): + self.worker.stop() + self.worker.wait() + +Expected behavior: the application calls each plugin's ``close()`` method +during shutdown so the plugin can release resources before exit. +""" + import importlib import logging import os @@ -24,22 +599,27 @@ 'callback', 'BasePlugin', 'MenuBuilder', + 'MenuContext', 'PluginManager', ] class Callback(object): - """Class wrapping a callable. At present only differs from a regular - function in that it has a "priority" attribute - lower numbers means - higher priority. If there are multiple callbacks triggered by the same - event, they will be returned in order of priority by get_callbacks""" + """Wrap a callable with priority metadata and method-style binding. + + ``priority`` is used when ``PluginManager.get_callbacks()`` gathers all + callbacks for a named event. Lower numbers run first. + + The descriptor protocol is implemented so that a ``Callback`` stored as a + class attribute binds like an instance method when accessed from a plugin + instance. This keeps decorated methods usable without extra wrapper code. + """ def __init__(self, func, priority=DEFAULT_PRIORITY): self.priority = priority self.func = func def __get__(self, instance, class_): - """Make sure our callable binds like an instance method. Otherwise - __call__ doesn't get the instance argument.""" + """Bind to ``instance`` the same way a normal function descriptor does.""" if instance is None: return self else: @@ -50,9 +630,13 @@ def __call__(self, *args, **kwargs): class callback(object): - """Decorator to turn a function into a Callback object. Presently - optional, and only required if the callback needs to have a non-default - priority set""" + """Decorator that turns a function into a :class:`Callback`. + + The decorator is optional. A plain method can still be returned from + ``get_callbacks()`` if it already behaves like a callback. Use the + decorator when you want to attach a non-default priority or make the + callback object explicit in the class body. + """ # Instantiate the decorator: def __init__(self, priority=DEFAULT_PRIORITY): self.priority = priority @@ -63,58 +647,149 @@ def __call__(self, func): class BasePlugin(object): - """Default no-op implementation of the existing plugin interface.""" + """Reference implementation of the plugin interface. + + Subclass this when you want a concrete starting point, but it is not + required. The manager only relies on the methods defined here; a plugin + can also satisfy the contract by duck typing. + + ``initial_settings`` contains the saved configuration for the plugin. + """ def __init__(self, initial_settings): + """Store the plugin's saved settings. + + Applications pass the dictionary returned by the previous + ``get_save_data()`` call, or an empty dictionary for a first run. + Subclasses commonly keep this dictionary and use it during + ``plugin_setup_complete()`` to restore UI state or background state. + """ self.initial_settings = initial_settings self.menu = None self.notifications = {} def get_menu_class(self): + """Return a menu class for this plugin, or ``None``. + + The class is constructed as ``MenuClass(data)`` during + ``PluginManager.setup_plugins()``. Its instance should provide + ``get_menu_items()``, returning the nested menu dictionary consumed by + :class:`MenuBuilder`. + """ return None def get_notification_classes(self): + """Return notification classes contributed by this plugin. + + The application-specific notification manager constructs the + notifications. Instances are later passed back to + ``set_notification_instances()`` in a dictionary keyed by notification + class. + """ return [] def get_setting_classes(self): + """Return settings-page classes contributed by this plugin. + + The application owns the settings UI and storage. The manager only + collects these classes and returns them to the application from + ``setup_plugins()``. + """ return [] def get_callbacks(self): + """Return callbacks keyed by event name, or ``None``. + + A typical return value is ``{'shot_complete': self.on_shot_complete}``. + Values may be plain callables or :class:`Callback` instances created + with the :class:`callback` decorator. The manager sorts callbacks for a + given event by their ``priority`` attribute. + """ return None - def get_tab_classes(self): - return {} + def get_ui_contributions(self): + """Return app-context UI contributions. - def tabs_created(self, tabs): - pass + Return an iterable of dictionaries. Each contribution has a + ``context`` key naming an application-registered context. The + application owns what that context means: a known tab area, a deferred + MDI workspace, a dialog registry, a fixed frame, or another + app-specific host. + """ + return [] + + def get_menu_contributions(self): + """Return shared menu contributions. + + Return an iterable of dictionaries. This is the app-neutral menu path. + Existing BLACS-style plugins can continue using ``get_menu_class()`` + and :class:`MenuBuilder`; new apps can register a ``menus`` context and + route these dictionaries through :class:`MenuContext`. + """ + return [] def set_menu_instance(self, menu): + """Receive the menu instance constructed for this plugin.""" self.menu = menu def set_notification_instances(self, notifications): + """Receive notification instances constructed for this plugin.""" self.notifications = notifications def plugin_setup_complete(self, data=None): + """Run after the application has finished plugin setup. + + ``data`` is an application-defined dictionary. For BLACS it contains + references such as the main UI, experiment config, plugin mapping, and + settings object. Older plugins may define this method without a + ``data`` argument; ``PluginManager.setup_complete()`` keeps that + compatibility behavior. + """ pass def get_save_data(self): + """Return serializable plugin state for the next application start.""" return {} def close(self): + """Clean up resources owned by the plugin during application shutdown.""" pass class MenuBuilder(object): - """Build menus from the nested dictionary format used by BLACS plugins.""" + """Build menus from the nested dictionary format used by plugins. + + Each menu description is a dictionary. Supported keys are: + + * ``name``: text for a menu or action + * ``menu_items``: child menu descriptions, making a submenu + * ``icon``: icon name passed to ``icon_factory`` for actions + * ``action``: callable connected to the action's ``triggered`` signal + * ``separator``: a truthy sentinel that inserts a separator + + ``icon_factory`` is optional and should return a Qt icon object or other + value accepted by ``addAction``. Supplying it keeps this module free of a + direct Qt dependency. + """ def __init__(self, icon_factory=None): + """Create a menu builder. + + Args: + icon_factory (callable, optional): Function called as + ``icon_factory(icon_name)`` before adding an icon-bearing menu + action. Applications using Qt typically pass ``QIcon``. + """ self.icon_factory = icon_factory def create_menu(self, parent, menu_parameters): + """Recursively build menus and actions from a nested menu dictionary.""" if 'name' in menu_parameters: if 'menu_items' in menu_parameters: child = parent.addMenu(menu_parameters['name']) for child_menu_params in menu_parameters['menu_items']: self.create_menu(child, child_menu_params) else: + # ``icon_factory`` stays outside this module so labscript-utils + # does not need to import a Qt binding directly. if 'icon' in menu_parameters and self.icon_factory is not None: child = parent.addAction( self.icon_factory(menu_parameters['icon']), @@ -130,8 +805,160 @@ def create_menu(self, parent, menu_parameters): parent.addSeparator() +class MenuContext(object): + """Collect and render contribution-based menu actions. + + Applications register stable top-level menu locations, such as ``file`` or + ``tools``. Plugins contribute action dictionaries that request a location, + optional submenu path, separator group, order, action text, and callback. + Rendering is deferred until all plugins have contributed so ordering and + group separators can be computed across plugins. + """ + def __init__(self, icon_factory=None, logger=None): + """Create a menu context. + + Args: + icon_factory (callable, optional): Function called as + ``icon_factory(icon_name)`` for icon-bearing actions. + logger (logging.Logger, optional): Logger used for skipped or + malformed menu contributions. + """ + self.icon_factory = icon_factory + self.logger = logger or logging.getLogger(__name__) + self.locations = {} + self.contributions = [] + + def register_location(self, name, menu): + """Register an application-owned menu object as a stable location.""" + self.locations[name] = menu + + def add(self, plugin_name, contribution, data): + """Collect one menu contribution for later rendering. + + ``data`` is accepted for the generic context contract. Menu + contributions already contain their callbacks, so this context does not + need to inspect app data directly. + """ + self.contributions.append((plugin_name, contribution)) + + def render(self): + """Render all collected menu contributions into registered locations.""" + grouped = {} + group_orders = {} + + for plugin_name, contribution in self.contributions: + if not isinstance(contribution, dict): + self.logger.error( + "Menu contribution from plugin '%s' is not a dictionary. " + "Skipping." % plugin_name + ) + continue + if 'location' not in contribution: + self.logger.error( + "Menu contribution from plugin '%s' missing location. " + "Skipping." % plugin_name + ) + continue + if 'name' not in contribution: + self.logger.error( + "Menu contribution from plugin '%s' missing name. " + "Skipping." % plugin_name + ) + continue + + location = contribution['location'] + if location not in self.locations: + self.logger.error( + "Menu contribution from plugin '%s' requested unknown " + "location '%s'. Skipping." % (plugin_name, location) + ) + continue + + path = contribution.get('path', ()) + if isinstance(path, str): + path = (path,) + else: + path = tuple(path) + + key = (location, path) + group = contribution.get('group', None) + if key not in group_orders: + group_orders[key] = {} + if group not in group_orders[key]: + group_orders[key][group] = len(group_orders[key]) + + grouped.setdefault(key, []).append((plugin_name, contribution)) + + menus = {} + for name, menu in self.locations.items(): + menus[(name, ())] = menu + + for key in sorted(grouped): + location, path = key + parent_path = () + for submenu_name in path: + submenu_path = parent_path + (submenu_name,) + submenu_key = (location, submenu_path) + if submenu_key not in menus: + menus[submenu_key] = menus[(location, parent_path)].addMenu( + submenu_name + ) + parent_path = submenu_path + menu = menus[(location, path)] + + contributions = sorted( + grouped[key], + key=lambda item: ( + group_orders[key][item[1].get('group', None)], + item[1].get('order', DEFAULT_PRIORITY), + item[0], + item[1]['name'], + ), + ) + + previous_group = None + for index, (plugin_name, contribution) in enumerate(contributions): + group = contribution.get('group', None) + if index and group != previous_group: + menu.addSeparator() + previous_group = group + + name = contribution['name'] + icon = contribution.get('icon', None) + if icon is not None and self.icon_factory is not None: + action = menu.addAction(self.icon_factory(icon), name) + else: + action = menu.addAction(name) + + callback = contribution.get('action', None) + if callback is not None: + action.triggered.connect(callback) + + shortcut = contribution.get('shortcut', None) + if shortcut is not None and hasattr(action, 'setShortcut'): + action.setShortcut(shortcut) + + checkable = contribution.get('checkable', False) + if hasattr(action, 'setCheckable'): + action.setCheckable(checkable) + + enabled = contribution.get('enabled', True) + if hasattr(action, 'setEnabled'): + action.setEnabled(enabled) + + class PluginManager(object): - """Manage discovery and setup of plugins using the existing plugin API.""" + """Manage plugin discovery, lifecycle, and callback lookup. + + The manager keeps two internal mappings: + + * ``modules``: imported plugin modules keyed by plugin name + * ``plugins``: instantiated plugin objects keyed by plugin name + + An application normally creates one manager per plugin directory and uses + it to discover modules, instantiate plugin objects, build UI pieces, and + close plugins on shutdown. + """ def __init__( self, plugin_package, @@ -141,6 +968,21 @@ def __init__( default_plugins=(), logger=None, ): + """Create a plugin manager. + + Args: + plugin_package (str): Import package containing plugin subpackages, + for example ``'myapp.plugins'``. + plugins_dir (str): Filesystem path scanned for plugin directories. + config: Config object with ``has_section()``, ``add_section()``, + ``items()``, ``set()``, and ``getboolean()`` methods. + config_section (str): Section containing plugin enable/disable + options. + default_plugins (iterable): Plugin names enabled by default when + first discovered. + logger (logging.Logger, optional): Logger used for plugin import, + setup, callback, and shutdown errors. + """ self.plugin_package = plugin_package self.plugins_dir = plugins_dir self.config = config @@ -149,8 +991,10 @@ def __init__( self.logger = logger or logging.getLogger(__name__) self.modules = {} self.plugins = {} + self.contexts = {} def discover_modules(self): + """Scan the plugin directory, update config defaults, and import enabled modules.""" if not self.config.has_section(self.config_section): self.config.add_section(self.config_section) @@ -164,7 +1008,7 @@ def discover_modules(self): if not os.path.isdir(module_path) or module_name == '__pycache__': continue - # If this is a new plugin, add it to the config. + # Keep the config in sync with what is present on disk. if module_name not in configured_plugins: self.config.set( self.config_section, @@ -190,6 +1034,12 @@ def discover_modules(self): return modules def instantiate_plugins(self, plugin_settings=None): + """Create ``Plugin`` instances for each imported module. + + ``plugin_settings`` should map plugin names to the saved configuration + dictionary for that plugin. Missing entries fall back to an empty + dictionary. Instantiation failures are logged and skipped. + """ if plugin_settings is None: plugin_settings = {} @@ -208,39 +1058,15 @@ def instantiate_plugins(self, plugin_settings=None): self.plugins = plugins return plugins - def create_tabs(self, tab_widget, settings_dict, tablist, settings, tab_data): - for module_name, plugin in self.plugins.items(): - try: - if hasattr(plugin, 'get_tab_classes'): - tab_dict = {} - - for tab_name, TabClass in plugin.get_tab_classes().items(): - settings_key = "{}: {}".format(module_name, tab_name) - settings_dict.setdefault(settings_key, {"tab_name": tab_name}) - settings_dict[settings_key]["front_panel_settings"] = ( - settings[settings_key] if settings_key in settings else {} - ) - settings_dict[settings_key]["saved_data"] = ( - tab_data[settings_key]['data'] - if settings_key in tab_data else {} - ) - - tablist[settings_key] = TabClass( - tab_widget, - settings_dict[settings_key], - ) - tab_dict[tab_name] = tablist[settings_key] - - if hasattr(plugin, 'tabs_created'): - plugin.tabs_created(tab_dict) - - except Exception: - self.logger.exception( - "Could not instantiate tab for plugin '%s'. Skipping" - % module_name - ) - def setup_plugins(self, data, notifications, menu_builder, menubar): + """Collect plugin UI and startup contributions. + + This is typically called after the application has created its legacy + menu and notification hosts and before it finishes wiring its settings + UI and callback registrations. The method returns a tuple of + ``(settings_pages, settings_callbacks)`` for the application to + consume. + """ settings_pages = [] settings_callbacks = [] @@ -280,7 +1106,110 @@ def setup_plugins(self, data, notifications, menu_builder, menubar): return settings_pages, settings_callbacks + def register_context(self, name, context): + """Register an application-owned plugin contribution context.""" + self.contexts[name] = context + + def _get_contributions(self, plugin, module_name, method_name, label): + """Return a plugin contribution iterable, logging malformed results.""" + if not hasattr(plugin, method_name): + return [] + + try: + contributions = getattr(plugin, method_name)() + except Exception: + self.logger.exception( + "Error getting %s contributions from plugin '%s'. Skipping." + % (label, module_name) + ) + return [] + + if contributions is None: + return [] + + if ( + isinstance(contributions, (dict, str, bytes)) + or not hasattr(contributions, '__iter__') + ): + self.logger.error( + "Plugin '%s' %s contributions must be an iterable of " + "dictionaries. Skipping." % (module_name, label) + ) + return [] + + return contributions + + def setup_contexts(self, data): + """Route plugin-declared UI and menu contributions to app contexts.""" + for module_name, plugin in self.plugins.items(): + ui_contributions = self._get_contributions( + plugin, + module_name, + 'get_ui_contributions', + 'UI', + ) + + for contribution in ui_contributions: + if not isinstance(contribution, dict): + self.logger.error( + "UI contribution from plugin '%s' is not a dictionary. " + "Skipping." % module_name + ) + continue + if 'context' not in contribution: + self.logger.error( + "UI contribution from plugin '%s' missing context. " + "Skipping." % module_name + ) + continue + + context_name = contribution['context'] + if context_name not in self.contexts: + self.logger.error( + "UI contribution from plugin '%s' requested unknown " + "context '%s'. Skipping." % (module_name, context_name) + ) + continue + + try: + self.contexts[context_name].add(module_name, contribution, data) + except Exception: + self.logger.exception( + "Error adding UI contribution from plugin '%s' to " + "context '%s'. Skipping." % (module_name, context_name) + ) + + menu_contributions = self._get_contributions( + plugin, + module_name, + 'get_menu_contributions', + 'menu', + ) + + if menu_contributions and 'menus' not in self.contexts: + self.logger.error( + "Plugin '%s' provided menu contributions, but no 'menus' " + "context is registered. Skipping." % module_name + ) + continue + + for contribution in menu_contributions: + if not isinstance(contribution, dict): + self.logger.error( + "Menu contribution from plugin '%s' is not a " + "dictionary. Skipping." % module_name + ) + continue + try: + self.contexts['menus'].add(module_name, contribution, data) + except Exception: + self.logger.exception( + "Error adding menu contribution from plugin '%s'. " + "Skipping." % module_name + ) + def setup_complete(self, data): + """Notify plugins that the application has finished startup.""" for module_name, plugin in self.plugins.items(): try: plugin.plugin_setup_complete(data) @@ -305,7 +1234,7 @@ def setup_complete(self, data): ) def get_callbacks(self, name): - """Return all callbacks for a particular name, in priority order.""" + """Return all callbacks registered for ``name``, sorted by priority.""" callbacks = [] for plugin in self.plugins.values(): @@ -323,6 +1252,7 @@ def get_callbacks(self, name): return callbacks def close_plugins(self): + """Call ``close()`` on every plugin during application shutdown.""" for module_name, plugin in self.plugins.items(): try: plugin.close() diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 026057b1..a6188a08 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -10,6 +10,7 @@ BasePlugin, Callback, MenuBuilder, + MenuContext, PluginManager, callback, ) @@ -56,7 +57,8 @@ def test_base_plugin_defaults_are_no_ops(): assert plugin.get_notification_classes() == [] assert plugin.get_setting_classes() == [] assert plugin.get_callbacks() is None - assert plugin.get_tab_classes() == {} + assert plugin.get_ui_contributions() == [] + assert plugin.get_menu_contributions() == [] assert plugin.get_save_data() == {} @@ -172,6 +174,18 @@ def __init__(self, name, icon=None): self.name = name self.icon = icon self.triggered = FakeSignal() + self.shortcut = None + self.checkable = None + self.enabled = None + + def setShortcut(self, shortcut): + self.shortcut = shortcut + + def setCheckable(self, checkable): + self.checkable = checkable + + def setEnabled(self, enabled): + self.enabled = enabled class FakeMenu(object): @@ -226,3 +240,203 @@ def action(): assert icon.name == 'Icon' assert icon.icon == 'icon:resource' assert icon.triggered.callbacks == [action] + + +class FakeContext(object): + def __init__(self): + self.items = [] + + def add(self, plugin_name, contribution, data): + self.items.append((plugin_name, contribution, data)) + + +class BrokenContext(object): + def add(self, plugin_name, contribution, data): + raise RuntimeError('broken context') + + +def test_setup_contexts_routes_ui_and_menu_contributions(): + class Plugin(object): + def get_ui_contributions(self): + return [{'context': 'mdi', 'key': 'browser'}] + + def get_menu_contributions(self): + return [{'location': 'file', 'name': 'Open'}] + + mdi_context = FakeContext() + menu_context = FakeContext() + data = object() + manager = PluginManager( + 'package.plugins', + 'plugins', + FakeConfig(), + 'app/plugins', + ) + manager.plugins = {'plugin': Plugin(), 'legacy': object()} + manager.register_context('mdi', mdi_context) + manager.register_context('menus', menu_context) + + manager.setup_contexts(data) + + assert mdi_context.items == [ + ('plugin', {'context': 'mdi', 'key': 'browser'}, data), + ] + assert menu_context.items == [ + ('plugin', {'location': 'file', 'name': 'Open'}, data), + ] + + +def test_setup_contexts_logs_and_skips_missing_unknown_and_broken_contexts(caplog): + class MissingContext(object): + def get_ui_contributions(self): + return [{'key': 'browser'}] + + class UnknownContext(object): + def get_ui_contributions(self): + return [{'context': 'unknown', 'key': 'browser'}] + + class BrokenGetter(object): + def get_ui_contributions(self): + raise RuntimeError('broken getter') + + class MalformedGetter(object): + def get_ui_contributions(self): + return {'context': 'mdi'} + + class MalformedItem(object): + def get_ui_contributions(self): + return ['not a dictionary'] + + class BrokenAdd(object): + def get_ui_contributions(self): + return [{'context': 'broken', 'key': 'browser'}] + + class MenuPlugin(object): + def get_menu_contributions(self): + return [{'location': 'file', 'name': 'Open'}] + + logger = logging.getLogger('test.plugins') + manager = PluginManager( + 'package.plugins', + 'plugins', + FakeConfig(), + 'app/plugins', + logger=logger, + ) + manager.plugins = { + 'missing': MissingContext(), + 'unknown': UnknownContext(), + 'broken_getter': BrokenGetter(), + 'malformed_getter': MalformedGetter(), + 'malformed_item': MalformedItem(), + 'broken_add': BrokenAdd(), + 'menu': MenuPlugin(), + } + manager.register_context('broken', BrokenContext()) + + with caplog.at_level(logging.ERROR, logger='test.plugins'): + manager.setup_contexts(object()) + + log_text = caplog.text.lower() + assert 'missing' in log_text + assert 'unknown' in log_text + assert 'broken_getter' in log_text + assert 'malformed_getter' in log_text + assert 'malformed_item' in log_text + assert 'broken_add' in log_text + assert 'menus' in log_text + + +def test_menu_context_renders_locations_paths_groups_order_and_action_options(caplog): + file_menu = FakeMenu('file') + tools_menu = FakeMenu('tools') + + def open_action(): + pass + + def heal_action(): + pass + + def save_action(): + pass + + logger = logging.getLogger('test.plugins') + context = MenuContext(icon_factory=lambda name: 'icon:' + name, logger=logger) + context.register_location('file', file_menu) + context.register_location('tools', tools_menu) + context.add( + 'plugin_b', + { + 'location': 'file', + 'path': ('Project',), + 'group': 'open', + 'order': 20, + 'name': 'Heal', + 'action': heal_action, + }, + {}, + ) + context.add( + 'plugin_a', + { + 'location': 'file', + 'path': ('Project',), + 'group': 'open', + 'order': 10, + 'name': 'Open', + 'icon': 'folder', + 'shortcut': 'Ctrl+O', + 'checkable': True, + 'enabled': False, + 'action': open_action, + }, + {}, + ) + context.add( + 'plugin_a', + { + 'location': 'file', + 'path': ('Project',), + 'group': 'save', + 'order': 5, + 'name': 'Save', + 'action': save_action, + }, + {}, + ) + context.add('plugin_c', {'location': 'tools', 'name': 'Tool'}, {}) + context.add('plugin_malformed', 'not a dictionary', {}) + context.add('plugin_skip', {'location': 'missing', 'name': 'Skip'}, {}) + + with caplog.at_level(logging.ERROR, logger='test.plugins'): + context.render() + + project_menu = file_menu.items[0][1] + open_item = project_menu.items[0][1] + heal_item = project_menu.items[1][1] + save_item = project_menu.items[3][1] + + assert file_menu.items[0][0] == 'menu' + assert project_menu.name == 'Project' + assert [item[0] for item in project_menu.items] == [ + 'action', + 'action', + 'separator', + 'action', + ] + assert [open_item.name, heal_item.name, save_item.name] == [ + 'Open', + 'Heal', + 'Save', + ] + assert open_item.icon == 'icon:folder' + assert open_item.shortcut == 'Ctrl+O' + assert open_item.checkable is True + assert open_item.enabled is False + assert open_item.triggered.callbacks == [open_action] + assert heal_item.triggered.callbacks == [heal_action] + assert save_item.checkable is False + assert save_item.enabled is True + assert tools_menu.items[0][1].name == 'Tool' + assert 'missing' in caplog.text.lower() + assert 'plugin_malformed' in caplog.text From dd8664f032c49782fb2e2957676312558e9b9731 Mon Sep 17 00:00:00 2001 From: spielman Date: Sun, 26 Apr 2026 12:10:24 -0400 Subject: [PATCH 32/66] Final revision. --- labscript_utils/plugins.py | 164 ++++++++++++++++++++++---- tests/test_plugins.py | 231 +++++++++++++++++++++++++++++++++++-- 2 files changed, 361 insertions(+), 34 deletions(-) diff --git a/labscript_utils/plugins.py b/labscript_utils/plugins.py index 982b32e2..1033a20a 100644 --- a/labscript_utils/plugins.py +++ b/labscript_utils/plugins.py @@ -232,7 +232,7 @@ def get_notification_classes(self): def get_setting_classes(self): return [SettingsPage] - def get_callbacks(self): + def get_event_handlers(self): return { 'settings_changed': self.on_settings_changed, 'shot_complete': self.on_shot_complete, @@ -528,7 +528,7 @@ class Notification(object): numbers run first when the application asks ``PluginManager.get_callbacks()``:: class SlowPlugin(BasePlugin): - def get_callbacks(self): + def get_event_handlers(self): return { 'shot_complete': self.on_slow, } @@ -538,7 +538,7 @@ def on_slow(self): pass class FastPlugin(BasePlugin): - def get_callbacks(self): + def get_event_handlers(self): return { 'shot_complete': self.on_fast, } @@ -588,6 +588,8 @@ def close(self): import importlib import logging import os +import warnings +from collections.abc import Mapping from types import MethodType @@ -604,6 +606,14 @@ def close(self): ] +def _log_once(logger, seen, level, key, message): + """Log ``message`` once for each hashable ``key``.""" + if key in seen: + return + seen.add(key) + logger.log(level, message) + + class Callback(object): """Wrap a callable with priority metadata and method-style binding. @@ -670,10 +680,13 @@ def __init__(self, initial_settings): def get_menu_class(self): """Return a menu class for this plugin, or ``None``. + Deprecated compatibility hook for BLACS-style nested menus. + The class is constructed as ``MenuClass(data)`` during ``PluginManager.setup_plugins()``. Its instance should provide ``get_menu_items()``, returning the nested menu dictionary consumed by - :class:`MenuBuilder`. + :class:`MenuBuilder`. New shared plugin code should prefer + :meth:`get_menu_contributions` instead. """ return None @@ -696,15 +709,30 @@ def get_setting_classes(self): """ return [] + def get_event_handlers(self): + """Return a mapping of event names to handlers, or ``None``. + + This is the preferred shared event hook surface. Return a mapping such + as ``{'shot_complete': self.on_shot_complete}``. Event names are + application-defined strings; this shared layer only collects, + normalizes, and orders handlers. Values may be plain callables or + :class:`Callback` instances created with the :class:`callback` + decorator. The manager sorts handlers for a given event by their + ``priority`` attribute. + """ + return None + def get_callbacks(self): """Return callbacks keyed by event name, or ``None``. - A typical return value is ``{'shot_complete': self.on_shot_complete}``. - Values may be plain callables or :class:`Callback` instances created - with the :class:`callback` decorator. The manager sorts callbacks for a - given event by their ``priority`` attribute. + Deprecated compatibility alias for :meth:`get_event_handlers`. + + Existing BLACS-style plugins may still override this method. New + shared plugin code should implement :meth:`get_event_handlers` + instead. ``BasePlugin`` routes this legacy name to the modern hook so + the preferred surface is the real implementation path. """ - return None + return self.get_event_handlers() def get_ui_contributions(self): """Return app-context UI contributions. @@ -713,7 +741,7 @@ def get_ui_contributions(self): ``context`` key naming an application-registered context. The application owns what that context means: a known tab area, a deferred MDI workspace, a dialog registry, a fixed frame, or another - app-specific host. + app-specific host. The remaining keys are application-defined. """ return [] @@ -723,7 +751,10 @@ def get_menu_contributions(self): Return an iterable of dictionaries. This is the app-neutral menu path. Existing BLACS-style plugins can continue using ``get_menu_class()`` and :class:`MenuBuilder`; new apps can register a ``menus`` context and - route these dictionaries through :class:`MenuContext`. + route these dictionaries through :class:`MenuContext`. Apart from + stable menu keys such as ``location``, ``path``, ``group``, ``order``, + ``name``, and ``action``, the concrete menu objects remain + application-owned. """ return [] @@ -738,7 +769,9 @@ def set_notification_instances(self, notifications): def plugin_setup_complete(self, data=None): """Run after the application has finished plugin setup. - ``data`` is an application-defined dictionary. For BLACS it contains + ``data`` is an application-defined dictionary. Plugins commonly store + references to services or state they will need later when handling + events, building menus, or opening UI. For BLACS it contains references such as the main UI, experiment config, plugin mapping, and settings object. Older plugins may define this method without a ``data`` argument; ``PluginManager.setup_complete()`` keeps that @@ -747,11 +780,20 @@ def plugin_setup_complete(self, data=None): pass def get_save_data(self): - """Return serializable plugin state for the next application start.""" + """Return serializable plugin state for the next application start. + + The shape of this data is application-owned. Return plain data + structures that the application can persist and pass back as + ``initial_settings`` during the next start. + """ return {} def close(self): - """Clean up resources owned by the plugin during application shutdown.""" + """Clean up resources owned by the plugin during application shutdown. + + Applications call this during shutdown. Stop timers, workers, + listeners, or other plugin-owned resources here. + """ pass @@ -992,6 +1034,7 @@ def __init__( self.modules = {} self.plugins = {} self.contexts = {} + self._logged_plugin_warnings = set() def discover_modules(self): """Scan the plugin directory, update config defaults, and import enabled modules.""" @@ -1076,10 +1119,19 @@ def setup_plugins(self, data, notifications, menu_builder, menubar): settings_pages.extend(plugin.get_setting_classes()) # Setup menu. - if plugin.get_menu_class(): + menu_class = plugin.get_menu_class() + if menu_class: + _log_once( + self.logger, + self._logged_plugin_warnings, + logging.WARNING, + (module_name, 'get_menu_class'), + "Plugin '%s' uses deprecated get_menu_class(); " + "prefer get_menu_contributions() instead." % module_name + ) # Must store a reference or else the methods called when # the menu actions are triggered will be garbage collected. - menu = plugin.get_menu_class()(data) + menu = menu_class(data) menu_builder.create_menu(menubar, menu.get_menu_items()) plugin.set_menu_instance(menu) @@ -1093,10 +1145,10 @@ def setup_plugins(self, data, notifications, menu_builder, menubar): plugin.set_notification_instances(plugin_notifications) # Register callbacks. - callbacks = plugin.get_callbacks() + callbacks = self._get_plugin_event_handlers(module_name, plugin) # Save the settings_changed callback in a separate list for # setting up later. - if isinstance(callbacks, dict) and 'settings_changed' in callbacks: + if callbacks and 'settings_changed' in callbacks: settings_callbacks.append(callbacks['settings_changed']) except Exception: @@ -1233,16 +1285,68 @@ def setup_complete(self, data): % module_name ) - def get_callbacks(self, name): - """Return all callbacks registered for ``name``, sorted by priority.""" + def _defines_hook(self, plugin, method_name): + """Return whether ``plugin`` overrides ``method_name`` meaningfully.""" + plugin_method = getattr(type(plugin), method_name, None) + if plugin_method is None: + return False + + base_method = getattr(BasePlugin, method_name, None) + return plugin_method is not base_method + + def _get_plugin_event_handlers(self, module_name, plugin): + """Return normalized event handlers for one plugin.""" + has_modern = self._defines_hook(plugin, 'get_event_handlers') + has_legacy = self._defines_hook(plugin, 'get_callbacks') + + if has_modern: + if has_legacy: + _log_once( + self.logger, + self._logged_plugin_warnings, + logging.WARNING, + (module_name, 'get_callbacks'), + "Plugin '%s' defines deprecated get_callbacks() and " + "preferred get_event_handlers(); ignoring get_callbacks()." + % module_name + ) + handlers = plugin.get_event_handlers() + elif has_legacy: + _log_once( + self.logger, + self._logged_plugin_warnings, + logging.WARNING, + (module_name, 'get_callbacks'), + "Plugin '%s' uses deprecated get_callbacks(); " + "implement get_event_handlers() instead." % module_name + ) + handlers = plugin.get_callbacks() + else: + return None + + if handlers is None: + return None + if not isinstance(handlers, Mapping): + self.logger.error( + "Plugin '%s' event handlers must be a mapping. Skipping." + % module_name + ) + return None + + return handlers + + def get_event_handlers(self, name): + """Return all event handlers registered for ``name``, sorted by priority.""" callbacks = [] - for plugin in self.plugins.values(): + for module_name, plugin in self.plugins.items(): try: - plugin_callbacks = plugin.get_callbacks() - if plugin_callbacks is not None: - if name in plugin_callbacks: - callbacks.append(plugin_callbacks[name]) + plugin_callbacks = self._get_plugin_event_handlers( + module_name, + plugin, + ) + if plugin_callbacks and name in plugin_callbacks: + callbacks.append(plugin_callbacks[name]) except Exception: self.logger.exception('Error getting callbacks from %s.' % str(plugin)) @@ -1251,6 +1355,16 @@ def get_callbacks(self, name): ) return callbacks + def get_callbacks(self, name): + """Deprecated compatibility wrapper for :meth:`get_event_handlers`.""" + warnings.warn( + "PluginManager.get_callbacks() is deprecated; use " + "PluginManager.get_event_handlers() instead.", + DeprecationWarning, + stacklevel=2, + ) + return self.get_event_handlers(name) + def close_plugins(self): """Call ``close()`` on every plugin during application shutdown.""" for module_name, plugin in self.plugins.items(): diff --git a/tests/test_plugins.py b/tests/test_plugins.py index a6188a08..47fad04a 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -1,6 +1,7 @@ import logging import sys import uuid +import warnings from pathlib import Path from types import SimpleNamespace @@ -56,6 +57,7 @@ def test_base_plugin_defaults_are_no_ops(): assert plugin.get_menu_class() is None assert plugin.get_notification_classes() == [] assert plugin.get_setting_classes() == [] + assert plugin.get_event_handlers() is None assert plugin.get_callbacks() is None assert plugin.get_ui_contributions() == [] assert plugin.get_menu_contributions() == [] @@ -77,6 +79,24 @@ def make_plugin_package(tmp_path, modules): return package + '.plugins', plugins_dir +def make_manager(logger=None): + return PluginManager( + 'package.plugins', + 'plugins', + FakeConfig(), + 'app/plugins', + logger=logger, + ) + + +class NoNotifications(object): + def add_notification(self, notification_class): + raise AssertionError('no notifications expected') + + def get_instance(self, notification_class): + raise AssertionError('no notifications expected') + + def test_discovery_defaults_config_and_imports_only_enabled_plugins(tmp_path): plugin_package, plugins_dir = make_plugin_package( tmp_path, @@ -119,7 +139,7 @@ def __init__(self, initial_settings): assert plugins['plugin'].initial_settings == {'answer': 42} -def test_get_callbacks_sorts_by_priority_and_logs_errors(caplog): +def test_get_event_handlers_sorts_legacy_callbacks_by_priority_and_logs_errors(caplog): class Slow(object): def get_callbacks(self): return {'event': self.on_event} @@ -141,13 +161,7 @@ def get_callbacks(self): raise RuntimeError('broken') logger = logging.getLogger('test.plugins') - manager = PluginManager( - 'package.plugins', - 'plugins', - FakeConfig(), - 'app/plugins', - logger=logger, - ) + manager = make_manager(logger=logger) manager.plugins = { 'slow': Slow(), 'broken': Broken(), @@ -155,12 +169,211 @@ def get_callbacks(self): } with caplog.at_level(logging.ERROR, logger='test.plugins'): - callbacks = manager.get_callbacks('event') + callbacks = manager.get_event_handlers('event') assert [callback.priority for callback in callbacks] == [5, 20] assert 'Error getting callbacks from' in caplog.text +def test_get_event_handlers_sorts_modern_handlers_by_priority(): + class Slow(object): + def get_event_handlers(self): + return {'event': self.on_event} + + @callback(priority=20) + def on_event(self): + pass + + class Fast(object): + def get_event_handlers(self): + return {'event': self.on_event} + + @callback(priority=5) + def on_event(self): + pass + + manager = make_manager() + manager.plugins = { + 'slow': Slow(), + 'fast': Fast(), + } + + callbacks = manager.get_event_handlers('event') + + assert [callback.priority for callback in callbacks] == [5, 20] + + +def test_get_callbacks_warns_and_delegates_to_get_event_handlers(): + class Plugin(object): + def get_event_handlers(self): + return {'event': self.on_event} + + @callback(priority=5) + def on_event(self): + pass + + manager = make_manager() + manager.plugins = {'plugin': Plugin()} + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter('always', DeprecationWarning) + callbacks = manager.get_callbacks('event') + + assert [callback.priority for callback in callbacks] == [5] + assert len(caught) == 1 + assert 'PluginManager.get_callbacks() is deprecated' in str(caught[0].message) + + +def test_get_event_handlers_prefers_modern_hook_and_warns_for_legacy_one(caplog): + class ModernAndLegacy(object): + def get_event_handlers(self): + return {'event': self.new_event} + + def get_callbacks(self): + return {'event': self.old_event} + + @callback(priority=5) + def new_event(self): + pass + + @callback(priority=20) + def old_event(self): + pass + + class LegacyOnly(object): + def get_callbacks(self): + return {'event': self.old_event} + + @callback(priority=10) + def old_event(self): + pass + + logger = logging.getLogger('test.plugins') + manager = make_manager(logger=logger) + manager.plugins = { + 'modern_and_legacy': ModernAndLegacy(), + 'legacy_only': LegacyOnly(), + } + + with caplog.at_level(logging.WARNING, logger='test.plugins'): + first_callbacks = manager.get_event_handlers('event') + second_callbacks = manager.get_event_handlers('event') + + assert [callback.priority for callback in first_callbacks] == [5, 10] + assert [callback.priority for callback in second_callbacks] == [5, 10] + warning_messages = [ + record.getMessage() + for record in caplog.records + if record.levelno == logging.WARNING + ] + assert sum('modern_and_legacy' in message for message in warning_messages) == 1 + assert sum('legacy_only' in message for message in warning_messages) == 1 + assert sum('get_callbacks()' in message for message in warning_messages) == 2 + + +def test_setup_plugins_prefers_modern_settings_callback_and_warns(caplog): + class Plugin(BasePlugin): + def get_event_handlers(self): + return {'settings_changed': self.new_settings_changed} + + def get_callbacks(self): + return {'settings_changed': self.old_settings_changed} + + @callback(priority=5) + def new_settings_changed(self): + pass + + @callback(priority=20) + def old_settings_changed(self): + pass + + logger = logging.getLogger('test.plugins') + manager = make_manager(logger=logger) + manager.plugins = {'plugin': Plugin({})} + + with caplog.at_level(logging.WARNING, logger='test.plugins'): + settings_pages, settings_callbacks = manager.setup_plugins( + data={}, + notifications=NoNotifications(), + menu_builder=MenuBuilder(), + menubar=FakeMenu(), + ) + callbacks = manager.get_event_handlers('settings_changed') + + assert settings_pages == [] + assert [callback.priority for callback in settings_callbacks] == [5] + assert [callback.priority for callback in callbacks] == [5] + warning_messages = [ + record.getMessage() + for record in caplog.records + if record.levelno == logging.WARNING + ] + assert sum('deprecated get_callbacks()' in message for message in warning_messages) == 1 + + +def test_setup_plugins_warns_when_deprecated_menu_class_is_used(caplog): + class Menu(object): + def __init__(self, data): + self.data = data + + def get_menu_items(self): + return {'name': 'Legacy'} + + class Plugin(BasePlugin): + def get_menu_class(self): + return Menu + + logger = logging.getLogger('test.plugins') + manager = make_manager(logger=logger) + manager.plugins = {'plugin': Plugin({})} + + with caplog.at_level(logging.WARNING, logger='test.plugins'): + manager.setup_plugins( + data={'example': True}, + notifications=NoNotifications(), + menu_builder=MenuBuilder(), + menubar=FakeMenu(), + ) + manager.setup_plugins( + data={'example': True}, + notifications=NoNotifications(), + menu_builder=MenuBuilder(), + menubar=FakeMenu(), + ) + + warning_messages = [ + record.getMessage() + for record in caplog.records + if record.levelno == logging.WARNING + ] + assert sum('deprecated get_menu_class()' in message for message in warning_messages) == 1 + + +def test_get_event_handlers_logs_and_skips_malformed_event_handler_mappings(caplog): + class BadModern(object): + def get_event_handlers(self): + return ['event'] + + class BadLegacy(object): + def get_callbacks(self): + return ['event'] + + logger = logging.getLogger('test.plugins') + manager = make_manager(logger=logger) + manager.plugins = { + 'bad_modern': BadModern(), + 'bad_legacy': BadLegacy(), + } + + with caplog.at_level(logging.ERROR, logger='test.plugins'): + callbacks = manager.get_event_handlers('event') + + assert callbacks == [] + assert "bad_modern" in caplog.text + assert "bad_legacy" in caplog.text + assert "event handlers must be a mapping" in caplog.text + + class FakeSignal(object): def __init__(self): self.callbacks = [] From 83f44ff3f49a111a1d65117e503afd81326f470e Mon Sep 17 00:00:00 2001 From: spielman Date: Sun, 26 Apr 2026 12:19:27 -0400 Subject: [PATCH 33/66] Add shared plugin framework to labscript-utils --- labscript_utils/plugins.py | 1377 ++++++++++++++++++++++++++++++++++++ 1 file changed, 1377 insertions(+) create mode 100644 labscript_utils/plugins.py diff --git a/labscript_utils/plugins.py b/labscript_utils/plugins.py new file mode 100644 index 00000000..1033a20a --- /dev/null +++ b/labscript_utils/plugins.py @@ -0,0 +1,1377 @@ +##################################################################### +# # +# plugins.py # +# # +# Copyright 2013, Monash University # +# # +# This file is part of the labscript suite (see # +# http://labscriptsuite.org) and is licensed under the Simplified # +# BSD License. See the license.txt file in the root of the project # +# for the full license. # +# # +##################################################################### + +"""Plugin discovery, instantiation, and callback plumbing for labscript apps. + +This module is the small framework layer that labscript-suite applications use +to discover plugins, instantiate them, collect app-neutral contributions, and +ask them for callbacks at runtime. The application owns the actual settings +storage, notification manager, menu objects, UI containers, and event firing. +``PluginManager`` only calls plugin hooks and routes contributions to contexts +the application registered explicitly. + +The usual package layout is:: + + myapp/ + plugins/ + __init__.py + example/ + __init__.py # defines class Plugin + +The module name becomes the plugin name. ``PluginManager`` looks for a module +attribute named ``Plugin`` and passes the saved per-plugin settings dictionary +to ``Plugin(initial_settings)``. Existing plugins do not have to inherit +``BasePlugin``; they only need to provide the same methods by duck typing. + +Configuration behavior is intentionally conservative: + +* Missing plugin entries are added to the config when discovered. +* Names listed in ``default_plugins`` are written as enabled by default. +* Only enabled modules are imported. +* Import or instantiation failures are logged and skipped so one broken plugin + does not stop the application from starting. + +``Callback`` and ``callback`` wrap event handlers with priority metadata. +Lower priority numbers run first. ``Callback`` behaves like a descriptor, so +decorated methods still bind to plugin instances normally when accessed through +an instance. + +Two menu paths are supported, intentionally with different contracts: + +* ``MenuBuilder`` is the legacy nested-dictionary renderer used by existing + BLACS-style plugins. It consumes dictionaries with ``name``, ``menu_items``, + ``icon``, ``action``, and ``separator`` keys and renders them immediately + into application menu objects. +* ``MenuContext`` is the shared contribution path for future applications. It + collects actions with explicit ``location``, ``path``, ``group``, ``order``, + ``name``, and ``action`` metadata, then renders all plugins together after + contribution routing is complete. + +``labscript_utils.plugins`` does not know whether a plugin UI belongs in a +tab, MDI subwindow, dock widget, dialog, fixed frame, or plugin-owned modal +container. Plugins declare a named ``context`` and applications register +objects that implement ``add(plugin_name, contribution, data)``. That is the +full shared UI contract. + +Minimal application integration example +--------------------------------------- + +The application owns plugin settings storage, concrete UI objects, and context +implementations. A compact integration with legacy settings/notifications plus +new contribution routing looks like this:: + + from labscript_utils.plugins import MenuBuilder, MenuContext, PluginManager + + + class App(object): + def __init__(self, config, logger): + self.config = config + self.logger = logger + self.plugin_manager = PluginManager( + plugin_package='myapp.plugins', + plugins_dir='/path/to/myapp/plugins', + config=self.config, + config_section='myapp/plugins', + default_plugins=('example',), + logger=self.logger, + ) + + self.plugin_manager.discover_modules() + + plugin_settings = self.load_plugin_settings() + self.plugin_manager.instantiate_plugins(plugin_settings) + + menu_builder = MenuBuilder(icon_factory=self.icon_factory) + settings_pages, settings_callbacks = self.plugin_manager.setup_plugins( + data=self.app_data, + notifications=self.notification_manager, + menu_builder=menu_builder, + menubar=self.menubar, + ) + + menus = MenuContext(icon_factory=self.icon_factory, logger=self.logger) + menus.register_location('file', self.file_menu) + menus.register_location('window', self.window_menu) + + self.plugin_manager.register_context('menus', menus) + self.plugin_manager.register_context('mdi', MDIContext(self.mdi_area)) + self.plugin_manager.register_context( + 'dialogs', + DialogContext(parent=self.main_window), + ) + self.plugin_manager.setup_contexts(self.app_data) + menus.render() + + self.settings_widget = self.build_settings_ui(settings_pages) + for callback in settings_callbacks: + self.register_settings_callback(callback) + + self.plugin_manager.setup_complete(self.app_data) + + def load_plugin_settings(self): + # App-owned storage. Return a mapping from plugin name to dict. + return { + 'example': {'enabled': True}, + } + + def build_settings_ui(self, settings_pages): + # App-owned settings UI. Each item in settings_pages is a plugin + # contributed settings page class. + return settings_pages + + def register_settings_callback(self, callback): + # App-owned settings wiring. + self.settings_changed_callbacks.append(callback) + + def fire_event(self, name): + # App-owned event firing. PluginManager only returns callbacks. + for callback in self.plugin_manager.get_callbacks(name): + callback() + + def close_plugins(self): + self.plugin_manager.close_plugins() + + +Minimal plugin package example +------------------------------ + +This example shows the shared plugin surface. The plugin contributes a legacy +menu, a contribution-based menu action, a settings page, a notification, an MDI +window request, callbacks, ``setup_complete()`` handling, save data, and +shutdown cleanup. Methods can return empty lists or dictionaries when a feature +is not used:: + + from labscript_utils.plugins import BasePlugin, callback + + + class SettingsPage(object): + pass + + + class Notification(object): + pass + + + class DataBrowser(object): + def __init__(self, parent, services): + self.parent = parent + self.services = services + + + class Menu(object): + def __init__(self, data): + self.data = data + + def get_menu_items(self): + return { + 'name': 'Example', + 'menu_items': [ + {'name': 'Legacy Open', 'action': self.open_action}, + {'separator': True}, + { + 'name': 'Legacy Settings', + 'icon': 'gear', + 'action': self.settings_action, + }, + ], + } + + def open_action(self): + pass + + def settings_action(self): + pass + + + class Plugin(BasePlugin): + def __init__(self, initial_settings): + super(Plugin, self).__init__(initial_settings) + self.saved_state = initial_settings + self.app_data = None + + def get_menu_class(self): + return Menu + + def get_menu_contributions(self): + return [ + { + 'location': 'file', + 'path': ('Examples',), + 'group': 'open', + 'order': 20, + 'name': 'Open Data Browser', + 'shortcut': 'Ctrl+D', + 'icon': 'folder-open', + 'action': self.open_data_browser, + }, + ] + + def get_ui_contributions(self): + return [ + { + 'context': 'mdi', + 'key': 'data_browser', + 'title': 'Data Browser', + 'factory': DataBrowser, + }, + ] + + def get_notification_classes(self): + return [Notification] + + def get_setting_classes(self): + return [SettingsPage] + + def get_event_handlers(self): + return { + 'settings_changed': self.on_settings_changed, + 'shot_complete': self.on_shot_complete, + } + + def plugin_setup_complete(self, data): + self.app_data = data + + def get_save_data(self): + return {'saved_state': self.saved_state} + + def close(self): + pass + + def open_data_browser(self): + pass + + @callback(priority=5) + def on_settings_changed(self): + pass + + @callback(priority=20) + def on_shot_complete(self): + pass + + +Feature excerpts and expected UI +-------------------------------- + +BasePlugin subclass with saved ``initial_settings`` and ``get_save_data()`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The constructor receives the per-plugin saved settings dictionary. Store it if +you need to restore state later. ``get_save_data()`` should return the +serializable data you want written back for the next application start:: + + class Plugin(BasePlugin): + def __init__(self, initial_settings): + super(Plugin, self).__init__(initial_settings) + self.initial_settings = initial_settings + + def get_save_data(self): + return { + 'window_geometry': self.window_geometry, + 'last_path': self.last_path, + } + +Settings page contribution skeleton +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``get_setting_classes()`` returns the page classes the application should add +to its settings UI. The application owns the widget construction and storage:: + + class Plugin(BasePlugin): + def get_setting_classes(self): + return [SettingsPage] + + class SettingsPage(object): + pass + +Expected UI result: the application shows one settings page for each returned +class, using the app's own settings shell and persistence rules. + +Legacy menu skeleton with MenuBuilder +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``get_menu_class()`` returns a class whose instance exposes +``get_menu_items()``. The nested dictionaries are translated into the app's +menu objects by ``MenuBuilder``. It expects menus and actions to support +``addMenu()``, ``addAction()``, and ``addSeparator()``:: + + class Plugin(BasePlugin): + def get_menu_class(self): + return Menu + + class Menu(object): + def __init__(self, data): + self.data = data + + def get_menu_items(self): + return { + 'name': 'Example', + 'menu_items': [ + {'name': 'Open', 'action': self.open_action}, + {'separator': True}, + { + 'name': 'Settings', + 'icon': 'gear', + 'action': self.settings_action, + }, + ], + } + + def open_action(self): + pass + + def settings_action(self): + pass + +Expected UI result: the app user sees a top-level menu labeled ``Example``, +with an ``Open`` action, a separator, and a ``Settings`` action. If the app +supplies ``icon_factory``, the ``Settings`` action can also show an icon. + +Contribution menu skeleton with MenuContext +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``get_menu_contributions()`` returns dictionaries routed to the application +``menus`` context. ``MenuContext`` understands these keys: + +* ``location``: app-registered top-level menu id, such as ``file`` or + ``window`` +* ``path``: optional submenu path below that location; defaults to ``()`` +* ``group``: separator group within the same location/path; defaults to + ``None`` +* ``order``: numeric order within the group; defaults to ``DEFAULT_PRIORITY`` +* ``name``: action text +* ``action``: callable connected to the action's ``triggered`` signal +* ``shortcut``, ``icon``, ``checkable``, and ``enabled``: optional action + properties applied when the concrete action object supports them + +File-dialog-style plugin example:: + + class FileDialogsPlugin(BasePlugin): + def __init__(self, initial_settings): + super(FileDialogsPlugin, self).__init__(initial_settings) + self.services = None + + def plugin_setup_complete(self, data): + self.services = data['project_services'] + + def get_menu_contributions(self): + return [ + { + 'location': 'file', + 'group': 'project', + 'order': 10, + 'name': 'New Project', + 'shortcut': 'Ctrl+N', + 'action': self.new_project, + }, + { + 'location': 'file', + 'group': 'project', + 'order': 20, + 'name': 'Open Project...', + 'shortcut': 'Ctrl+O', + 'action': self.open_project, + }, + { + 'location': 'file', + 'group': 'save', + 'order': 10, + 'name': 'Save', + 'shortcut': 'Ctrl+S', + 'action': self.save_project, + }, + { + 'location': 'file', + 'group': 'save', + 'order': 20, + 'name': 'Save As...', + 'shortcut': 'Ctrl+Shift+S', + 'action': self.save_project_as, + }, + { + 'location': 'file', + 'group': 'application', + 'order': 100, + 'name': 'Quit', + 'shortcut': 'Ctrl+Q', + 'action': self.quit_application, + }, + ] + + def new_project(self): + self.services.new_project_dialog.open() + + def open_project(self): + self.services.load_project_dialog.open() + + def save_project(self): + self.services.save_project_command.execute() + + def save_project_as(self): + self.services.save_as_project_dialog.open() + + def quit_application(self): + self.services.quit_command.execute() + +Expected UI result: the application renders the actions in its registered +``file`` menu, creates separators between the ``project``, ``save``, and +``application`` groups, and applies shortcuts without the shared framework +knowing what a project is. + +App-neutral UI homing with contexts +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``get_ui_contributions()`` returns dictionaries with a required ``context``. +The rest of the dictionary is application-defined. This is enough to support +known containers, deferred containers, and plugin-provided containers without +adding type-specific methods such as ``create_tabs()`` or +``create_mdi_windows()`` to ``PluginManager``:: + + class KnownContainerContext(object): + def __init__(self, tab_widget): + self.tab_widget = tab_widget + + def add(self, plugin_name, contribution, data): + widget = contribution['factory']( + self.tab_widget, + data['settings'][contribution['key']], + ) + self.tab_widget.addTab(widget, contribution['title']) + + + class DeferredMDIContext(object): + def __init__(self, mdi_area): + self.mdi_area = mdi_area + self.openers = {} + + def add(self, plugin_name, contribution, data): + key = contribution['key'] + + def open_window(): + widget = contribution['factory'](parent=self.mdi_area, data=data) + self.mdi_area.addSubWindow(widget) + widget.show() + return widget + + self.openers[key] = open_window + + + class DialogContext(object): + def __init__(self, parent): + self.parent = parent + self.dialogs = {} + + def add(self, plugin_name, contribution, data): + self.dialogs[contribution['key']] = contribution['factory']( + parent=self.parent, + services=data['services'], + ) + + + class Plugin(BasePlugin): + def get_ui_contributions(self): + return [ + { + 'context': 'tabs', + 'key': 'overview', + 'title': 'Overview', + 'factory': OverviewWidget, + }, + { + 'context': 'mdi', + 'key': 'data_browser', + 'title': 'Data Browser', + 'factory': DataBrowser, + }, + { + 'context': 'dialogs', + 'key': 'open_project', + 'title': 'Open Project', + 'factory': OpenProjectDialog, + }, + ] + +Expected UI result: each application-owned context decides how to instantiate, +store, parent, restore, defer, activate, or display its contribution. Unknown +contexts are logged and skipped by ``PluginManager.setup_contexts()``. + +Notification contribution skeleton +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``get_notification_classes()`` returns notification classes. The application +owns the notification manager and decides how the resulting widgets are shown:: + + class Plugin(BasePlugin): + def get_notification_classes(self): + return [Notification] + + class Notification(object): + pass + +Expected UI result: the application creates whatever notification widgets or +panels it normally uses for those classes and inserts them into its own +notification area. + +Callbacks with priorities +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``@callback(priority=...)`` wraps a method in ``Callback`` metadata. Lower +numbers run first when the application asks ``PluginManager.get_callbacks()``:: + + class SlowPlugin(BasePlugin): + def get_event_handlers(self): + return { + 'shot_complete': self.on_slow, + } + + @callback(priority=20) + def on_slow(self): + pass + + class FastPlugin(BasePlugin): + def get_event_handlers(self): + return { + 'shot_complete': self.on_fast, + } + + @callback(priority=5) + def on_fast(self): + pass + +Expected runtime order: when the application fires ``shot_complete`` and calls +``plugin_manager.get_callbacks('shot_complete')``, the priority 5 callback +runs before the priority 20 callback, even if the plugins were discovered in a +different order. + +``setup_complete()`` for app data +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``setup_complete()`` is called after the application has finished startup. +Use it to access app-owned state that was not available during construction, +such as the main window, settings object, registered services, or concrete +context objects:: + + class Plugin(BasePlugin): + def plugin_setup_complete(self, data): + self.app = data['app'] + self.settings = data['settings'] + self.services = data['services'] + +The ``data`` dictionary is app-defined. Old plugins may still define +``plugin_setup_complete(self)`` without the ``data`` argument, and the manager +keeps that compatibility path. + +Shutdown cleanup with ``close()`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``close()`` is the plugin shutdown hook. Use it to stop timers, threads, +listeners, or any other plugin-owned resources:: + + class Plugin(BasePlugin): + def close(self): + self.worker.stop() + self.worker.wait() + +Expected behavior: the application calls each plugin's ``close()`` method +during shutdown so the plugin can release resources before exit. +""" + +import importlib +import logging +import os +import warnings +from collections.abc import Mapping +from types import MethodType + + +DEFAULT_PRIORITY = 10 + +__all__ = [ + 'DEFAULT_PRIORITY', + 'Callback', + 'callback', + 'BasePlugin', + 'MenuBuilder', + 'MenuContext', + 'PluginManager', +] + + +def _log_once(logger, seen, level, key, message): + """Log ``message`` once for each hashable ``key``.""" + if key in seen: + return + seen.add(key) + logger.log(level, message) + + +class Callback(object): + """Wrap a callable with priority metadata and method-style binding. + + ``priority`` is used when ``PluginManager.get_callbacks()`` gathers all + callbacks for a named event. Lower numbers run first. + + The descriptor protocol is implemented so that a ``Callback`` stored as a + class attribute binds like an instance method when accessed from a plugin + instance. This keeps decorated methods usable without extra wrapper code. + """ + def __init__(self, func, priority=DEFAULT_PRIORITY): + self.priority = priority + self.func = func + + def __get__(self, instance, class_): + """Bind to ``instance`` the same way a normal function descriptor does.""" + if instance is None: + return self + else: + return MethodType(self, instance) + + def __call__(self, *args, **kwargs): + return self.func(*args, **kwargs) + + +class callback(object): + """Decorator that turns a function into a :class:`Callback`. + + The decorator is optional. A plain method can still be returned from + ``get_callbacks()`` if it already behaves like a callback. Use the + decorator when you want to attach a non-default priority or make the + callback object explicit in the class body. + """ + # Instantiate the decorator: + def __init__(self, priority=DEFAULT_PRIORITY): + self.priority = priority + + # Call the decorator + def __call__(self, func): + return Callback(func, self.priority) + + +class BasePlugin(object): + """Reference implementation of the plugin interface. + + Subclass this when you want a concrete starting point, but it is not + required. The manager only relies on the methods defined here; a plugin + can also satisfy the contract by duck typing. + + ``initial_settings`` contains the saved configuration for the plugin. + """ + def __init__(self, initial_settings): + """Store the plugin's saved settings. + + Applications pass the dictionary returned by the previous + ``get_save_data()`` call, or an empty dictionary for a first run. + Subclasses commonly keep this dictionary and use it during + ``plugin_setup_complete()`` to restore UI state or background state. + """ + self.initial_settings = initial_settings + self.menu = None + self.notifications = {} + + def get_menu_class(self): + """Return a menu class for this plugin, or ``None``. + + Deprecated compatibility hook for BLACS-style nested menus. + + The class is constructed as ``MenuClass(data)`` during + ``PluginManager.setup_plugins()``. Its instance should provide + ``get_menu_items()``, returning the nested menu dictionary consumed by + :class:`MenuBuilder`. New shared plugin code should prefer + :meth:`get_menu_contributions` instead. + """ + return None + + def get_notification_classes(self): + """Return notification classes contributed by this plugin. + + The application-specific notification manager constructs the + notifications. Instances are later passed back to + ``set_notification_instances()`` in a dictionary keyed by notification + class. + """ + return [] + + def get_setting_classes(self): + """Return settings-page classes contributed by this plugin. + + The application owns the settings UI and storage. The manager only + collects these classes and returns them to the application from + ``setup_plugins()``. + """ + return [] + + def get_event_handlers(self): + """Return a mapping of event names to handlers, or ``None``. + + This is the preferred shared event hook surface. Return a mapping such + as ``{'shot_complete': self.on_shot_complete}``. Event names are + application-defined strings; this shared layer only collects, + normalizes, and orders handlers. Values may be plain callables or + :class:`Callback` instances created with the :class:`callback` + decorator. The manager sorts handlers for a given event by their + ``priority`` attribute. + """ + return None + + def get_callbacks(self): + """Return callbacks keyed by event name, or ``None``. + + Deprecated compatibility alias for :meth:`get_event_handlers`. + + Existing BLACS-style plugins may still override this method. New + shared plugin code should implement :meth:`get_event_handlers` + instead. ``BasePlugin`` routes this legacy name to the modern hook so + the preferred surface is the real implementation path. + """ + return self.get_event_handlers() + + def get_ui_contributions(self): + """Return app-context UI contributions. + + Return an iterable of dictionaries. Each contribution has a + ``context`` key naming an application-registered context. The + application owns what that context means: a known tab area, a deferred + MDI workspace, a dialog registry, a fixed frame, or another + app-specific host. The remaining keys are application-defined. + """ + return [] + + def get_menu_contributions(self): + """Return shared menu contributions. + + Return an iterable of dictionaries. This is the app-neutral menu path. + Existing BLACS-style plugins can continue using ``get_menu_class()`` + and :class:`MenuBuilder`; new apps can register a ``menus`` context and + route these dictionaries through :class:`MenuContext`. Apart from + stable menu keys such as ``location``, ``path``, ``group``, ``order``, + ``name``, and ``action``, the concrete menu objects remain + application-owned. + """ + return [] + + def set_menu_instance(self, menu): + """Receive the menu instance constructed for this plugin.""" + self.menu = menu + + def set_notification_instances(self, notifications): + """Receive notification instances constructed for this plugin.""" + self.notifications = notifications + + def plugin_setup_complete(self, data=None): + """Run after the application has finished plugin setup. + + ``data`` is an application-defined dictionary. Plugins commonly store + references to services or state they will need later when handling + events, building menus, or opening UI. For BLACS it contains + references such as the main UI, experiment config, plugin mapping, and + settings object. Older plugins may define this method without a + ``data`` argument; ``PluginManager.setup_complete()`` keeps that + compatibility behavior. + """ + pass + + def get_save_data(self): + """Return serializable plugin state for the next application start. + + The shape of this data is application-owned. Return plain data + structures that the application can persist and pass back as + ``initial_settings`` during the next start. + """ + return {} + + def close(self): + """Clean up resources owned by the plugin during application shutdown. + + Applications call this during shutdown. Stop timers, workers, + listeners, or other plugin-owned resources here. + """ + pass + + +class MenuBuilder(object): + """Build menus from the nested dictionary format used by plugins. + + Each menu description is a dictionary. Supported keys are: + + * ``name``: text for a menu or action + * ``menu_items``: child menu descriptions, making a submenu + * ``icon``: icon name passed to ``icon_factory`` for actions + * ``action``: callable connected to the action's ``triggered`` signal + * ``separator``: a truthy sentinel that inserts a separator + + ``icon_factory`` is optional and should return a Qt icon object or other + value accepted by ``addAction``. Supplying it keeps this module free of a + direct Qt dependency. + """ + def __init__(self, icon_factory=None): + """Create a menu builder. + + Args: + icon_factory (callable, optional): Function called as + ``icon_factory(icon_name)`` before adding an icon-bearing menu + action. Applications using Qt typically pass ``QIcon``. + """ + self.icon_factory = icon_factory + + def create_menu(self, parent, menu_parameters): + """Recursively build menus and actions from a nested menu dictionary.""" + if 'name' in menu_parameters: + if 'menu_items' in menu_parameters: + child = parent.addMenu(menu_parameters['name']) + for child_menu_params in menu_parameters['menu_items']: + self.create_menu(child, child_menu_params) + else: + # ``icon_factory`` stays outside this module so labscript-utils + # does not need to import a Qt binding directly. + if 'icon' in menu_parameters and self.icon_factory is not None: + child = parent.addAction( + self.icon_factory(menu_parameters['icon']), + menu_parameters['name'], + ) + else: + child = parent.addAction(menu_parameters['name']) + + if 'action' in menu_parameters: + child.triggered.connect(menu_parameters['action']) + + elif 'separator' in menu_parameters: + parent.addSeparator() + + +class MenuContext(object): + """Collect and render contribution-based menu actions. + + Applications register stable top-level menu locations, such as ``file`` or + ``tools``. Plugins contribute action dictionaries that request a location, + optional submenu path, separator group, order, action text, and callback. + Rendering is deferred until all plugins have contributed so ordering and + group separators can be computed across plugins. + """ + def __init__(self, icon_factory=None, logger=None): + """Create a menu context. + + Args: + icon_factory (callable, optional): Function called as + ``icon_factory(icon_name)`` for icon-bearing actions. + logger (logging.Logger, optional): Logger used for skipped or + malformed menu contributions. + """ + self.icon_factory = icon_factory + self.logger = logger or logging.getLogger(__name__) + self.locations = {} + self.contributions = [] + + def register_location(self, name, menu): + """Register an application-owned menu object as a stable location.""" + self.locations[name] = menu + + def add(self, plugin_name, contribution, data): + """Collect one menu contribution for later rendering. + + ``data`` is accepted for the generic context contract. Menu + contributions already contain their callbacks, so this context does not + need to inspect app data directly. + """ + self.contributions.append((plugin_name, contribution)) + + def render(self): + """Render all collected menu contributions into registered locations.""" + grouped = {} + group_orders = {} + + for plugin_name, contribution in self.contributions: + if not isinstance(contribution, dict): + self.logger.error( + "Menu contribution from plugin '%s' is not a dictionary. " + "Skipping." % plugin_name + ) + continue + if 'location' not in contribution: + self.logger.error( + "Menu contribution from plugin '%s' missing location. " + "Skipping." % plugin_name + ) + continue + if 'name' not in contribution: + self.logger.error( + "Menu contribution from plugin '%s' missing name. " + "Skipping." % plugin_name + ) + continue + + location = contribution['location'] + if location not in self.locations: + self.logger.error( + "Menu contribution from plugin '%s' requested unknown " + "location '%s'. Skipping." % (plugin_name, location) + ) + continue + + path = contribution.get('path', ()) + if isinstance(path, str): + path = (path,) + else: + path = tuple(path) + + key = (location, path) + group = contribution.get('group', None) + if key not in group_orders: + group_orders[key] = {} + if group not in group_orders[key]: + group_orders[key][group] = len(group_orders[key]) + + grouped.setdefault(key, []).append((plugin_name, contribution)) + + menus = {} + for name, menu in self.locations.items(): + menus[(name, ())] = menu + + for key in sorted(grouped): + location, path = key + parent_path = () + for submenu_name in path: + submenu_path = parent_path + (submenu_name,) + submenu_key = (location, submenu_path) + if submenu_key not in menus: + menus[submenu_key] = menus[(location, parent_path)].addMenu( + submenu_name + ) + parent_path = submenu_path + menu = menus[(location, path)] + + contributions = sorted( + grouped[key], + key=lambda item: ( + group_orders[key][item[1].get('group', None)], + item[1].get('order', DEFAULT_PRIORITY), + item[0], + item[1]['name'], + ), + ) + + previous_group = None + for index, (plugin_name, contribution) in enumerate(contributions): + group = contribution.get('group', None) + if index and group != previous_group: + menu.addSeparator() + previous_group = group + + name = contribution['name'] + icon = contribution.get('icon', None) + if icon is not None and self.icon_factory is not None: + action = menu.addAction(self.icon_factory(icon), name) + else: + action = menu.addAction(name) + + callback = contribution.get('action', None) + if callback is not None: + action.triggered.connect(callback) + + shortcut = contribution.get('shortcut', None) + if shortcut is not None and hasattr(action, 'setShortcut'): + action.setShortcut(shortcut) + + checkable = contribution.get('checkable', False) + if hasattr(action, 'setCheckable'): + action.setCheckable(checkable) + + enabled = contribution.get('enabled', True) + if hasattr(action, 'setEnabled'): + action.setEnabled(enabled) + + +class PluginManager(object): + """Manage plugin discovery, lifecycle, and callback lookup. + + The manager keeps two internal mappings: + + * ``modules``: imported plugin modules keyed by plugin name + * ``plugins``: instantiated plugin objects keyed by plugin name + + An application normally creates one manager per plugin directory and uses + it to discover modules, instantiate plugin objects, build UI pieces, and + close plugins on shutdown. + """ + def __init__( + self, + plugin_package, + plugins_dir, + config, + config_section, + default_plugins=(), + logger=None, + ): + """Create a plugin manager. + + Args: + plugin_package (str): Import package containing plugin subpackages, + for example ``'myapp.plugins'``. + plugins_dir (str): Filesystem path scanned for plugin directories. + config: Config object with ``has_section()``, ``add_section()``, + ``items()``, ``set()``, and ``getboolean()`` methods. + config_section (str): Section containing plugin enable/disable + options. + default_plugins (iterable): Plugin names enabled by default when + first discovered. + logger (logging.Logger, optional): Logger used for plugin import, + setup, callback, and shutdown errors. + """ + self.plugin_package = plugin_package + self.plugins_dir = plugins_dir + self.config = config + self.config_section = config_section + self.default_plugins = set(default_plugins) + self.logger = logger or logging.getLogger(__name__) + self.modules = {} + self.plugins = {} + self.contexts = {} + self._logged_plugin_warnings = set() + + def discover_modules(self): + """Scan the plugin directory, update config defaults, and import enabled modules.""" + if not self.config.has_section(self.config_section): + self.config.add_section(self.config_section) + + configured_plugins = set( + name for name, val in self.config.items(self.config_section) + ) + + modules = {} + for module_name in os.listdir(self.plugins_dir): + module_path = os.path.join(self.plugins_dir, module_name) + if not os.path.isdir(module_path) or module_name == '__pycache__': + continue + + # Keep the config in sync with what is present on disk. + if module_name not in configured_plugins: + self.config.set( + self.config_section, + module_name, + str(module_name in self.default_plugins), + ) + configured_plugins.add(module_name) + + # Only load activated plugins. + if self.config.getboolean(self.config_section, module_name): + try: + module = importlib.import_module( + self.plugin_package + '.' + module_name + ) + except Exception: + self.logger.exception( + "Could not import plugin '%s'. Skipping." % module_name + ) + else: + modules[module_name] = module + + self.modules = modules + return modules + + def instantiate_plugins(self, plugin_settings=None): + """Create ``Plugin`` instances for each imported module. + + ``plugin_settings`` should map plugin names to the saved configuration + dictionary for that plugin. Missing entries fall back to an empty + dictionary. Instantiation failures are logged and skipped. + """ + if plugin_settings is None: + plugin_settings = {} + + plugins = {} + for module_name, module in self.modules.items(): + try: + plugins[module_name] = module.Plugin( + plugin_settings[module_name] + if module_name in plugin_settings else {} + ) + except Exception: + self.logger.exception( + "Could not instantiate plugin '%s'. Skipping" % module_name + ) + + self.plugins = plugins + return plugins + + def setup_plugins(self, data, notifications, menu_builder, menubar): + """Collect plugin UI and startup contributions. + + This is typically called after the application has created its legacy + menu and notification hosts and before it finishes wiring its settings + UI and callback registrations. The method returns a tuple of + ``(settings_pages, settings_callbacks)`` for the application to + consume. + """ + settings_pages = [] + settings_callbacks = [] + + for module_name, plugin in self.plugins.items(): + try: + # Setup settings page. + settings_pages.extend(plugin.get_setting_classes()) + + # Setup menu. + menu_class = plugin.get_menu_class() + if menu_class: + _log_once( + self.logger, + self._logged_plugin_warnings, + logging.WARNING, + (module_name, 'get_menu_class'), + "Plugin '%s' uses deprecated get_menu_class(); " + "prefer get_menu_contributions() instead." % module_name + ) + # Must store a reference or else the methods called when + # the menu actions are triggered will be garbage collected. + menu = menu_class(data) + menu_builder.create_menu(menubar, menu.get_menu_items()) + plugin.set_menu_instance(menu) + + # Setup notifications. + plugin_notifications = {} + for notification_class in plugin.get_notification_classes(): + notifications.add_notification(notification_class) + plugin_notifications[notification_class] = ( + notifications.get_instance(notification_class) + ) + plugin.set_notification_instances(plugin_notifications) + + # Register callbacks. + callbacks = self._get_plugin_event_handlers(module_name, plugin) + # Save the settings_changed callback in a separate list for + # setting up later. + if callbacks and 'settings_changed' in callbacks: + settings_callbacks.append(callbacks['settings_changed']) + + except Exception: + self.logger.exception( + "Plugin '%s' error. Plugin may not be functional." % module_name + ) + + return settings_pages, settings_callbacks + + def register_context(self, name, context): + """Register an application-owned plugin contribution context.""" + self.contexts[name] = context + + def _get_contributions(self, plugin, module_name, method_name, label): + """Return a plugin contribution iterable, logging malformed results.""" + if not hasattr(plugin, method_name): + return [] + + try: + contributions = getattr(plugin, method_name)() + except Exception: + self.logger.exception( + "Error getting %s contributions from plugin '%s'. Skipping." + % (label, module_name) + ) + return [] + + if contributions is None: + return [] + + if ( + isinstance(contributions, (dict, str, bytes)) + or not hasattr(contributions, '__iter__') + ): + self.logger.error( + "Plugin '%s' %s contributions must be an iterable of " + "dictionaries. Skipping." % (module_name, label) + ) + return [] + + return contributions + + def setup_contexts(self, data): + """Route plugin-declared UI and menu contributions to app contexts.""" + for module_name, plugin in self.plugins.items(): + ui_contributions = self._get_contributions( + plugin, + module_name, + 'get_ui_contributions', + 'UI', + ) + + for contribution in ui_contributions: + if not isinstance(contribution, dict): + self.logger.error( + "UI contribution from plugin '%s' is not a dictionary. " + "Skipping." % module_name + ) + continue + if 'context' not in contribution: + self.logger.error( + "UI contribution from plugin '%s' missing context. " + "Skipping." % module_name + ) + continue + + context_name = contribution['context'] + if context_name not in self.contexts: + self.logger.error( + "UI contribution from plugin '%s' requested unknown " + "context '%s'. Skipping." % (module_name, context_name) + ) + continue + + try: + self.contexts[context_name].add(module_name, contribution, data) + except Exception: + self.logger.exception( + "Error adding UI contribution from plugin '%s' to " + "context '%s'. Skipping." % (module_name, context_name) + ) + + menu_contributions = self._get_contributions( + plugin, + module_name, + 'get_menu_contributions', + 'menu', + ) + + if menu_contributions and 'menus' not in self.contexts: + self.logger.error( + "Plugin '%s' provided menu contributions, but no 'menus' " + "context is registered. Skipping." % module_name + ) + continue + + for contribution in menu_contributions: + if not isinstance(contribution, dict): + self.logger.error( + "Menu contribution from plugin '%s' is not a " + "dictionary. Skipping." % module_name + ) + continue + try: + self.contexts['menus'].add(module_name, contribution, data) + except Exception: + self.logger.exception( + "Error adding menu contribution from plugin '%s'. " + "Skipping." % module_name + ) + + def setup_complete(self, data): + """Notify plugins that the application has finished startup.""" + for module_name, plugin in self.plugins.items(): + try: + plugin.plugin_setup_complete(data) + except Exception: + self.logger.exception( + "Error in plugin_setup_complete() for plugin '%s'. " + "Trying again with old call signature..." % module_name + ) + # Backwards compatibility for old plugins. + try: + plugin.plugin_setup_complete() + self.logger.warning( + "Plugin '%s' using old API. Please update " + "Plugin.plugin_setup_complete method to accept a " + "dictionary of blacs_data as the only argument." + % module_name + ) + except Exception: + self.logger.exception( + "Plugin '%s' error. Plugin may not be functional." + % module_name + ) + + def _defines_hook(self, plugin, method_name): + """Return whether ``plugin`` overrides ``method_name`` meaningfully.""" + plugin_method = getattr(type(plugin), method_name, None) + if plugin_method is None: + return False + + base_method = getattr(BasePlugin, method_name, None) + return plugin_method is not base_method + + def _get_plugin_event_handlers(self, module_name, plugin): + """Return normalized event handlers for one plugin.""" + has_modern = self._defines_hook(plugin, 'get_event_handlers') + has_legacy = self._defines_hook(plugin, 'get_callbacks') + + if has_modern: + if has_legacy: + _log_once( + self.logger, + self._logged_plugin_warnings, + logging.WARNING, + (module_name, 'get_callbacks'), + "Plugin '%s' defines deprecated get_callbacks() and " + "preferred get_event_handlers(); ignoring get_callbacks()." + % module_name + ) + handlers = plugin.get_event_handlers() + elif has_legacy: + _log_once( + self.logger, + self._logged_plugin_warnings, + logging.WARNING, + (module_name, 'get_callbacks'), + "Plugin '%s' uses deprecated get_callbacks(); " + "implement get_event_handlers() instead." % module_name + ) + handlers = plugin.get_callbacks() + else: + return None + + if handlers is None: + return None + if not isinstance(handlers, Mapping): + self.logger.error( + "Plugin '%s' event handlers must be a mapping. Skipping." + % module_name + ) + return None + + return handlers + + def get_event_handlers(self, name): + """Return all event handlers registered for ``name``, sorted by priority.""" + callbacks = [] + + for module_name, plugin in self.plugins.items(): + try: + plugin_callbacks = self._get_plugin_event_handlers( + module_name, + plugin, + ) + if plugin_callbacks and name in plugin_callbacks: + callbacks.append(plugin_callbacks[name]) + except Exception: + self.logger.exception('Error getting callbacks from %s.' % str(plugin)) + + callbacks.sort( + key=lambda callback: getattr(callback, 'priority', DEFAULT_PRIORITY) + ) + return callbacks + + def get_callbacks(self, name): + """Deprecated compatibility wrapper for :meth:`get_event_handlers`.""" + warnings.warn( + "PluginManager.get_callbacks() is deprecated; use " + "PluginManager.get_event_handlers() instead.", + DeprecationWarning, + stacklevel=2, + ) + return self.get_event_handlers(name) + + def close_plugins(self): + """Call ``close()`` on every plugin during application shutdown.""" + for module_name, plugin in self.plugins.items(): + try: + plugin.close() + except Exception as e: + self.logger.error( + 'Could not close plugin %s. Error was: %s' + % (module_name, str(e)) + ) From aaa642b912dae1a507bd520326876a6df0f5157f Mon Sep 17 00:00:00 2001 From: spielman Date: Mon, 27 Apr 2026 12:26:12 -0400 Subject: [PATCH 34/66] Added ability to share services. --- labscript_utils/plugins.py | 58 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/labscript_utils/plugins.py b/labscript_utils/plugins.py index 1033a20a..36dc0865 100644 --- a/labscript_utils/plugins.py +++ b/labscript_utils/plugins.py @@ -788,6 +788,16 @@ def get_save_data(self): """ return {} + def get_services(self): + """Return named services exposed by this plugin. + + Return a mapping of service names to concrete objects. Applications can + aggregate these mappings into a shared service registry before calling + :meth:`plugin_setup_complete`, making plugin-to-plugin dependencies + order-independent. + """ + return {} + def close(self): """Clean up resources owned by the plugin during application shutdown. @@ -1033,6 +1043,7 @@ def __init__( self.logger = logger or logging.getLogger(__name__) self.modules = {} self.plugins = {} + self.services = {} self.contexts = {} self._logged_plugin_warnings = set() @@ -1260,6 +1271,53 @@ def setup_contexts(self, data): "Skipping." % module_name ) + def collect_services(self, base_services=None): + """Merge app-owned and plugin-owned services into one registry.""" + if base_services is None: + services = {} + elif isinstance(base_services, Mapping): + services = dict(base_services) + else: + self.logger.error( + "Base services must be a mapping. Ignoring invalid registry." + ) + services = {} + + for module_name, plugin in self.plugins.items(): + if not hasattr(plugin, "get_services"): + continue + + try: + plugin_services = plugin.get_services() + except Exception: + self.logger.exception( + "Error getting services from plugin '%s'. Skipping." + % module_name + ) + continue + + if plugin_services is None: + continue + if not isinstance(plugin_services, Mapping): + self.logger.error( + "Plugin '%s' services must be a mapping. Skipping." + % module_name + ) + continue + + for service_name, service in plugin_services.items(): + if service_name in services: + self.logger.error( + "Plugin '%s' service '%s' conflicts with an existing " + "service. Keeping the original registration." + % (module_name, service_name) + ) + continue + services[service_name] = service + + self.services = services + return services + def setup_complete(self, data): """Notify plugins that the application has finished startup.""" for module_name, plugin in self.plugins.items(): From 44f7aab15117d3d59b7c390ada24668518fbb998 Mon Sep 17 00:00:00 2001 From: spielman Date: Mon, 27 Apr 2026 14:07:54 -0400 Subject: [PATCH 35/66] Add plugin service aggregation tests --- tests/test_plugins.py | 106 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 tests/test_plugins.py diff --git a/tests/test_plugins.py b/tests/test_plugins.py new file mode 100644 index 00000000..c583d32f --- /dev/null +++ b/tests/test_plugins.py @@ -0,0 +1,106 @@ +import logging +import importlib.util +from pathlib import Path +import unittest + +MODULE_PATH = Path(__file__).resolve().parents[1] / "labscript_utils" / "plugins.py" +SPEC = importlib.util.spec_from_file_location("labscript_utils.plugins", MODULE_PATH) +plugins_module = importlib.util.module_from_spec(SPEC) +assert SPEC.loader is not None +SPEC.loader.exec_module(plugins_module) + +BasePlugin = plugins_module.BasePlugin +PluginManager = plugins_module.PluginManager + + +class DummyConfig: + def has_section(self, section): + del section + return True + + def add_section(self, section): + del section + + def items(self, section): + del section + return [] + + def set(self, section, name, value): + del section, name, value + + def getboolean(self, section, name): + del section, name + return False + + +class RecordingHandler(logging.Handler): + def __init__(self): + super().__init__() + self.records = [] + + def emit(self, record): + self.records.append(record) + + +class ServicePlugin(BasePlugin): + def __init__(self, services): + super().__init__({}) + self._services = services + + def get_services(self): + return self._services + + +class TestPluginServices(unittest.TestCase): + def make_manager(self): + logger = logging.getLogger("labscript_utils.plugins.tests") + logger.setLevel(logging.DEBUG) + handler = RecordingHandler() + logger.addHandler(handler) + self.addCleanup(logger.removeHandler, handler) + manager = PluginManager( + plugin_package="unused", + plugins_dir="unused", + config=DummyConfig(), + config_section="unused", + logger=logger, + ) + return manager, handler + + def test_base_plugin_exposes_no_services_by_default(self): + plugin = BasePlugin({}) + self.assertEqual(plugin.get_services(), {}) + + def test_collect_services_merges_base_and_plugin_services(self): + manager, _handler = self.make_manager() + manager.plugins = { + "alpha": ServicePlugin({"answer": 42}), + "beta": ServicePlugin({"greeter": str.upper}), + } + + services = manager.collect_services({"execute_command": object()}) + + self.assertIn("execute_command", services) + self.assertEqual(services["answer"], 42) + self.assertIs(services["greeter"], str.upper) + self.assertIs(manager.services, services) + + def test_collect_services_skips_invalid_and_conflicting_entries(self): + manager, handler = self.make_manager() + manager.plugins = { + "alpha": ServicePlugin({"shared": "first", "unique": "ok"}), + "beta": ServicePlugin({"shared": "second"}), + "gamma": ServicePlugin(["not", "a", "mapping"]), + } + + services = manager.collect_services() + + self.assertEqual(services["shared"], "first") + self.assertEqual(services["unique"], "ok") + messages = [record.getMessage() for record in handler.records] + self.assertTrue(any("must be a mapping" in message for message in messages)) + self.assertTrue(any("conflicts with an existing service" in message for message in messages)) + + +if __name__ == "__main__": + unittest.main() From 57c889fc35c2752408038f195ef538388f9da881 Mon Sep 17 00:00:00 2001 From: spielman Date: Mon, 11 May 2026 09:44:23 -0400 Subject: [PATCH 36/66] Add ordered plugin setup activities --- labscript_utils/plugins.py | 141 ++++++++++++++++++++++++++++++-- tests/test_plugins.py | 160 +++++++++++++++++++++++++++++++++++++ 2 files changed, 296 insertions(+), 5 deletions(-) diff --git a/labscript_utils/plugins.py b/labscript_utils/plugins.py index 36dc0865..46306648 100644 --- a/labscript_utils/plugins.py +++ b/labscript_utils/plugins.py @@ -558,7 +558,8 @@ def on_fast(self): ``setup_complete()`` is called after the application has finished startup. Use it to access app-owned state that was not available during construction, such as the main window, settings object, registered services, or concrete -context objects:: +context objects. Existing plugins can keep overriding +``plugin_setup_complete()``:: class Plugin(BasePlugin): def plugin_setup_complete(self, data): @@ -570,6 +571,28 @@ def plugin_setup_complete(self, data): ``plugin_setup_complete(self)`` without the ``data`` argument, and the manager keeps that compatibility path. +Plugins that need ordered startup work can return setup activity records from +``get_setup_activities()``. Each record has ``name``, ``priority``, and +``action`` keys; lower priority values run first. The default +``get_setup_activities()`` implementation returns one activity for +``plugin_setup_complete()``, so existing plugins use the same ordered pipeline +as modern plugins:: + + class Plugin(BasePlugin): + def get_setup_activities(self): + return [ + { + 'name': 'bind_services', + 'priority': DEFAULT_SETUP_PRIORITY, + 'action': self.bind_services, + }, + { + 'name': 'start_worker', + 'priority': DEFAULT_SETUP_PRIORITY + 10, + 'action': self.start_worker, + }, + ] + Shutdown cleanup with ``close()`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -594,9 +617,11 @@ def close(self): DEFAULT_PRIORITY = 10 +DEFAULT_SETUP_PRIORITY = 0 __all__ = [ 'DEFAULT_PRIORITY', + 'DEFAULT_SETUP_PRIORITY', 'Callback', 'callback', 'BasePlugin', @@ -779,6 +804,23 @@ def plugin_setup_complete(self, data=None): """ pass + def get_setup_activities(self): + """Return ordered setup activities for this plugin. + + Existing plugins can override :meth:`plugin_setup_complete` and inherit + this method. Modern plugins can override this method to return multiple + activity records. Each record must provide ``name``, ``priority``, and + ``action`` keys; the manager executes all plugin activities through one + ordered setup pipeline. + """ + return [ + { + 'name': 'plugin_setup_complete', + 'priority': DEFAULT_SETUP_PRIORITY, + 'action': self.plugin_setup_complete, + }, + ] + def get_save_data(self): """Return serializable plugin state for the next application start. @@ -1319,18 +1361,107 @@ def collect_services(self, base_services=None): return services def setup_complete(self, data): - """Notify plugins that the application has finished startup.""" + """Run plugin setup activities after application startup.""" + setup_activities = [] + for module_name, plugin in self.plugins.items(): + if hasattr(plugin, 'get_setup_activities'): + try: + plugin_activities = plugin.get_setup_activities() + except Exception: + self.logger.exception( + "Error getting setup activities from plugin '%s'. " + "Skipping." % module_name + ) + continue + elif hasattr(plugin, 'plugin_setup_complete'): + plugin_activities = [ + { + 'name': 'plugin_setup_complete', + 'priority': DEFAULT_SETUP_PRIORITY, + 'action': plugin.plugin_setup_complete, + }, + ] + else: + continue + + if plugin_activities is None: + continue + if ( + isinstance(plugin_activities, (dict, str, bytes)) + or not hasattr(plugin_activities, '__iter__') + ): + self.logger.error( + "Plugin '%s' setup activities must be an iterable of " + "dictionaries. Skipping." % module_name + ) + continue + + for activity in plugin_activities: + if not isinstance(activity, Mapping): + self.logger.error( + "Setup activity from plugin '%s' is not a dictionary. " + "Skipping." % module_name + ) + continue + + try: + name = activity['name'] + priority = activity['priority'] + action = activity['action'] + except KeyError as exc: + self.logger.error( + "Setup activity from plugin '%s' missing key '%s'. " + "Skipping." % (module_name, exc.args[0]) + ) + continue + + if not callable(action): + self.logger.error( + "Setup activity '%s' from plugin '%s' action is not " + "callable. Skipping." % (name, module_name) + ) + continue + + setup_activities.append( + { + 'module_name': module_name, + 'plugin': plugin, + 'name': name, + 'priority': priority, + 'action': action, + } + ) + + setup_activities.sort( + key=lambda activity: ( + activity['priority'], + activity['module_name'], + activity['name'], + ) + ) + + for activity in setup_activities: + module_name = activity['module_name'] + activity_name = activity['name'] + action = activity['action'] try: - plugin.plugin_setup_complete(data) + action(data) except Exception: + if activity_name != 'plugin_setup_complete': + self.logger.exception( + "Error in setup activity '%s' for plugin '%s'. " + "Plugin may not be functional." + % (activity_name, module_name) + ) + continue + self.logger.exception( "Error in plugin_setup_complete() for plugin '%s'. " "Trying again with old call signature..." % module_name ) - # Backwards compatibility for old plugins. try: - plugin.plugin_setup_complete() + action() self.logger.warning( "Plugin '%s' using old API. Please update " "Plugin.plugin_setup_complete method to accept a " diff --git a/tests/test_plugins.py b/tests/test_plugins.py index c583d32f..3ebc1ca9 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -11,6 +11,7 @@ BasePlugin = plugins_module.BasePlugin PluginManager = plugins_module.PluginManager +DEFAULT_SETUP_PRIORITY = plugins_module.DEFAULT_SETUP_PRIORITY class DummyConfig: @@ -102,5 +103,164 @@ def test_collect_services_skips_invalid_and_conflicting_entries(self): self.assertTrue(any("conflicts with an existing service" in message for message in messages)) +class TestPluginSetupActivities(unittest.TestCase): + def make_manager(self): + logger = logging.getLogger("labscript_utils.plugins.setup_tests") + logger.setLevel(logging.DEBUG) + handler = RecordingHandler() + logger.addHandler(handler) + self.addCleanup(logger.removeHandler, handler) + manager = PluginManager( + plugin_package="unused", + plugins_dir="unused", + config=DummyConfig(), + config_section="unused", + logger=logger, + ) + return manager, handler + + def test_legacy_plugin_setup_complete_runs_as_default_activity(self): + calls = [] + data = {"services": object()} + + class LegacySetupPlugin(BasePlugin): + def plugin_setup_complete(self, setup_data): + calls.append(("legacy", setup_data)) + + manager, _handler = self.make_manager() + manager.plugins = {"legacy": LegacySetupPlugin({})} + + manager.setup_complete(data) + + self.assertEqual(calls, [("legacy", data)]) + + def test_duck_typed_plugin_setup_complete_still_runs(self): + calls = [] + data = {"services": object()} + + class DuckTypedSetupPlugin: + def plugin_setup_complete(self, setup_data): + calls.append(("duck", setup_data)) + + manager, _handler = self.make_manager() + manager.plugins = {"duck": DuckTypedSetupPlugin()} + + manager.setup_complete(data) + + self.assertEqual(calls, [("duck", data)]) + + def test_multiple_setup_activities_run_in_priority_order(self): + calls = [] + data = {"app": object()} + + class ActivityPlugin(BasePlugin): + def get_setup_activities(self): + return [ + { + "name": "late", + "priority": DEFAULT_SETUP_PRIORITY + 10, + "action": self.late, + }, + { + "name": "early", + "priority": DEFAULT_SETUP_PRIORITY - 10, + "action": self.early, + }, + ] + + def early(self, setup_data): + calls.append(("early", setup_data)) + + def late(self, setup_data): + calls.append(("late", setup_data)) + + manager, _handler = self.make_manager() + manager.plugins = {"activity": ActivityPlugin({})} + + manager.setup_complete(data) + + self.assertEqual(calls, [("early", data), ("late", data)]) + + def test_setup_activity_ordering_is_deterministic_across_plugins(self): + calls = [] + + class NamedActivityPlugin(BasePlugin): + def __init__(self, label): + super().__init__({}) + self.label = label + + def get_setup_activities(self): + return [ + { + "name": "same_priority", + "priority": DEFAULT_SETUP_PRIORITY, + "action": self.record, + }, + ] + + def record(self, data): + del data + calls.append(self.label) + + manager, _handler = self.make_manager() + manager.plugins = { + "zeta": NamedActivityPlugin("zeta"), + "alpha": NamedActivityPlugin("alpha"), + } + + manager.setup_complete({}) + + self.assertEqual(calls, ["alpha", "zeta"]) + + def test_setup_activity_names_break_priority_and_plugin_ties(self): + calls = [] + + class TiedActivityPlugin(BasePlugin): + def get_setup_activities(self): + return [ + { + "name": "second", + "priority": DEFAULT_SETUP_PRIORITY, + "action": self.second, + }, + { + "name": "first", + "priority": DEFAULT_SETUP_PRIORITY, + "action": self.first, + }, + ] + + def first(self, data): + del data + calls.append("first") + + def second(self, data): + del data + calls.append("second") + + manager, _handler = self.make_manager() + manager.plugins = {"plugin": TiedActivityPlugin({})} + + manager.setup_complete({}) + + self.assertEqual(calls, ["first", "second"]) + + def test_no_argument_plugin_setup_complete_fallback_still_runs(self): + calls = [] + + class NoArgumentSetupPlugin(BasePlugin): + def plugin_setup_complete(self): + calls.append("legacy-no-arg") + + manager, handler = self.make_manager() + manager.plugins = {"legacy": NoArgumentSetupPlugin({})} + + manager.setup_complete({"unused": object()}) + + messages = [record.getMessage() for record in handler.records] + self.assertEqual(calls, ["legacy-no-arg"]) + self.assertTrue(any("using old API" in message for message in messages)) + + if __name__ == "__main__": unittest.main() From 17e36cc8b5a45f61157f24decba3de4a2b686230 Mon Sep 17 00:00:00 2001 From: spielman Date: Tue, 12 May 2026 11:01:51 -0400 Subject: [PATCH 37/66] Set macOS Qt application names from desktop app config --- labscript_utils/splash.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/labscript_utils/splash.py b/labscript_utils/splash.py index 6b24b095..8d8d7205 100644 --- a/labscript_utils/splash.py +++ b/labscript_utils/splash.py @@ -11,7 +11,9 @@ # # ##################################################################### +import json import sys +from pathlib import Path from labscript_utils import dedent try: @@ -45,7 +47,23 @@ def configure_qapplication(qapplication): if sys.platform == 'darwin': icon_path = qapplication.property('_labscript_icon_path') if icon_path: - icon = QtGui.QIcon(icon_path) + icon_path = Path(icon_path) + try: + config = json.loads( + (icon_path.parent / 'desktop-app.json').read_text(encoding='utf8') + ) + except (OSError, json.JSONDecodeError): + application_name = None + else: + module_config = config.get('modules', {}).get(icon_path.stem, {}) + application_name = ( + module_config.get('short_display_name') + or module_config.get('display_name') + ) + if application_name: + qapplication.setApplicationName(application_name) + qapplication.setApplicationDisplayName(application_name) + icon = QtGui.QIcon(str(icon_path)) if not icon.isNull(): qapplication.setWindowIcon(icon) if qapplication.property('_labscript_qapplication_configured'): From b8989e341716bc5a9ec8dc19657f8bc7691cf442 Mon Sep 17 00:00:00 2001 From: spielman Date: Tue, 12 May 2026 11:12:22 -0400 Subject: [PATCH 38/66] Read macOS Qt app name from application property --- labscript_utils/splash.py | 31 +++++++++++-------------------- 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/labscript_utils/splash.py b/labscript_utils/splash.py index 8d8d7205..82370340 100644 --- a/labscript_utils/splash.py +++ b/labscript_utils/splash.py @@ -11,9 +11,7 @@ # # ##################################################################### -import json import sys -from pathlib import Path from labscript_utils import dedent try: @@ -45,25 +43,13 @@ def configure_qapplication(qapplication): """Apply labscript-wide QApplication configuration.""" qapplication.setAttribute(Qt.AA_DontShowIconsInMenus, False) if sys.platform == 'darwin': + application_name = qapplication.property('_labscript_application_name') + if application_name: + qapplication.setApplicationName(application_name) + qapplication.setApplicationDisplayName(application_name) icon_path = qapplication.property('_labscript_icon_path') if icon_path: - icon_path = Path(icon_path) - try: - config = json.loads( - (icon_path.parent / 'desktop-app.json').read_text(encoding='utf8') - ) - except (OSError, json.JSONDecodeError): - application_name = None - else: - module_config = config.get('modules', {}).get(icon_path.stem, {}) - application_name = ( - module_config.get('short_display_name') - or module_config.get('display_name') - ) - if application_name: - qapplication.setApplicationName(application_name) - qapplication.setApplicationDisplayName(application_name) - icon = QtGui.QIcon(str(icon_path)) + icon = QtGui.QIcon(icon_path) if not icon.isNull(): qapplication.setWindowIcon(icon) if qapplication.property('_labscript_qapplication_configured'): @@ -93,11 +79,16 @@ class Splash(QtWidgets.QFrame): BG = '#ffffff' FG = '#000000' - def __init__(self, imagepath): + def __init__(self, imagepath, application_name=None): + if application_name is not None: + QtCore.QCoreApplication.setApplicationName(application_name) + QtGui.QGuiApplication.setApplicationDisplayName(application_name) self.qapplication = QtWidgets.QApplication.instance() if self.qapplication is None: self.qapplication = QtWidgets.QApplication(sys.argv) self.qapplication.setProperty('_labscript_icon_path', imagepath) + if application_name is not None: + self.qapplication.setProperty('_labscript_application_name', application_name) configure_qapplication(self.qapplication) super().__init__() self.icon = QtGui.QPixmap() From a95f8835dc005a7d6b1fef202c4d65d8094339b1 Mon Sep 17 00:00:00 2001 From: spielman Date: Tue, 12 May 2026 11:39:41 -0400 Subject: [PATCH 39/66] Set macOS QApplication argv name from splash --- labscript_utils/splash.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/labscript_utils/splash.py b/labscript_utils/splash.py index 82370340..d284e8db 100644 --- a/labscript_utils/splash.py +++ b/labscript_utils/splash.py @@ -43,10 +43,6 @@ def configure_qapplication(qapplication): """Apply labscript-wide QApplication configuration.""" qapplication.setAttribute(Qt.AA_DontShowIconsInMenus, False) if sys.platform == 'darwin': - application_name = qapplication.property('_labscript_application_name') - if application_name: - qapplication.setApplicationName(application_name) - qapplication.setApplicationDisplayName(application_name) icon_path = qapplication.property('_labscript_icon_path') if icon_path: icon = QtGui.QIcon(icon_path) @@ -68,7 +64,6 @@ def configure_qapplication(qapplication): qapplication.setProperty('_labscript_qapplication_configured', True) return qapplication - class Splash(QtWidgets.QFrame): w = 250 h = 230 @@ -80,15 +75,14 @@ class Splash(QtWidgets.QFrame): FG = '#000000' def __init__(self, imagepath, application_name=None): - if application_name is not None: - QtCore.QCoreApplication.setApplicationName(application_name) - QtGui.QGuiApplication.setApplicationDisplayName(application_name) self.qapplication = QtWidgets.QApplication.instance() if self.qapplication is None: - self.qapplication = QtWidgets.QApplication(sys.argv) + argv = sys.argv + if application_name is not None: + # Create a new argv so QApplication can alter it without mutating sys.argv. + argv = [application_name] + argv[1:] + self.qapplication = QtWidgets.QApplication(argv) self.qapplication.setProperty('_labscript_icon_path', imagepath) - if application_name is not None: - self.qapplication.setProperty('_labscript_application_name', application_name) configure_qapplication(self.qapplication) super().__init__() self.icon = QtGui.QPixmap() From c5cbf710e28211cd80901e7c954427511afdc189 Mon Sep 17 00:00:00 2001 From: spielman Date: Tue, 12 May 2026 11:43:45 -0400 Subject: [PATCH 40/66] Configure splash icon through QApplication helper --- labscript_utils/splash.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/labscript_utils/splash.py b/labscript_utils/splash.py index 1a93223c..9d478dc3 100644 --- a/labscript_utils/splash.py +++ b/labscript_utils/splash.py @@ -64,7 +64,7 @@ def configure_qapplication(qapplication): qapplication.setProperty('_labscript_qapplication_configured', True) return qapplication -def get_qapplication(argv=None, application_name=None): +def get_qapplication(argv=None, application_name=None, icon_path=None): qapplication = QtWidgets.QApplication.instance() if qapplication is None: @@ -75,6 +75,8 @@ def get_qapplication(argv=None, application_name=None): qapplication = QtWidgets.QApplication(argv) + if icon_path is not None: + qapplication.setProperty('_labscript_icon_path', icon_path) return configure_qapplication(qapplication) @@ -89,9 +91,9 @@ class Splash(QtWidgets.QFrame): FG = '#000000' def __init__(self, imagepath, application_name=None): - self.qapplication = get_qapplication(application_name=application_name) - self.qapplication.setProperty('_labscript_icon_path', imagepath) - configure_qapplication(self.qapplication) + self.qapplication = get_qapplication( + application_name=application_name, icon_path=imagepath + ) super().__init__() self.icon = QtGui.QPixmap() self.icon.load(imagepath) From d5fccbf20397d8fd27f90498e8d13c1667d27838 Mon Sep 17 00:00:00 2001 From: spielman Date: Tue, 12 May 2026 11:43:45 -0400 Subject: [PATCH 41/66] Configure splash icon through QApplication helper --- labscript_utils/splash.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/labscript_utils/splash.py b/labscript_utils/splash.py index 1a93223c..9d478dc3 100644 --- a/labscript_utils/splash.py +++ b/labscript_utils/splash.py @@ -64,7 +64,7 @@ def configure_qapplication(qapplication): qapplication.setProperty('_labscript_qapplication_configured', True) return qapplication -def get_qapplication(argv=None, application_name=None): +def get_qapplication(argv=None, application_name=None, icon_path=None): qapplication = QtWidgets.QApplication.instance() if qapplication is None: @@ -75,6 +75,8 @@ def get_qapplication(argv=None, application_name=None): qapplication = QtWidgets.QApplication(argv) + if icon_path is not None: + qapplication.setProperty('_labscript_icon_path', icon_path) return configure_qapplication(qapplication) @@ -89,9 +91,9 @@ class Splash(QtWidgets.QFrame): FG = '#000000' def __init__(self, imagepath, application_name=None): - self.qapplication = get_qapplication(application_name=application_name) - self.qapplication.setProperty('_labscript_icon_path', imagepath) - configure_qapplication(self.qapplication) + self.qapplication = get_qapplication( + application_name=application_name, icon_path=imagepath + ) super().__init__() self.icon = QtGui.QPixmap() self.icon.load(imagepath) From 6dae69b897933708a127c3052364eea796644c8a Mon Sep 17 00:00:00 2001 From: spielman Date: Tue, 12 May 2026 11:46:05 -0400 Subject: [PATCH 42/66] Rename splash image path argument --- labscript_utils/splash.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/labscript_utils/splash.py b/labscript_utils/splash.py index 9d478dc3..46922905 100644 --- a/labscript_utils/splash.py +++ b/labscript_utils/splash.py @@ -90,15 +90,15 @@ class Splash(QtWidgets.QFrame): BG = '#ffffff' FG = '#000000' - def __init__(self, imagepath, application_name=None): + def __init__(self, icon_path, application_name=None): self.qapplication = get_qapplication( - application_name=application_name, icon_path=imagepath + application_name=application_name, icon_path=icon_path ) super().__init__() self.icon = QtGui.QPixmap() - self.icon.load(imagepath) + self.icon.load(icon_path) if self.icon.isNull(): - raise ValueError("Invalid image file: {}.\n".format(imagepath)) + raise ValueError("Invalid image file: {}.\n".format(icon_path)) self.icon = self.icon.scaled( self.imwidth, self.imheight, Qt.AspectRatioMode.KeepAspectRatio, Qt.TransformationMode.SmoothTransformation ) From 98f9f1704cb752144bb39daa20a1975838f04ebb Mon Sep 17 00:00:00 2001 From: spielman Date: Tue, 12 May 2026 11:46:05 -0400 Subject: [PATCH 43/66] Rename splash image path argument --- labscript_utils/splash.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/labscript_utils/splash.py b/labscript_utils/splash.py index 9d478dc3..46922905 100644 --- a/labscript_utils/splash.py +++ b/labscript_utils/splash.py @@ -90,15 +90,15 @@ class Splash(QtWidgets.QFrame): BG = '#ffffff' FG = '#000000' - def __init__(self, imagepath, application_name=None): + def __init__(self, icon_path, application_name=None): self.qapplication = get_qapplication( - application_name=application_name, icon_path=imagepath + application_name=application_name, icon_path=icon_path ) super().__init__() self.icon = QtGui.QPixmap() - self.icon.load(imagepath) + self.icon.load(icon_path) if self.icon.isNull(): - raise ValueError("Invalid image file: {}.\n".format(imagepath)) + raise ValueError("Invalid image file: {}.\n".format(icon_path)) self.icon = self.icon.scaled( self.imwidth, self.imheight, Qt.AspectRatioMode.KeepAspectRatio, Qt.TransformationMode.SmoothTransformation ) From 7862f9256ee04d81db6f2124ec3ef967c9dcc6db Mon Sep 17 00:00:00 2001 From: spielman Date: Tue, 12 May 2026 12:02:21 -0400 Subject: [PATCH 44/66] Configure macOS Qt application presentation --- labscript_utils/splash.py | 39 +++++++++++++++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/labscript_utils/splash.py b/labscript_utils/splash.py index 75cedf59..88f73cda 100644 --- a/labscript_utils/splash.py +++ b/labscript_utils/splash.py @@ -39,6 +39,31 @@ QtWidgets.QApplication.setAttribute(Qt.AA_UseHighDpiPixmaps, True) +def configure_qapplication(qapplication): + """Apply labscript-wide QApplication configuration.""" + qapplication.setAttribute(Qt.AA_DontShowIconsInMenus, False) + if sys.platform == 'darwin': + icon_path = qapplication.property('_labscript_icon_path') + if icon_path: + icon = QtGui.QIcon(icon_path) + if not icon.isNull(): + qapplication.setWindowIcon(icon) + if qapplication.property('_labscript_qapplication_configured'): + return qapplication + # Native macOS widget styling makes many Qt controls look inconsistent + # with the rest of the suite. Use Qt's own style, but preserve the + # current palette so dark/light appearance still follows the active + # theme. + palette = QtGui.QPalette(qapplication.palette()) + style = QtWidgets.QStyleFactory.create('Fusion') + if style is not None: + qapplication.setStyle(style) + qapplication.setPalette(palette) + elif qapplication.property('_labscript_qapplication_configured'): + return qapplication + qapplication.setProperty('_labscript_qapplication_configured', True) + return qapplication + class Splash(QtWidgets.QFrame): w = 250 h = 230 @@ -49,15 +74,21 @@ class Splash(QtWidgets.QFrame): BG = '#ffffff' FG = '#000000' - def __init__(self, imagepath): + def __init__(self, icon_path, application_name=None): self.qapplication = QtWidgets.QApplication.instance() if self.qapplication is None: - self.qapplication = QtWidgets.QApplication(sys.argv) + argv = sys.argv + if application_name is not None: + # Create a new argv so QApplication can alter it without mutating sys.argv. + argv = [application_name] + argv[1:] + self.qapplication = QtWidgets.QApplication(argv) + self.qapplication.setProperty('_labscript_icon_path', icon_path) + configure_qapplication(self.qapplication) super().__init__() self.icon = QtGui.QPixmap() - self.icon.load(imagepath) + self.icon.load(icon_path) if self.icon.isNull(): - raise ValueError("Invalid image file: {}.\n".format(imagepath)) + raise ValueError("Invalid image file: {}.\n".format(icon_path)) self.icon = self.icon.scaled( self.imwidth, self.imheight, Qt.AspectRatioMode.KeepAspectRatio, Qt.TransformationMode.SmoothTransformation ) From bdf25bb57773e4b8f54c19a9a0d61fe6b08e57ba Mon Sep 17 00:00:00 2001 From: spielman Date: Fri, 28 Aug 2026 14:44:49 -0400 Subject: [PATCH 45/66] Stringify zlock port for subprocess argv get_config() falls back to zprocess.zlock.DEFAULT_PORT, an int, whenever the labconfig has no [ports] zlock entry. That value was placed directly into the argv list passed to subprocess, which accepts only str, bytes or os.PathLike: TypeError: expected str, bytes or os.PathLike object, not int So labscript-zlock could not start from a labconfig that omits the port. The sibling launchers zlog.py and remote.py already wrap their ports with str(); do the same here. Co-Authored-By: Claude Opus 5 --- labscript_utils/zlock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/labscript_utils/zlock.py b/labscript_utils/zlock.py index 5878997f..baac341b 100644 --- a/labscript_utils/zlock.py +++ b/labscript_utils/zlock.py @@ -40,7 +40,7 @@ def main(): '-m', 'zprocess.zlock', '--port', - config['zlock_port'], + str(config['zlock_port']), '-l', LOG_PATH, ] From e760e0c9f1b4a93ac2d2e3a9c86a45282c169224 Mon Sep 17 00:00:00 2001 From: spielman Date: Fri, 28 Aug 2026 14:45:08 -0400 Subject: [PATCH 46/66] Capture the original showwarning once at import set_logger() stashed the currently-installed handler into warnings._showwarning before installing logwarning. On a second set_logger() call the stash therefore captured logwarning itself, so logwarning recursed into itself and the next warning raised: RecursionError: maximum recursion depth exceeded This is reachable whenever a process installs the labscript excepthook more than once, such as a subprocess that re-runs application startup or an app that reconfigures logging. Capture the real handler once at module import into a module-level name instead, which is also idempotent and stops writing to a private-looking attribute of the warnings module. Co-Authored-By: Claude Opus 5 --- labscript_utils/excepthook/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/labscript_utils/excepthook/__init__.py b/labscript_utils/excepthook/__init__.py index 9ac5975d..c6551fc8 100644 --- a/labscript_utils/excepthook/__init__.py +++ b/labscript_utils/excepthook/__init__.py @@ -28,6 +28,7 @@ class l: logger = None child_processes = [] +_original_showwarning = warnings.showwarning def install_thread_excepthook(): @@ -83,12 +84,11 @@ def tkhandler(exceptclass, exception, exec_info, reraise=True): def logwarning(message, category, filename, lineno, file=None, line=None): logmessage = warnings.formatwarning(message, category, filename, lineno, line) l.logger.warn(logmessage) - warnings._showwarning(message, category, filename, lineno, file, line) + _original_showwarning(message, category, filename, lineno, file, line) def set_logger(logger): l.logger = logger - warnings._showwarning = warnings.showwarning warnings.showwarning = logwarning # Check for tkinter availability. Tkinter is frustratingly not available From 5196548c5dfeb735aa590b055083a52ec910b56e Mon Sep 17 00:00:00 2001 From: spielman Date: Fri, 28 Aug 2026 14:45:17 -0400 Subject: [PATCH 47/66] Serialise numpy floating and boolean scalars _default() is json.dumps' fallback for objects it cannot encode, and only handled np.integer. np.float64 survives by accident because it subclasses Python float, which masked the gap for the other numpy scalar types: int64 ok float64 ok float32 TypeError bool_ TypeError Any device or plugin writing an np.float32 or np.bool_ into shot properties therefore failed in serialise(). Add the two missing branches. Co-Authored-By: Claude Opus 5 --- labscript_utils/properties.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/labscript_utils/properties.py b/labscript_utils/properties.py index 092df528..5f8488ed 100644 --- a/labscript_utils/properties.py +++ b/labscript_utils/properties.py @@ -61,6 +61,10 @@ def _default(o): # Workaround for https://bugs.python.org/issue24313 if isinstance(o, np.integer): return int(o) + if isinstance(o, np.floating): + return float(o) + if isinstance(o, np.bool_): + return bool(o) raise TypeError From 960fc65a8e6a4b10e39e55b926fc34cdc0e08296 Mon Sep 17 00:00:00 2001 From: spielman Date: Fri, 28 Aug 2026 14:45:39 -0400 Subject: [PATCH 48/66] Test socket_class against SecureSocket, not SecureContext The comment directly above states the intent: only insert the security-related kwargs when the socket is going to be a SecureSocket. The test used SecureContext, which is a Context class and can never be a socket class, so the branch was unreachable for any explicit socket_class: issubclass(zmq.Socket, SecureContext) = False SecureSocket = False issubclass(SecureSocket, SecureContext) = False SecureSocket = True The common pyzmq 25 path, where ThreadAuthenticator passes zmq.Socket, gives the intended result either way, which is why this went unnoticed. A caller that explicitly passes SecureSocket, however, silently lost its allow_insecure configuration. Co-Authored-By: Claude Opus 5 --- labscript_utils/ls_zprocess.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/labscript_utils/ls_zprocess.py b/labscript_utils/ls_zprocess.py index 4d9e9575..96424433 100644 --- a/labscript_utils/ls_zprocess.py +++ b/labscript_utils/ls_zprocess.py @@ -257,7 +257,7 @@ def socket(self, socket_type, socket_class=None, **kwargs): # be a SecureSocket. If caller has explicitly requested a different socket type # (e.g since pyzmq 25, ThreadAuthenticator sets up an internal socket by calling # `Context.socket(..., socket_class=zmq.Socket)), then don't.` - if socket_class is None or issubclass(socket_class, SecureContext): + if socket_class is None or issubclass(socket_class, SecureSocket): config = get_config() kwargs['allow_insecure'] = config['allow_insecure'] return SecureContext.socket(self, socket_type=socket_type, **kwargs) From 635aea9af3258362dde409e4b1f13242e72c63dc Mon Sep 17 00:00:00 2001 From: spielman Date: Fri, 28 Aug 2026 14:46:33 -0400 Subject: [PATCH 49/66] Check for a missing shared secret on both config paths The guard that raises _ERR_NO_SHARED_SECRET was nested inside the except clause, so it only ran when [security] allow_insecure was absent from the labconfig. A labconfig that explicitly sets [security] allow_insecure = False with no shared_secret skipped the check entirely. Inside the except the condition was also dead weight, since allow_insecure had just been set to False on the line above. _ERR_NO_SHARED_SECRET states the intended contract: configure a shared_secret, or opt out with allow_insecure = True. Explicitly writing allow_insecure = False without a secret is precisely the misconfiguration the message exists to catch, so run the guard on both paths. BEHAVIOUR CHANGE: an installation that explicitly sets allow_insecure = False with no shared_secret now fails at startup with the message above instead of continuing. Such a setup previously worked as long as all communication stayed on loopback, because zprocess only enforces this at send time (zprocess/security.py). Those users must now either supply a shared secret or set allow_insecure = True. This commit is deliberately kept separate so it can be dropped if that trade-off is unwanted. Co-Authored-By: Claude Opus 5 --- labscript_utils/ls_zprocess.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/labscript_utils/ls_zprocess.py b/labscript_utils/ls_zprocess.py index 96424433..6fd47459 100644 --- a/labscript_utils/ls_zprocess.py +++ b/labscript_utils/ls_zprocess.py @@ -106,8 +106,8 @@ def get_config(): config['allow_insecure'] = labconfig.getboolean('security', 'allow_insecure') except (labconfig.NoOptionError, labconfig.NoSectionError): config['allow_insecure'] = False - if config['shared_secret'] is None and not config['allow_insecure']: - raise ValueError(_ERR_NO_SHARED_SECRET.replace('/', os.sep)) + if config['shared_secret'] is None and not config['allow_insecure']: + raise ValueError(_ERR_NO_SHARED_SECRET.replace('/', os.sep)) try: config['logging_maxBytes'] = labconfig.getint('logging', 'maxBytes') except (labconfig.NoOptionError, labconfig.NoSectionError): From 679bb78bbce72730bd3db7c5aca9ed7db8066c06 Mon Sep 17 00:00:00 2001 From: spielman Date: Fri, 28 Aug 2026 15:15:12 -0400 Subject: [PATCH 50/66] Accept TOML arrays for list-valued labconfig options The TOML migration made LabConfig.get() return native types, so an option written the natural TOML way userlib = ['/srv/labscript/userlib', '/home/u/userlib'] comes back as a list. Two consumers still assumed the INI-era comma-separated string and called .split(','): AttributeError: 'list' object has no attribute 'split' labscript_profile.add_userlib_and_pythonlib() runs from labscript-suite.pth at *every* interpreter start, and the .pth guard only catches ModuleNotFoundError, so the failure escaped into site.addpackage(): Error processing line 1 of .../labscript-suite.pth: AttributeError: 'list' object has no attribute 'split' Remainder of file ignored Python still started, but userlib and pythonlib were never added to sys.path, so every labscriptlib sequence and lyse analysis routine lost its imports, with a traceback printed on every python invocation. _get_device_dirs() has the same defect, and its result is computed at module scope (LABSCRIPT_DEVICES_DIRS), so importing labscript_utils.device_registry raised outright, taking BLACS, runviewer, labscript compilation and labscript-devices with it. Add as_config_list() next to the other shared TOML config helpers and use it in both places. It accepts a comma-separated string, a TOML array, or a bare scalar, so existing INI-derived configs keep working unchanged. Co-Authored-By: Claude Opus 5 --- labscript_profile/__init__.py | 12 ++++++++---- labscript_profile/toml_config.py | 16 ++++++++++++++++ .../device_registry/_device_registry.py | 5 +++-- 3 files changed, 27 insertions(+), 6 deletions(-) diff --git a/labscript_profile/__init__.py b/labscript_profile/__init__.py index f2b12580..e9577611 100644 --- a/labscript_profile/__init__.py +++ b/labscript_profile/__init__.py @@ -8,7 +8,12 @@ import socket from getpass import getuser -from .toml_config import TomlConfigParser, dump_toml_file, load_toml_file +from .toml_config import ( + TomlConfigParser, + as_config_list, + dump_toml_file, + load_toml_file, +) # The contents of this file are imported every time the Python interpreter starts up, # owing to our custom .pth file that runs the below two functions. This ensures that @@ -68,9 +73,8 @@ def add_userlib_and_pythonlib(): paths = config.get('default', option) except (configparser.NoSectionError, configparser.NoOptionError): paths = '' - if paths: - for path in paths.split(','): - site.addsitedir(path) + for path in as_config_list(paths): + site.addsitedir(path) def ensure_labconfig(): diff --git a/labscript_profile/toml_config.py b/labscript_profile/toml_config.py index e5c6d756..f2385a80 100644 --- a/labscript_profile/toml_config.py +++ b/labscript_profile/toml_config.py @@ -8,6 +8,22 @@ import tomli_w +def as_config_list(value): + """Return a labconfig value as a list of stripped strings. + + TOML gives options native types, so a path-list option may arrive either as + a comma-separated string, as it always did under INI, or as a real TOML + array. Accept both, and treat anything else as a single entry. + """ + if value is None: + return [] + if isinstance(value, str): + return [item.strip() for item in value.split(',') if item.strip()] + if isinstance(value, (list, tuple)): + return [str(item).strip() for item in value if str(item).strip()] + return [str(value).strip()] + + def load_toml_file(filename): with open(filename, 'rb') as f: return tomllib.load(f) diff --git a/labscript_utils/device_registry/_device_registry.py b/labscript_utils/device_registry/_device_registry.py index 0e2348a0..d67ea11d 100644 --- a/labscript_utils/device_registry/_device_registry.py +++ b/labscript_utils/device_registry/_device_registry.py @@ -6,6 +6,7 @@ import inspect from labscript_utils import dedent from labscript_utils.labconfig import LabConfig +from labscript_profile.toml_config import as_config_list """This file contains the machinery for registering and looking up what BLACS tab and @@ -77,8 +78,8 @@ def _get_device_dirs(): user_devices = LabConfig().get('default', 'user_devices') except (LabConfig.NoOptionError, LabConfig.NoSectionError): user_devices = 'user_devices' - # Split on commas, remove whitespace: - user_devices = [s.strip() for s in user_devices.split(',')] + # Accept a comma-separated string or a TOML array: + user_devices = as_config_list(user_devices) return _get_import_paths(['labscript_devices'] + user_devices) From 7f3fc69971bbe6b855762e2b070c4c6c6b4ae928 Mon Sep 17 00:00:00 2001 From: spielman Date: Fri, 28 Aug 2026 15:16:16 -0400 Subject: [PATCH 51/66] Append rather than replace the app config extension _resolve_appconfig_save_path() used Path.with_suffix('.toml'), which replaces everything after the last dot rather than appending. That is only correct when the path already ends in an app config extension, and app config paths are routinely built from names that legitimately contain dots. lyse builds its plot-window settings path from the analysis script stem and loads it back as settings_path + '.toml', so the two disagreed: my_analysis.py save->lyse-my_analysis.toml load->lyse-my_analysis.toml same my.analysis.v2.py save->lyse-my.analysis.toml load->lyse-my.analysis.v2.toml MISMATCH rb.87.imaging.py save->lyse-rb.87.toml load->lyse-rb.87.imaging.toml MISMATCH Window geometry therefore never restored for those scripts. Worse, save_geometry() is a read-modify-write over a file it could not read, so each plot window erased every other window's geometry, and scripts sharing a prefix up to the last dot - rb.87.imaging.py and rb.87.absorption.py - collapsed onto one file and overwrote each other. runviewer's on_save_channel_config() had the same problem with user-typed filenames. Add appconfig_path_with_suffix(), which replaces only a recognised app config extension and appends otherwise, and route both the save and load resolvers through it. Fixing it here means lyse and runviewer need no change: their existing paths resolve identically to before, and the dotted cases now agree in both directions. _resolve_appconfig_load_path() collapses to the same three steps for every input, which also removes a branch whose two arms returned the same value. Co-Authored-By: Claude Opus 5 --- labscript_utils/labconfig.py | 42 +++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/labscript_utils/labconfig.py b/labscript_utils/labconfig.py index 398ac744..f49a7787 100644 --- a/labscript_utils/labconfig.py +++ b/labscript_utils/labconfig.py @@ -195,37 +195,39 @@ def _read_path(self, path): raise RuntimeError(msg) +# LEGACY INI COMPATIBILITY. The '.ini' entry is deprecated and will be removed. +APPCONFIG_SUFFIXES = ('.toml', '.ini') + + +def appconfig_path_with_suffix(path, suffix): + """Return ``path`` carrying ``suffix``. + + ``Path.with_suffix()`` replaces everything after the last dot, which is only + correct when the path already ends in an app config extension. App config + paths are routinely built from names that legitimately contain dots, such as + an analysis script called ``rb.87.imaging``, where replacing would silently + address a different file. Append in that case instead. + """ + path = Path(path) + if path.suffix.lower() in APPCONFIG_SUFFIXES: + return path.with_suffix(suffix) + return path.with_name(path.name + suffix) + + def _resolve_appconfig_load_path(filename): path = Path(filename) - suffix = path.suffix.lower() - # LEGACY INI COMPATIBILITY. DEPRECATED CODE, WILL BE REMOVED. - if suffix == '.ini': - toml_path = path.with_suffix('.toml') - if toml_path.exists(): - return toml_path - if path.exists(): - return path - return path - if suffix == '.toml': - if path.exists(): - return path - # LEGACY INI COMPATIBILITY. DEPRECATED CODE, WILL BE REMOVED. - legacy_path = path.with_suffix('.ini') - if legacy_path.exists(): - return legacy_path - return path - toml_path = path.with_suffix('.toml') + toml_path = appconfig_path_with_suffix(path, '.toml') if toml_path.exists(): return toml_path # LEGACY INI COMPATIBILITY. DEPRECATED CODE, WILL BE REMOVED. - legacy_path = path.with_suffix('.ini') + legacy_path = appconfig_path_with_suffix(path, '.ini') if legacy_path.exists(): return legacy_path return toml_path def _resolve_appconfig_save_path(filename): - return Path(filename).with_suffix('.toml') + return appconfig_path_with_suffix(filename, '.toml') # LEGACY INI COMPATIBILITY. DEPRECATED CODE, WILL BE REMOVED. def backup_legacy_config(path): From 8a1687b498e7264ee19ae876a4123cb504d44d44 Mon Sep 17 00:00:00 2001 From: spielman Date: Fri, 28 Aug 2026 15:17:08 -0400 Subject: [PATCH 52/66] Round-trip None options through TOML app configs The INI implementation stored None as pformat(None) and read it back with literal_eval, so None round-tripped. The TOML implementation rejects it: save_appconfig({'s': {'k': None}}) -> TypeError: s/k value None is not representable in TOML app config Nothing in any app's save_configuration() catches that, so a single None anywhere in the saved state turns "Save configuration" into an unhandled traceback. No current payload produces one, so this is a latent regression rather than a live failure, but it is a fuse on the save path for any new or plugin-contributed state. TOML has no null and expresses absence by omitting the key, so omit options whose value is None. load_appconfig() returns a mapping and every consumer reads optional state with .get(), which yields None again, so the round trip is preserved: {'k': None, 'kept': 1} -> file has only `kept` -> .get('k') is None None nested inside a list still raises, because dropping an element there would shift every later index. The message now says so explicitly instead of reporting it as an unrepresentable value. Co-Authored-By: Claude Opus 5 --- labscript_utils/labconfig.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/labscript_utils/labconfig.py b/labscript_utils/labconfig.py index f49a7787..942ce13b 100644 --- a/labscript_utils/labconfig.py +++ b/labscript_utils/labconfig.py @@ -263,6 +263,12 @@ def _to_toml_compatible(value, location='value'): ] if isinstance(value, (str, bool, int, float)): return value + if value is None: + # A None option is dropped by save_appconfig(), which never reaches + # here. Inside a list there is no way to drop one without shifting + # every later index, so that stays an error. + msg = f"{location} is None, which a TOML list cannot represent" + raise TypeError(msg) msg = f"{location} value {value!r} is not representable in TOML app config" raise TypeError(msg) @@ -278,11 +284,17 @@ def _load_legacy_appconfig_value(value): def save_appconfig(filename, data): - """Save a dictionary as a TOML app config.""" + """Save a dictionary as a TOML app config. + + TOML has no null, and represents absence by omitting the key. Options whose + value is ``None`` are therefore left out; ``load_appconfig()`` returns a + mapping, so callers reading them back with ``.get()`` see ``None`` again. + """ data = { section_name: { name: _to_toml_compatible(value, f"{section_name}/{name}") for name, value in section.items() + if value is not None } for section_name, section in data.items() } From a54da76a23ca5dadb740a652dbebd0f06c1f8579 Mon Sep 17 00:00:00 2001 From: spielman Date: Fri, 28 Aug 2026 15:17:26 -0400 Subject: [PATCH 53/66] Keep a default section when loading an app config save_appconfig() writes whatever sections it is given, but load_appconfig() dropped any table named 'default', so the round trip lost data with no error: saved {'default': {'k': 1}, 'other': {'j': 2}} loaded {'other': {'j': 2}} App configs are plain section/option documents. Unlike a labconfig they have no DEFAULT inheritance, so the name carries no special meaning and there is nothing to filter out. The INI implementation returned it too, so this also restores the earlier behaviour. No application currently writes a section by that name, so this is latent, but the asymmetry is a trap for anything that later does. Co-Authored-By: Claude Opus 5 --- labscript_utils/labconfig.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/labscript_utils/labconfig.py b/labscript_utils/labconfig.py index 942ce13b..56ab4db4 100644 --- a/labscript_utils/labconfig.py +++ b/labscript_utils/labconfig.py @@ -333,10 +333,14 @@ def load_appconfig(filename, return_save_path=False): backup_legacy_config(filename) else: raw = load_toml_file(filename) if filename.exists() else {} + # Every table is returned, including one named 'default'. App configs + # are plain section/option documents with no DEFAULT inheritance, so + # dropping that name here would silently discard a section + # save_appconfig() had just written. data = { section_name: dict(section.items()) for section_name, section in raw.items() - if section_name != 'default' and isinstance(section, dict) + if isinstance(section, dict) } if filename.suffix.lower() == '.toml': save_path = filename From 35b76df3ff5492e4de6ea00a13a58a651dafe187 Mon Sep 17 00:00:00 2001 From: spielman Date: Fri, 28 Aug 2026 15:18:20 -0400 Subject: [PATCH 54/66] Do not read config defaults as plugin enable flags PluginRefactor lifted this loop out of BLACS and reproduced the defect it carried. Config sections inherit the defaults section, so config.items(section) returns the section's own options plus every labconfig default. Against a real labconfig ten non-plugin keys are seen as configured plugins: analysislib, app_saved_configs, apparatus_name, experiment_shot_storage, labscript_suite, labscriptlib, pythonlib, shared_drive, user_devices, userlib A plugin directory whose name collides with one of those never had an enable flag written into the section, so getboolean() read the default's value instead: ValueError: Not a boolean: /some/path/userlib That call was unguarded, and blacs/plugins/__init__.py calls discover_modules() at module scope, so `import blacs.plugins` raised and BLACS did not start at all rather than skipping one plugin. Ask has_option() instead of scanning items(), and write a real flag into the section when the inherited value is not a boolean; a section option shadows the defaults, so the plugin resolves from then on. An installation that sets the flag in both places is unaffected, since the section value already wins: collision only in defaults before: ValueError after: starts, plugin disabled set in both places before: enabled after: enabled This is the labscript-utils counterpart of blacs 87c3774, which fixed the same code before PluginRefactor moved it here. Co-Authored-By: Claude Opus 5 --- labscript_utils/plugins.py | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/labscript_utils/plugins.py b/labscript_utils/plugins.py index 1033a20a..522f3c9d 100644 --- a/labscript_utils/plugins.py +++ b/labscript_utils/plugins.py @@ -1017,7 +1017,7 @@ def __init__( for example ``'myapp.plugins'``. plugins_dir (str): Filesystem path scanned for plugin directories. config: Config object with ``has_section()``, ``add_section()``, - ``items()``, ``set()``, and ``getboolean()`` methods. + ``has_option()``, ``set()``, and ``getboolean()`` methods. config_section (str): Section containing plugin enable/disable options. default_plugins (iterable): Plugin names enabled by default when @@ -1041,10 +1041,6 @@ def discover_modules(self): if not self.config.has_section(self.config_section): self.config.add_section(self.config_section) - configured_plugins = set( - name for name, val in self.config.items(self.config_section) - ) - modules = {} for module_name in os.listdir(self.plugins_dir): module_path = os.path.join(self.plugins_dir, module_name) @@ -1052,16 +1048,34 @@ def discover_modules(self): continue # Keep the config in sync with what is present on disk. - if module_name not in configured_plugins: + if not self.config.has_option(self.config_section, module_name): + self.config.set( + self.config_section, + module_name, + str(module_name in self.default_plugins), + ) + + try: + enabled = self.config.getboolean(self.config_section, module_name) + except ValueError: + # Config sections inherit the defaults section, so a config + # default whose name collides with a plugin directory is read + # here as that plugin's enable flag. Write a real flag into the + # section, which shadows the inherited value, rather than + # letting a path or other non-boolean default abort startup. + self.logger.warning( + "Plugin '%s' shares its name with a config default; using " + "the default enabled state." % module_name + ) self.config.set( self.config_section, module_name, str(module_name in self.default_plugins), ) - configured_plugins.add(module_name) + enabled = module_name in self.default_plugins # Only load activated plugins. - if self.config.getboolean(self.config_section, module_name): + if enabled: try: module = importlib.import_module( self.plugin_package + '.' + module_name From d6309993e7c8004cfc1139c1b764755f90d3620c Mon Sep 17 00:00:00 2001 From: spielman Date: Fri, 28 Aug 2026 15:18:44 -0400 Subject: [PATCH 55/66] Write plugin enable flags as booleans Newly discovered plugins were seeded with str(bool), which was harmless under INI where every value is a string, but writes a quoted string into a TOML config beside hand-written booleans: connection_table = true # written by hand cycle_time = "False" # written by discover_modules getboolean() still reads it correctly, so nothing broke, but the config file ends up internally inconsistent and a user editing it sees two spellings of the same flag. TomlConfigParser preserves native types, so pass the bool through: connection_table = true cycle_time = false theme = true Co-Authored-By: Claude Opus 5 --- labscript_utils/plugins.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/labscript_utils/plugins.py b/labscript_utils/plugins.py index 522f3c9d..8deebbc6 100644 --- a/labscript_utils/plugins.py +++ b/labscript_utils/plugins.py @@ -1052,7 +1052,7 @@ def discover_modules(self): self.config.set( self.config_section, module_name, - str(module_name in self.default_plugins), + module_name in self.default_plugins, ) try: @@ -1070,7 +1070,7 @@ def discover_modules(self): self.config.set( self.config_section, module_name, - str(module_name in self.default_plugins), + module_name in self.default_plugins, ) enabled = module_name in self.default_plugins From 4cf6f52c4c62dd5720e6820831bbcc0dd4a0eda7 Mon Sep 17 00:00:00 2001 From: spielman Date: Fri, 28 Aug 2026 15:19:26 -0400 Subject: [PATCH 56/66] Detect the plugin_setup_complete signature instead of retrying PluginRefactor carried BLACS's old-API fallback across unchanged: the retry is driven by catching Exception from the call itself, so any failure inside a correctly-written plugin_setup_complete(data) triggers a second invocation. That matters more here than it did in BLACS, because BasePlugin in this module declares exactly the signature that double-runs: def plugin_setup_complete(self, data=None) For that shape the retry re-enters the body, so whatever the first, partially completed call had already done - started a thread, registered a listener, opened a connection - happens again: plugin_setup_complete(self, data=None) before: side effect ran 2x after: side effect ran 1x Plugins with a strict arity were never double-run, because the retry failed on argument count before reaching the body. Bind the signature first and call once with the arguments it accepts. Old no-argument plugins still work and still get the deprecation warning; plugins with no introspectable signature fall back to the current API. A genuine error now produces one clear log entry instead of a misleading "Trying again with old call signature" followed by an arity error. This is the labscript-utils counterpart of blacs e3e5d2f. Co-Authored-By: Claude Opus 5 --- labscript_utils/plugins.py | 46 ++++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/labscript_utils/plugins.py b/labscript_utils/plugins.py index 8deebbc6..644fe1a3 100644 --- a/labscript_utils/plugins.py +++ b/labscript_utils/plugins.py @@ -586,6 +586,7 @@ def close(self): """ import importlib +import inspect import logging import os import warnings @@ -1274,30 +1275,41 @@ def setup_contexts(self, data): "Skipping." % module_name ) + def _setup_complete_args(self, module_name, plugin, data): + """Return the arguments ``plugin_setup_complete`` accepts. + + Old plugins define the hook without a ``data`` argument. Detecting that + by letting a call fail would re-run whatever side effects a hook that + got partway through had already performed, so bind the signature + instead and call it exactly once. + """ + try: + inspect.signature(plugin.plugin_setup_complete).bind(data) + except TypeError: + # Backwards compatibility for old plugins. + self.logger.warning( + "Plugin '%s' using old API. Please update " + "Plugin.plugin_setup_complete method to accept a " + "dictionary of blacs_data as the only argument." + % module_name + ) + return () + except ValueError: + # No introspectable signature; assume the current API. + pass + return (data,) + def setup_complete(self, data): """Notify plugins that the application has finished startup.""" for module_name, plugin in self.plugins.items(): + args = self._setup_complete_args(module_name, plugin, data) try: - plugin.plugin_setup_complete(data) + plugin.plugin_setup_complete(*args) except Exception: self.logger.exception( - "Error in plugin_setup_complete() for plugin '%s'. " - "Trying again with old call signature..." % module_name + "Plugin '%s' error. Plugin may not be functional." + % module_name ) - # Backwards compatibility for old plugins. - try: - plugin.plugin_setup_complete() - self.logger.warning( - "Plugin '%s' using old API. Please update " - "Plugin.plugin_setup_complete method to accept a " - "dictionary of blacs_data as the only argument." - % module_name - ) - except Exception: - self.logger.exception( - "Plugin '%s' error. Plugin may not be functional." - % module_name - ) def _defines_hook(self, plugin, method_name): """Return whether ``plugin`` overrides ``method_name`` meaningfully.""" From 890fc2563264f333b599236f5c81ed0ab27db2f4 Mon Sep 17 00:00:00 2001 From: spielman Date: Fri, 28 Aug 2026 15:20:09 -0400 Subject: [PATCH 57/66] Order menu groups deterministically and make render() idempotent Two problems in MenuContext.render(). Group order was assigned first-seen: if group not in group_orders[key]: group_orders[key][group] = len(group_orders[key]) First-seen order follows self.contributions, which follows self.plugins, which follows os.listdir(self.plugins_dir). Every other component of the sort key is deterministic, so this one made menu group order depend on the filesystem: discovery order 1 -> ['M-act', '---', 'A-act', '---', 'Z-act'] discovery order 2 -> ['Z-act', '---', 'A-act', '---', 'M-act'] (before) -> ['M-act', '---', 'A-act', '---', 'Z-act'] (after) Sort group names instead, with an ungrouped contribution first. render() also never cleared self.contributions, so calling it a second time duplicated every entry. Consume them, which leaves a repeat call a no-op. Co-Authored-By: Claude Opus 5 --- labscript_utils/plugins.py | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/labscript_utils/plugins.py b/labscript_utils/plugins.py index 644fe1a3..48b1f998 100644 --- a/labscript_utils/plugins.py +++ b/labscript_utils/plugins.py @@ -885,11 +885,16 @@ def add(self, plugin_name, contribution, data): self.contributions.append((plugin_name, contribution)) def render(self): - """Render all collected menu contributions into registered locations.""" + """Render all collected menu contributions into registered locations. + + Collected contributions are consumed, so calling this twice does not + duplicate every menu entry. + """ + contributions, self.contributions = self.contributions, [] grouped = {} group_orders = {} - for plugin_name, contribution in self.contributions: + for plugin_name, contribution in contributions: if not isinstance(contribution, dict): self.logger.error( "Menu contribution from plugin '%s' is not a dictionary. " @@ -924,14 +929,22 @@ def render(self): path = tuple(path) key = (location, path) - group = contribution.get('group', None) - if key not in group_orders: - group_orders[key] = {} - if group not in group_orders[key]: - group_orders[key][group] = len(group_orders[key]) - + group_orders.setdefault(key, set()).add(contribution.get('group', None)) grouped.setdefault(key, []).append((plugin_name, contribution)) + # Order groups by name rather than by the order plugins happened to be + # discovered, which follows os.listdir() and so varies between machines. + # An ungrouped contribution sorts first. + group_orders = { + key: { + group: index + for index, group in enumerate( + sorted(groups, key=lambda g: (g is not None, str(g))) + ) + } + for key, groups in group_orders.items() + } + menus = {} for name, menu in self.locations.items(): menus[(name, ())] = menu From 6b7a9d0337b0784304f7761d8b8c45029e3fae5f Mon Sep 17 00:00:00 2001 From: spielman Date: Fri, 28 Aug 2026 15:20:29 -0400 Subject: [PATCH 58/66] Materialise plugin contributions before testing them _get_contributions() accepts any iterable, including a generator, but setup_contexts() then tests it for emptiness: if menu_contributions and 'menus' not in self.contexts: self.logger.error("... provided menu contributions, but no 'menus' context is registered. Skipping.") continue A generator is truthy even when it yields nothing, so a plugin that contributed no menus was reported as having contributed some, and the continue skipped the rest of that plugin's routing. Return a list, which makes emptiness meaningful and lets the result be iterated more than once. A generator that raises while being consumed is logged like any other malformed contribution rather than escaping into the caller. Co-Authored-By: Claude Opus 5 --- labscript_utils/plugins.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/labscript_utils/plugins.py b/labscript_utils/plugins.py index 48b1f998..a4a85ed2 100644 --- a/labscript_utils/plugins.py +++ b/labscript_utils/plugins.py @@ -1217,7 +1217,17 @@ def _get_contributions(self, plugin, module_name, method_name, label): ) return [] - return contributions + # Materialise so that callers can test emptiness and iterate. A + # generator is truthy even when it yields nothing, which would report a + # plugin that contributed none as having contributed some. + try: + return list(contributions) + except Exception: + self.logger.exception( + "Error reading %s contributions from plugin '%s'. Skipping." + % (label, module_name) + ) + return [] def setup_contexts(self, data): """Route plugin-declared UI and menu contributions to app contexts.""" From 348442248be6c7fbd2b45b5112806b7b6568e77a Mon Sep 17 00:00:00 2001 From: spielman Date: Fri, 28 Aug 2026 15:22:03 -0400 Subject: [PATCH 59/66] Make the plugin tests runnable and cover the discovery regressions Two problems, one of which explains the other. tests/test_plugins.py uses the pytest caplog fixture, but pytest was not declared anywhere - there was only a docs extra - so there was no supported way to run these tests at all. More importantly, FakeConfig diverged from a real LabConfig in exactly the two ways that matter for plugin discovery. It returned only a section's own options, modelling none of the defaults inheritance that caused the defaults-as-plugin-names bug, and its getboolean() called .lower() on the value, assuming the INI-era strings rather than the native booleans TOML returns. A fake that cannot express the bug cannot catch it. Add a test extra with pytest, teach FakeConfig both behaviours, and add regression tests for the fixes in this series: - a config default is never read as a plugin's enable flag - an explicit flag set in both places still wins - seeded flags are written as booleans - an optional-argument plugin_setup_complete() runs exactly once - empty generator contributions do not report a missing context - menu group order is independent of discovery order - render() is idempotent The pre-existing discovery test asserted the old str(bool) seeding and is updated to expect real booleans, matching the change in d630999. 22 passed Co-Authored-By: Claude Opus 5 --- pyproject.toml | 3 + tests/test_plugins.py | 191 ++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 189 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index fed85e39..8a93a780 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,6 +56,9 @@ Downloads = "https://github.com/labscript-suite/labscript-utils/releases/" Tracker = "https://github.com/labscript-suite/labscript-utils/issues/" [project.optional-dependencies] +test = [ + "pytest>=7.0", +] docs = [ "PyQt5", "Sphinx==8.2.3", diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 47fad04a..795a80f9 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -18,8 +18,16 @@ class FakeConfig(object): - def __init__(self): + """Minimal stand-in for LabConfig, including the two behaviours that matter. + + A real config inherits its defaults section into every section, and returns + native TOML types rather than strings. Modelling neither is what let the + defaults-as-plugin-names bug survive in the first place. + """ + + def __init__(self, defaults=None): self.sections = {} + self.defaults = dict(defaults or {}) def has_section(self, name): return name in self.sections @@ -28,13 +36,30 @@ def add_section(self, name): self.sections[name] = {} def items(self, name): - return list(self.sections[name].items()) + merged = dict(self.defaults) + merged.update(self.sections[name]) + return list(merged.items()) + + def has_option(self, section, option): + return option in self.sections.get(section, {}) or option in self.defaults + + def get(self, section, option): + if option in self.sections.get(section, {}): + return self.sections[section][option] + return self.defaults[option] def set(self, section, option, value): self.sections[section][option] = value def getboolean(self, section, option): - return self.sections[section][option].lower() == 'true' + value = self.get(section, option) + if isinstance(value, bool): + return value + if str(value).lower() in ('true', 'yes', 'on', '1'): + return True + if str(value).lower() in ('false', 'no', 'off', '0'): + return False + raise ValueError('Not a boolean: %s' % value) def test_callback_binds_as_method_and_keeps_priority(): @@ -117,8 +142,8 @@ def test_discovery_defaults_config_and_imports_only_enabled_plugins(tmp_path): modules = manager.discover_modules() assert set(modules) == {'enabled'} - assert config.sections['app/plugins']['enabled'] == 'True' - assert config.sections['app/plugins']['disabled'] == 'False' + assert config.sections['app/plugins']['enabled'] is True + assert config.sections['app/plugins']['disabled'] is False def test_instantiate_plugins_uses_saved_settings(): @@ -653,3 +678,159 @@ def save_action(): assert tools_menu.items[0][1].name == 'Tool' assert 'missing' in caplog.text.lower() assert 'plugin_malformed' in caplog.text + + +def _make_plugin_dirs(tmp_path, names): + for name in names: + (tmp_path / name).mkdir(parents=True) + return str(tmp_path) + + +def test_discover_modules_ignores_config_defaults(tmp_path, caplog): + """A config default must never be read as a plugin's enable flag.""" + config = FakeConfig(defaults={'userlib': '/some/path/userlib'}) + plugins_dir = _make_plugin_dirs(tmp_path, ['userlib']) + logger = logging.getLogger('test.plugins.discover.defaults') + manager = PluginManager( + 'package.plugins', plugins_dir, config, 'app/plugins', + default_plugins=(), logger=logger, + ) + + with caplog.at_level(logging.WARNING, logger=logger.name): + modules = manager.discover_modules() + + # Before the fix this raised ValueError: Not a boolean: /some/path/userlib + assert modules == {} + assert config.sections['app/plugins']['userlib'] is False + assert 'shares its name with a config default' in caplog.text + + +def test_discover_modules_keeps_a_flag_set_in_both_places(tmp_path): + """A section option shadows the default, so an explicit flag still wins.""" + config = FakeConfig(defaults={'userlib': '/some/path/userlib'}) + config.sections['app/plugins'] = {'userlib': True} + plugins_dir = _make_plugin_dirs(tmp_path, ['userlib']) + manager = PluginManager( + 'package.plugins', plugins_dir, config, 'app/plugins', + default_plugins=(), logger=logging.getLogger('test.plugins.discover.both'), + ) + + manager.discover_modules() + + assert config.sections['app/plugins']['userlib'] is True + + +def test_discover_modules_seeds_flags_as_booleans(tmp_path): + config = FakeConfig() + plugins_dir = _make_plugin_dirs(tmp_path, ['enabled_one', 'disabled_one']) + manager = PluginManager( + 'package.plugins', plugins_dir, config, 'app/plugins', + default_plugins=('enabled_one',), + logger=logging.getLogger('test.plugins.discover.bools'), + ) + + manager.discover_modules() + + assert config.sections['app/plugins']['enabled_one'] is True + assert config.sections['app/plugins']['disabled_one'] is False + + +def test_setup_complete_runs_optional_argument_hook_once(): + """The BasePlugin signature must not be re-run by an old-API retry.""" + calls = [] + + class OptionalArgumentPlugin(BasePlugin): + def plugin_setup_complete(self, data=None): + calls.append(data) + raise RuntimeError('failed after the side effect') + + manager = make_manager(logger=logging.getLogger('test.plugins.setup.once')) + manager.plugins = {'optional': OptionalArgumentPlugin({})} + data = {'settings': object()} + + manager.setup_complete(data) + + assert calls == [data] + + +def test_empty_generator_contributions_do_not_report_a_missing_context(): + class GeneratorPlugin(BasePlugin): + def get_menu_contributions(self): + return (contribution for contribution in []) + + logger = logging.getLogger('test.plugins.contributions.generator') + manager = make_manager(logger=logger) + manager.plugins = {'gen': GeneratorPlugin({})} + manager.contexts = {} + + records = [] + handler = logging.Handler() + handler.emit = lambda record: records.append(record.getMessage()) + logger.addHandler(handler) + try: + manager.setup_contexts({}) + finally: + logger.removeHandler(handler) + + assert not [message for message in records if 'menus' in message] + + +def test_menu_context_group_order_is_independent_of_discovery_order(): + def render_with(order): + menu = _RecordingMenu() + context = MenuContext(logger=logging.getLogger('test.plugins.menus.order')) + context.register_location('file', menu) + for plugin_name, group, name in order: + context.add( + plugin_name, + {'location': 'file', 'group': group, 'name': name}, + {}, + ) + context.render() + return menu.entries + + order = [('zeta', 'beta', 'Z'), ('alpha', 'alpha', 'A'), ('mid', None, 'M')] + assert render_with(order) == render_with(list(reversed(order))) + + +def test_menu_context_render_is_idempotent(): + menu = _RecordingMenu() + context = MenuContext(logger=logging.getLogger('test.plugins.menus.idempotent')) + context.register_location('file', menu) + context.add('one', {'location': 'file', 'name': 'A'}, {}) + + context.render() + entries_after_first = list(menu.entries) + context.render() + + assert menu.entries == entries_after_first + + +class _RecordingAction(object): + def __init__(self): + self.triggered = SimpleNamespace(connect=lambda callback: None) + + def setShortcut(self, shortcut): + pass + + def setCheckable(self, checkable): + pass + + def setEnabled(self, enabled): + pass + + +class _RecordingMenu(object): + def __init__(self): + self.entries = [] + + def addAction(self, *args): + self.entries.append(args[-1]) + return _RecordingAction() + + def addSeparator(self): + self.entries.append('---') + + def addMenu(self, name): + self.entries.append(('submenu', name)) + return _RecordingMenu() From 650646d5595d6c3ac6e4478af5d7158ac7330a9e Mon Sep 17 00:00:00 2001 From: spielman Date: Fri, 28 Aug 2026 15:25:30 -0400 Subject: [PATCH 60/66] Make preserved-lookup output safe to format again runmanager formats these templates in two passes: once in new_sequence_details() with preserve_unresolved_roots=('globals',), and again per shot to resolve globals. The first pass un-escaped braces, as any formatter does, so its output was not a valid template for the second: 'run{{special}}/{sequence_index:04d}' pass1='run{special}/0007' pass2-> UnresolvedLookupError: special A resolved value containing a brace failed the same way, since the second pass read it as a field: script_basename='my{script}' pass1='my{script}/0007' pass2-> UnresolvedLookupError: script Preserved placeholders only mean anything to a later pass, so enabling preservation is exactly the signal that this pass is producing a template. In that case escape literal text and resolved values, leaving preserved placeholders parseable; with no preservation the result is final text and is emitted verbatim. 'run{{special}}/{sequence_index:04d}' pass1='run{{special}}/0007' pass2='run{special}/0007' script_basename='my{script}' pass2='my{script}/0007' Output for templates without literal braces is unchanged, which covers every shipped example. One consequence worth knowing: runmanager shows the first-pass result in its shot output folder field, so a template written with literal {{ }} now displays them escaped there. That only affects templates that contain literal braces, which previously could not be used at all. _UnresolvedLookupError is renamed UnresolvedLookupField and is now public, so callers can catch an unresolved lookup by name. Co-Authored-By: Claude Opus 5 --- labscript_utils/lookup_format.py | 36 +++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/labscript_utils/lookup_format.py b/labscript_utils/lookup_format.py index 51d1cb73..68040db8 100644 --- a/labscript_utils/lookup_format.py +++ b/labscript_utils/lookup_format.py @@ -18,7 +18,7 @@ class InvalidLookupFormatField(ValueError): """Raised when a lookup-format field is not valid lookup-only syntax.""" -class _UnresolvedLookupError(LookupError): +class UnresolvedLookupField(LookupError): """Raised when a lookup could not be resolved from the provided context.""" def __init__(self, root_name): @@ -26,6 +26,11 @@ def __init__(self, root_name): self.root_name = root_name +def _escape_braces(text): + """Escape braces so ``text`` survives a further formatting pass verbatim.""" + return text.replace('{', '{{').replace('}', '}}') + + def format_lookup_string( template, context, @@ -42,6 +47,10 @@ def format_lookup_string( preserve_unresolved (bool): If True, unresolved lookups are left intact. preserve_unresolved_roots (iterable): Root names that may be preserved when a lookup below them is unresolved. + + Preserving unresolved lookups makes the result a template for a further pass, + so literal text and resolved values are brace-escaped to survive it. Without + preservation the result is final text and is emitted verbatim. """ if dt is not None: template = dt.strftime(template) @@ -63,12 +72,18 @@ def __init__( self.context = context self.preserve_unresolved = preserve_unresolved self.preserve_unresolved_roots = set(preserve_unresolved_roots) + # Preserved placeholders only mean anything to a later pass, so when any + # preservation is enabled this pass is producing a template rather than + # final text. + self.produces_template = bool(preserve_unresolved or self.preserve_unresolved_roots) self.formatter = string.Formatter() - def format(self, template): + def format(self, template, produces_template=None): + if produces_template is None: + produces_template = self.produces_template parts = [] for literal_text, field_name, format_spec, conversion in self.formatter.parse(template): - parts.append(literal_text) + parts.append(_escape_braces(literal_text) if produces_template else literal_text) if field_name is None: continue placeholder = self._reconstruct_placeholder( @@ -79,10 +94,15 @@ def format(self, template): if conversion: value = self.formatter.convert_field(value, conversion) if format_spec: - format_spec = self.format(format_spec) - parts.append(self.formatter.format_field(value, format_spec)) - except _UnresolvedLookupError as exc: + # A format spec is consumed by this pass, never re-parsed. + format_spec = self.format(format_spec, produces_template=False) + formatted = self.formatter.format_field(value, format_spec) + # A resolved value containing a brace would otherwise be read as + # a field by the next pass. + parts.append(_escape_braces(formatted) if produces_template else formatted) + except UnresolvedLookupField as exc: if self.preserve_unresolved or exc.root_name in self.preserve_unresolved_roots: + # The placeholder must stay parseable for the next pass. parts.append(placeholder) else: raise @@ -93,12 +113,12 @@ def _resolve_field(self, field_name): try: value = self.context[root_name] except KeyError: - raise _UnresolvedLookupError(root_name) + raise UnresolvedLookupField(root_name) for lookup in lookups: try: value = value[lookup] except (IndexError, KeyError, TypeError): - raise _UnresolvedLookupError(root_name) + raise UnresolvedLookupField(root_name) return value @staticmethod From a220994392c501aa696f4a2f22ea5170b6a4c13c Mon Sep 17 00:00:00 2001 From: spielman Date: Fri, 28 Aug 2026 15:42:54 -0400 Subject: [PATCH 61/66] Add a public inverse for template brace escaping format_lookup_string() escapes braces when preservation is enabled, since the result is a template for a further pass that will unescape them again. A caller that displays the intermediate rather than formatting it needs to do that unescaping itself: 'run{{special}}/{sequence_index:04d}' -> template 'run{{special}}/0007' display 'run{special}/0007' Expose unescape_braces() beside _escape_braces() so the pair stays in sync, rather than having callers hand-roll the inverse. Preserved placeholders are untouched, so a preview still shows unresolved globals[...] lookups as written. Co-Authored-By: Claude Opus 5 --- labscript_utils/lookup_format.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/labscript_utils/lookup_format.py b/labscript_utils/lookup_format.py index 68040db8..0e9e26df 100644 --- a/labscript_utils/lookup_format.py +++ b/labscript_utils/lookup_format.py @@ -31,6 +31,16 @@ def _escape_braces(text): return text.replace('{', '{{').replace('}', '}}') +def unescape_braces(text): + """Undo the brace escaping applied when producing a template. + + Use this to display a partially formatted template, whose remaining passes + will otherwise do the unescaping. Kept beside :func:`_escape_braces` so the + two cannot drift apart. + """ + return text.replace('{{', '{').replace('}}', '}') + + def format_lookup_string( template, context, From 451d42366f76130e0ac0a35843c2a3c2ec4ce85c Mon Sep 17 00:00:00 2001 From: spielman Date: Fri, 28 Aug 2026 16:28:45 -0400 Subject: [PATCH 62/66] Add an opt-out for the graphical error dialog Each unhandled exception spawns a tkinter subprocess window, up to MAX_WINDOWS at a time. All four GUI apps and lyse's analysis subprocess import this module in their first few lines, so a debugging run can leave many windows to close by hand. Guard the window-spawning block on LABSCRIPT_NO_ERROR_DIALOG (or the NO_ERROR_DIALOG module attribute, for toggling at runtime). The guard sits inside tkhandler rather than around the sys.excepthook installation, so exceptions are still logged and still reach stderr when the dialog is off. The default is unchanged: standard labscript runs keep the dialog. Co-Authored-By: Claude Opus 5 --- labscript_utils/excepthook/__init__.py | 27 ++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/labscript_utils/excepthook/__init__.py b/labscript_utils/excepthook/__init__.py index c6551fc8..4a4b6ed0 100644 --- a/labscript_utils/excepthook/__init__.py +++ b/labscript_utils/excepthook/__init__.py @@ -21,6 +21,12 @@ # The maximum number of windows the excepthook will spawn: MAX_WINDOWS = 10 +# Set the LABSCRIPT_NO_ERROR_DIALOG environment variable, or set this to True at +# runtime, to stop exceptions spawning graphical error windows. Exceptions are +# still logged and printed to stderr, which is what you usually want when +# running under a debugger: +NO_ERROR_DIALOG = bool(os.environ.get('LABSCRIPT_NO_ERROR_DIALOG')) + subprocess_script_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'tk_exception.py') @@ -67,16 +73,17 @@ def tkhandler(exceptclass, exception, exec_info, reraise=True): if exceptclass in [KeyboardInterrupt, SystemExit]: sys.__excepthook__(exceptclass, exception, exec_info) else: - for previous_process in child_processes[:]: - if previous_process.poll() is not None: - child_processes.remove(previous_process) - if len(child_processes) >= MAX_WINDOWS: - shortmessage = "Too many errors" - longmessage = ("Too many errors: Further errors will " + - "not be shown graphically until some error windows are closed") - if len(child_processes) < MAX_WINDOWS + 1: - process = subprocess.Popen([sys.executable, subprocess_script_path, script, shortmessage, longmessage]) - child_processes.append(process) + if not NO_ERROR_DIALOG: + for previous_process in child_processes[:]: + if previous_process.poll() is not None: + child_processes.remove(previous_process) + if len(child_processes) >= MAX_WINDOWS: + shortmessage = "Too many errors" + longmessage = ("Too many errors: Further errors will " + + "not be shown graphically until some error windows are closed") + if len(child_processes) < MAX_WINDOWS + 1: + process = subprocess.Popen([sys.executable, subprocess_script_path, script, shortmessage, longmessage]) + child_processes.append(process) if reraise: sys.__excepthook__(exceptclass, exception, exec_info) From 9f157a33d41036edf2c826cdceb49be7c4a79472 Mon Sep 17 00:00:00 2001 From: spielman Date: Fri, 28 Aug 2026 16:28:45 -0400 Subject: [PATCH 63/66] Add an opt-out for the graphical error dialog Each unhandled exception spawns a tkinter subprocess window, up to MAX_WINDOWS at a time. All four GUI apps and lyse's analysis subprocess import this module in their first few lines, so a debugging run can leave many windows to close by hand. Guard the window-spawning block on LABSCRIPT_NO_ERROR_DIALOG (or the NO_ERROR_DIALOG module attribute, for toggling at runtime). The guard sits inside tkhandler rather than around the sys.excepthook installation, so exceptions are still logged and still reach stderr when the dialog is off. The default is unchanged: standard labscript runs keep the dialog. Co-Authored-By: Claude Opus 5 --- labscript_utils/excepthook/__init__.py | 27 ++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/labscript_utils/excepthook/__init__.py b/labscript_utils/excepthook/__init__.py index c6551fc8..4a4b6ed0 100644 --- a/labscript_utils/excepthook/__init__.py +++ b/labscript_utils/excepthook/__init__.py @@ -21,6 +21,12 @@ # The maximum number of windows the excepthook will spawn: MAX_WINDOWS = 10 +# Set the LABSCRIPT_NO_ERROR_DIALOG environment variable, or set this to True at +# runtime, to stop exceptions spawning graphical error windows. Exceptions are +# still logged and printed to stderr, which is what you usually want when +# running under a debugger: +NO_ERROR_DIALOG = bool(os.environ.get('LABSCRIPT_NO_ERROR_DIALOG')) + subprocess_script_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'tk_exception.py') @@ -67,16 +73,17 @@ def tkhandler(exceptclass, exception, exec_info, reraise=True): if exceptclass in [KeyboardInterrupt, SystemExit]: sys.__excepthook__(exceptclass, exception, exec_info) else: - for previous_process in child_processes[:]: - if previous_process.poll() is not None: - child_processes.remove(previous_process) - if len(child_processes) >= MAX_WINDOWS: - shortmessage = "Too many errors" - longmessage = ("Too many errors: Further errors will " + - "not be shown graphically until some error windows are closed") - if len(child_processes) < MAX_WINDOWS + 1: - process = subprocess.Popen([sys.executable, subprocess_script_path, script, shortmessage, longmessage]) - child_processes.append(process) + if not NO_ERROR_DIALOG: + for previous_process in child_processes[:]: + if previous_process.poll() is not None: + child_processes.remove(previous_process) + if len(child_processes) >= MAX_WINDOWS: + shortmessage = "Too many errors" + longmessage = ("Too many errors: Further errors will " + + "not be shown graphically until some error windows are closed") + if len(child_processes) < MAX_WINDOWS + 1: + process = subprocess.Popen([sys.executable, subprocess_script_path, script, shortmessage, longmessage]) + child_processes.append(process) if reraise: sys.__excepthook__(exceptclass, exception, exec_info) From dd8ebbb14d5c3c49c786e36dd23731b601e8312c Mon Sep 17 00:00:00 2001 From: spielman Date: Sat, 29 Aug 2026 11:48:50 -0400 Subject: [PATCH 64/66] Migrate the Qt binding to PyQt6 Point the docs extra at PyQt6 and raise two dependency floors that were too low to actually support it: qtutils to 4.1.3, the release that added the PyQt6 shim and what the rest of the suite already requires, and pyqtgraph to 0.13.0. Co-Authored-By: Claude Opus 5 --- pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8a93a780..e07e31f3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,8 +34,8 @@ dependencies = [ "h5py>=2.9", "numpy>=1.15", "packaging>=20.4", - "pyqtgraph>=0.11.0rc0", - "qtutils>=4.0", + "pyqtgraph>=0.13.0", + "qtutils>=4.1.3", "scipy", "zprocess>=2.18.0", "setuptools_scm>=4.1.0", @@ -60,7 +60,7 @@ test = [ "pytest>=7.0", ] docs = [ - "PyQt5", + "PyQt6", "Sphinx==8.2.3", "sphinx-rtd-theme==3.0.1", "myst_parser==4.0.1", From 229e485ecd91ed49895092ec117774696f8caa43 Mon Sep 17 00:00:00 2001 From: spielman Date: Sat, 29 Aug 2026 13:06:27 -0400 Subject: [PATCH 65/66] Drop PyQt5 support and fix the Qt4-era wheel events The suite targets PyQt6 only, so remove the version gates rather than carrying two code paths: - splash.py: the PyQt5-only AA_EnableHighDpiScaling / AA_UseHighDpiPixmaps block; both are default in Qt6 and warn if set. - enumoutput.py, digitaloutput.py: the 'QT_ENV != PYQT5' gate on the scroll-wheel passthrough. - dragdroptab.py: debug prints for PyQt4/PyQt5. The wheel passthrough those gates guarded had been dead since Qt4 -- it called pos(), globalPos(), delta() and orientation() on a QWheelEvent, and delta()/orientation() were removed back in Qt5, so the branch raised AttributeError on every binding. Rebuilt on the Qt6 API (position(), globalPosition(), pixelDelta(), angleDelta(), phase(), inverted()); the QPointF conversion is required because mapToParent() returns a QPoint. Also switch the shot-queue context menu from QMenu.exec_() to exec(): qtutils only aliases exec_ onto QCoreApplication and QDialog, so QMenu raised AttributeError on right-click under PyQt6. Co-Authored-By: Claude Opus 5 --- labscript_utils/qtwidgets/digitaloutput.py | 14 +++++++++----- labscript_utils/qtwidgets/dragdroptab.py | 3 +-- labscript_utils/qtwidgets/enumoutput.py | 14 +++++++++----- labscript_utils/qtwidgets/imageoutput.py | 10 +++++++--- labscript_utils/qtwidgets/shotqueue.py | 2 +- labscript_utils/splash.py | 12 +----------- 6 files changed, 28 insertions(+), 27 deletions(-) diff --git a/labscript_utils/qtwidgets/digitaloutput.py b/labscript_utils/qtwidgets/digitaloutput.py index 451ccdb7..20369d34 100644 --- a/labscript_utils/qtwidgets/digitaloutput.py +++ b/labscript_utils/qtwidgets/digitaloutput.py @@ -12,7 +12,7 @@ ##################################################################### import sys -from qtutils.qt import QtCore, QtGui, QtWidgets, QT_ENV, PYQT5 +from qtutils.qt import QtCore, QtGui, QtWidgets class DigitalOutput(QtWidgets.QPushButton): @@ -50,10 +50,14 @@ def eventFilter(self, obj, event): # pass scrollwheel events of disabled buttons through to the parent # code adapted from: http://www.qtforum.org/article/28540/disabled-widgets-and-wheel-events.html - elif obj and not obj.isEnabled() and event.type() == QtCore.QEvent.Type.Wheel and QT_ENV != PYQT5: - newEvent = QtGui.QWheelEvent(obj.mapToParent(event.pos()), event.globalPos(), - event.delta(), event.buttons(), - event.modifiers(), event.orientation()) + elif obj and not obj.isEnabled() and event.type() == QtCore.QEvent.Type.Wheel: + newEvent = QtGui.QWheelEvent( + QtCore.QPointF(obj.mapToParent(event.position().toPoint())), + event.globalPosition(), + event.pixelDelta(), event.angleDelta(), + event.buttons(), event.modifiers(), + event.phase(), event.inverted(), + ) QtWidgets.QApplication.instance().postEvent(obj.parent(), newEvent) return True diff --git a/labscript_utils/qtwidgets/dragdroptab.py b/labscript_utils/qtwidgets/dragdroptab.py index 10e9a3cc..c7626eb6 100644 --- a/labscript_utils/qtwidgets/dragdroptab.py +++ b/labscript_utils/qtwidgets/dragdroptab.py @@ -39,8 +39,7 @@ def wrapper(*args, **kwargs): if debug.DEBUG: import sys print('sys.version:', sys.version) - print('PyQt4:', 'PyQt4' in sys.modules) - print('PyQt5:', 'PyQt5' in sys.modules) + print('PyQt6:', 'PyQt6' in sys.modules) print('PySide:', 'PySide' in sys.modules) print('qtutils:', 'qtutils' in sys.modules) print('qtutils.qt:', 'qtutils.qt' in sys.modules) diff --git a/labscript_utils/qtwidgets/enumoutput.py b/labscript_utils/qtwidgets/enumoutput.py index e9ddb757..5e50aee4 100644 --- a/labscript_utils/qtwidgets/enumoutput.py +++ b/labscript_utils/qtwidgets/enumoutput.py @@ -12,7 +12,7 @@ ##################################################################### import sys -from qtutils.qt import QtCore, QtGui, QtWidgets, QT_ENV, PYQT5 +from qtutils.qt import QtCore, QtGui, QtWidgets class EnumOutput(QtWidgets.QWidget): @@ -132,10 +132,14 @@ def eventFilter(self, obj, event): # pass scrollwheel events of disabled buttons through to the parent # code adapted from: http://www.qtforum.org/article/28540/disabled-widgets-and-wheel-events.html - elif obj and not obj.isEnabled() and event.type() == QtCore.QEvent.Type.Wheel and QT_ENV != PYQT5: - newEvent = QtGui.QWheelEvent(obj.mapToParent(event.pos()), event.globalPos(), - event.delta(), event.buttons(), - event.modifiers(), event.orientation()) + elif obj and not obj.isEnabled() and event.type() == QtCore.QEvent.Type.Wheel: + newEvent = QtGui.QWheelEvent( + QtCore.QPointF(obj.mapToParent(event.position().toPoint())), + event.globalPosition(), + event.pixelDelta(), event.angleDelta(), + event.buttons(), event.modifiers(), + event.phase(), event.inverted(), + ) QtWidgets.QApplication.instance().postEvent(obj.parent(), newEvent) return True diff --git a/labscript_utils/qtwidgets/imageoutput.py b/labscript_utils/qtwidgets/imageoutput.py index b7535381..76757fe4 100644 --- a/labscript_utils/qtwidgets/imageoutput.py +++ b/labscript_utils/qtwidgets/imageoutput.py @@ -178,9 +178,13 @@ def eventFilter(self, obj, event): # pass scrollwheel events of disabled buttons through to the parent # code adapted from: http://www.qtforum.org/article/28540/disabled-widgets-and-wheel-events.html elif obj and not obj.isEnabled() and event.type() == QEvent.Type.Wheel: - newEvent = QtGui.QWheelEvent(obj.mapToParent(event.pos()), event.globalPos(), - event.delta(), event.buttons(), - event.modifiers(), event.orientation()) + newEvent = QtGui.QWheelEvent( + QtCore.QPointF(obj.mapToParent(event.position().toPoint())), + event.globalPosition(), + event.pixelDelta(), event.angleDelta(), + event.buttons(), event.modifiers(), + event.phase(), event.inverted(), + ) QtWidgets.QApplication.instance().postEvent(obj.parent(), newEvent) return True diff --git a/labscript_utils/qtwidgets/shotqueue.py b/labscript_utils/qtwidgets/shotqueue.py index 239dbfa1..a3366939 100644 --- a/labscript_utils/qtwidgets/shotqueue.py +++ b/labscript_utils/qtwidgets/shotqueue.py @@ -68,7 +68,7 @@ def contextMenuEvent(self, event): menu = QMenu(self) delete_action = menu.addAction('Delete selected rows') delete_action.setEnabled(bool(self.selectionModel().selectedRows())) - chosen_action = menu.exec_(event.globalPos()) + chosen_action = menu.exec(event.globalPos()) if chosen_action is delete_action: self.deleteRequested.emit() event.accept() diff --git a/labscript_utils/splash.py b/labscript_utils/splash.py index 46922905..5d4e5fd2 100644 --- a/labscript_utils/splash.py +++ b/labscript_utils/splash.py @@ -15,7 +15,7 @@ from labscript_utils import dedent try: - from qtutils.qt import QtWidgets, QtCore, QtGui, QT_ENV + from qtutils.qt import QtWidgets, QtCore, QtGui except ImportError as e: if 'DLL load failed' in str(e): msg = """Failed to load Qt DLL. This can be caused by application shortcuts @@ -28,16 +28,6 @@ Qt = QtCore.Qt -# These are default in Qt6 and print a warning if set -if QT_ENV == 'PyQt5': - # Set auto high-DPI scaling - this ensures pixel metrics are scaled - # appropriately so that we don't get a weird mix of large fonts and small - # everything else on High DPI displays: - QtWidgets.QApplication.setAttribute(Qt.AA_EnableHighDpiScaling, True) - # Use high res pixmaps if available, instead of rendering at low resolution and - # upscaling: - QtWidgets.QApplication.setAttribute(Qt.AA_UseHighDpiPixmaps, True) - def configure_qapplication(qapplication): """Apply labscript-wide QApplication configuration.""" From bba7e424053ecd7d232fe990bed60cdeeb72896c Mon Sep 17 00:00:00 2001 From: spielman Date: Sat, 29 Aug 2026 13:31:40 -0400 Subject: [PATCH 66/66] Call exec() instead of the PyQt5-era exec_() alias exec_() exists under PyQt6 only because qtutils aliases it onto QCoreApplication and QDialog. The suite targets PyQt6 only, so call the real method and stop depending on that shim. Co-Authored-By: Claude Opus 5 --- labscript_utils/qtwidgets/fingertab.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/labscript_utils/qtwidgets/fingertab.py b/labscript_utils/qtwidgets/fingertab.py index 91364005..980c6173 100644 --- a/labscript_utils/qtwidgets/fingertab.py +++ b/labscript_utils/qtwidgets/fingertab.py @@ -52,5 +52,5 @@ def __init__(self, parent, *args): tabs.addTab(widget, d) tabs.setTabPosition(QtWidgets.QTabWidget.TabPosition.East) tabs.show() - sys.exit(app.exec_()) + sys.exit(app.exec())