From a2acdc7bac108712ed9c5b233b6bca86af2146d8 Mon Sep 17 00:00:00 2001 From: Dongcheng Lin <256920284+Limdongcheng@users.noreply.github.com> Date: Sat, 1 Aug 2026 01:02:43 +0800 Subject: [PATCH 1/2] feat: import verified Origin OPJ worksheets --- .gitattributes | 2 + Cargo.lock | 1 + Cargo.toml | 1 + crates/app/Cargo.toml | 3 + crates/app/src/ui/canvas/mod.rs | 2 +- crates/app/src/ui/commands.rs | 4 + crates/app/src/ui/commands/identity.rs | 2 +- crates/app/src/ui/commands_tests.rs | 28 + crates/app/src/ui/file_dialogs.rs | 57 +- crates/app/src/ui/file_dialogs/origin.rs | 527 ++++++++++++ .../file_dialogs/origin_allocation_tests.rs | 114 +++ .../app/src/ui/file_dialogs/origin_tests.rs | 781 ++++++++++++++++++ crates/app/src/ui/file_dialogs/preview.rs | 32 +- crates/app/src/ui/file_dialogs/recent.rs | 373 ++++++++- .../ui/file_dialogs/recent_report_tests.rs | 117 +++ crates/app/src/ui/file_dialogs/tests.rs | 105 ++- crates/app/src/ui/file_dialogs/xlsx.rs | 13 +- crates/app/src/ui/shortcuts.rs | 2 +- crates/core/src/lib.rs | 7 + crates/core/src/origin.rs | 720 ++++++++++++++++ crates/core/src/origin/names.rs | 100 +++ crates/core/src/origin/preflight.rs | 682 +++++++++++++++ crates/core/src/origin/preflight/model.rs | 109 +++ crates/core/src/origin_hardening_tests.rs | 286 +++++++ crates/core/src/origin_tests.rs | 772 +++++++++++++++++ crates/io/src/lib.rs | 1 + crates/io/src/origin.rs | 772 +++++++++++++++++ crates/io/src/origin/opj.rs | 621 ++++++++++++++ crates/io/src/origin/opj/metadata.rs | 775 +++++++++++++++++ crates/io/src/origin/opj/metadata/cursor.rs | 226 +++++ crates/io/src/origin/opj/metadata/tail.rs | 212 +++++ .../src/origin/opj/metadata_modern_tests.rs | 191 +++++ crates/io/src/origin/opj/metadata_tests.rs | 632 ++++++++++++++ crates/io/src/origin/opj/records.rs | 548 ++++++++++++ .../io/src/origin/opj/records_modern_tests.rs | 288 +++++++ crates/io/src/origin/opj/records_tests.rs | 668 +++++++++++++++ crates/io/src/origin/opju.rs | 11 + crates/io/src/origin/reader.rs | 358 ++++++++ crates/io/src/origin/reader_tests.rs | 303 +++++++ crates/io/src/origin/tests.rs | 716 ++++++++++++++++ .../tests/fixtures/origin/OPENOPJ-LICENSE.txt | 20 + crates/io/tests/fixtures/origin/README.md | 38 + ...awData_Locust_Revision1_TIS_Mechanism.opju | Bin 0 -> 64954 bytes .../fixtures/origin/test-origin-7.0552.opj | Bin 0 -> 282034 bytes crates/io/tests/origin_fixtures.rs | 238 ++++++ xtask/about.hbs | 35 +- xtask/src/main.rs | 22 + 47 files changed, 11455 insertions(+), 60 deletions(-) create mode 100644 crates/app/src/ui/file_dialogs/origin.rs create mode 100644 crates/app/src/ui/file_dialogs/origin_allocation_tests.rs create mode 100644 crates/app/src/ui/file_dialogs/origin_tests.rs create mode 100644 crates/app/src/ui/file_dialogs/recent_report_tests.rs create mode 100644 crates/core/src/origin.rs create mode 100644 crates/core/src/origin/names.rs create mode 100644 crates/core/src/origin/preflight.rs create mode 100644 crates/core/src/origin/preflight/model.rs create mode 100644 crates/core/src/origin_hardening_tests.rs create mode 100644 crates/core/src/origin_tests.rs create mode 100644 crates/io/src/origin.rs create mode 100644 crates/io/src/origin/opj.rs create mode 100644 crates/io/src/origin/opj/metadata.rs create mode 100644 crates/io/src/origin/opj/metadata/cursor.rs create mode 100644 crates/io/src/origin/opj/metadata/tail.rs create mode 100644 crates/io/src/origin/opj/metadata_modern_tests.rs create mode 100644 crates/io/src/origin/opj/metadata_tests.rs create mode 100644 crates/io/src/origin/opj/records.rs create mode 100644 crates/io/src/origin/opj/records_modern_tests.rs create mode 100644 crates/io/src/origin/opj/records_tests.rs create mode 100644 crates/io/src/origin/opju.rs create mode 100644 crates/io/src/origin/reader.rs create mode 100644 crates/io/src/origin/reader_tests.rs create mode 100644 crates/io/src/origin/tests.rs create mode 100644 crates/io/tests/fixtures/origin/OPENOPJ-LICENSE.txt create mode 100644 crates/io/tests/fixtures/origin/README.md create mode 100644 crates/io/tests/fixtures/origin/RawData_Locust_Revision1_TIS_Mechanism.opju create mode 100644 crates/io/tests/fixtures/origin/test-origin-7.0552.opj create mode 100644 crates/io/tests/origin_fixtures.rs diff --git a/.gitattributes b/.gitattributes index 9fc52fc..bd4afdb 100644 --- a/.gitattributes +++ b/.gitattributes @@ -2,3 +2,5 @@ *.ico binary *.png binary +*.opj binary +*.opju binary diff --git a/Cargo.lock b/Cargo.lock index 7eedb61..a297eaf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4047,6 +4047,7 @@ dependencies = [ "egui_extras", "fontdb", "image", + "libc", "log", "muda", "num-complex", diff --git a/Cargo.toml b/Cargo.toml index d5e0052..09f4a7a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -75,6 +75,7 @@ muda = { version = "0.19.3", default-features = false } rfd = "0.17" zip = { version = "8.6", default-features = false, features = ["deflate"] } image = { version = "0.25", default-features = false, features = ["jpeg", "png", "tiff"] } +libc = "0.2" pdf-writer = "0.12" resvg = "0.47" svg2pdf = "0.13" diff --git a/crates/app/Cargo.toml b/crates/app/Cargo.toml index 30b26e4..7d37318 100644 --- a/crates/app/Cargo.toml +++ b/crates/app/Cargo.toml @@ -38,6 +38,9 @@ uuid.workspace = true raw-window-handle.workspace = true windows-sys.workspace = true +[target.'cfg(unix)'.dependencies] +libc.workspace = true + [target.'cfg(target_os = "macos")'.dependencies] fontdb.workspace = true muda.workspace = true diff --git a/crates/app/src/ui/canvas/mod.rs b/crates/app/src/ui/canvas/mod.rs index fad8fb6..954168d 100644 --- a/crates/app/src/ui/canvas/mod.rs +++ b/crates/app/src/ui/canvas/mod.rs @@ -429,7 +429,7 @@ fn welcome_start(app: &mut PlotxApp, ui: &mut Ui) { if welcome_action(ui, icon::FOLDER_OPEN, "Open project…") { crate::ui::file_dialogs::open_project(app); } - if welcome_action(ui, icon::TABLE, "Import table / CSV…") { + if welcome_action(ui, icon::TABLE, "Import table…") { crate::ui::file_dialogs::import_delimited_table(app); } if welcome_action(ui, icon::FILE_PLUS, "New empty data table") { diff --git a/crates/app/src/ui/commands.rs b/crates/app/src/ui/commands.rs index bf86b9a..3f1644a 100644 --- a/crates/app/src/ui/commands.rs +++ b/crates/app/src/ui/commands.rs @@ -400,6 +400,10 @@ pub fn describe(app: &PlotxApp, id: CommandId) -> CommandDescriptor { !app.session.recent_files.is_empty(), "Open a file or project to build the recent list.", ), + CommandId::ImportTable => requires( + app.session.ui.table_import_preview.is_none(), + "Finish or cancel the current table import preview before importing another table.", + ), CommandId::ExportData => requires( dataset().is_some_and(|dataset| { !plotx_core::data_export::DataExportAvailability::for_dataset(dataset).is_empty() diff --git a/crates/app/src/ui/commands/identity.rs b/crates/app/src/ui/commands/identity.rs index 6f5ef6f..5637f9d 100644 --- a/crates/app/src/ui/commands/identity.rs +++ b/crates/app/src/ui/commands/identity.rs @@ -52,7 +52,7 @@ pub(super) fn command_identity( ), CommandId::ClearRecentFiles => plain("Clear Recent Files", None), CommandId::HelpManual => plain("User Manual", Some(icon::BOOK_OPEN)), - CommandId::ImportTable => plain("Import Table / CSV…", Some(icon::TABLE)), + CommandId::ImportTable => plain("Import Table…", Some(icon::TABLE)), CommandId::PasteTable => plain("Paste Table from Clipboard", Some(icon::CLIPBOARD_TEXT)), CommandId::SaveProject => plain("Save Project", Some(icon::FLOPPY_DISK)), CommandId::NewTable => plain("New Empty Data Table", Some(icon::TABLE)), diff --git a/crates/app/src/ui/commands_tests.rs b/crates/app/src/ui/commands_tests.rs index 550f8a5..5521a5f 100644 --- a/crates/app/src/ui/commands_tests.rs +++ b/crates/app/src/ui/commands_tests.rs @@ -253,6 +253,34 @@ fn spacing_commands_are_registered_checked_and_execute() { ); } +#[test] +fn origin_import_reuses_import_table_command_identity() { + let app = app(); + assert_eq!(CommandId::ImportTable.stable_id(), "file.import_table"); + assert_eq!( + describe(&app, CommandId::ImportTable).label, + "Import Table…" + ); +} + +#[test] +fn import_table_is_disabled_while_a_table_preview_is_pending() { + let mut app = app(); + crate::ui::file_dialogs::import_delimited_text( + &mut app, + "x,y\n0,1\n", + crate::ui::file_dialogs::DelimitedTableSource::Clipboard, + ); + assert!(app.session.ui.table_import_preview.is_some()); + + let command = describe(&app, CommandId::ImportTable); + assert!(!command.enabled); + assert_eq!( + command.disabled_reason, + Some("Finish or cancel the current table import preview before importing another table.") + ); +} + #[test] fn automation_is_a_global_menu_and_palette_command() { let app = app(); diff --git a/crates/app/src/ui/file_dialogs.rs b/crates/app/src/ui/file_dialogs.rs index d3102d9..59e1b1c 100644 --- a/crates/app/src/ui/file_dialogs.rs +++ b/crates/app/src/ui/file_dialogs.rs @@ -8,6 +8,7 @@ use plotx_core::state::ProcessingSchemeDialogState; mod delimited; mod discovery; +mod origin; mod path; mod preview; mod recent; @@ -21,26 +22,23 @@ use xlsx::import_xlsx_table_path; pub(crate) fn import_delimited_table(app: &mut PlotxApp) { let Some(path) = rfd::FileDialog::new() .add_filter( - "Table (*.csv, *.tsv, *.txt, *.xlsx)", - &["csv", "tsv", "txt", "xlsx"], + "Table (*.csv, *.tsv, *.txt, *.xlsx, *.opj)", + origin::IMPORT_TABLE_FILTER_EXTENSIONS, + ) + .add_filter( + origin::ORIGIN_PROJECT_FILTER_LABEL, + origin::ORIGIN_PROJECT_FILTER_EXTENSIONS, ) .add_filter("Excel workbook (*.xlsx)", &["xlsx"]) .add_filter("CSV (*.csv)", &["csv"]) .add_filter("TSV (*.tsv)", &["tsv"]) .add_filter("All files", &["*"]) - .set_title("Import a comma, tab, or semicolon delimited table") + .set_title("Import a table") .pick_file() else { return; }; - if path - .extension() - .is_some_and(|extension| extension.eq_ignore_ascii_case("xlsx")) - { - import_xlsx_table_path(app, &path); - } else { - import_delimited_table_path(app, &path); - } + open_recent_path(app, &path); } fn import_delimited_table_path(app: &mut PlotxApp, path: &std::path::Path) { @@ -288,9 +286,34 @@ pub(crate) fn import_delimited_text_with_schema( } pub(crate) fn commit_table_import_preview(app: &mut PlotxApp) -> bool { + commit_table_import_preview_with_recent(app, PlotxApp::note_recent_file) +} + +pub(crate) fn commit_table_import_preview_with_recent( + app: &mut PlotxApp, + mut note_recent_file: F, +) -> bool +where + F: FnMut(&mut PlotxApp, &std::path::Path), +{ let Some(preview) = app.session.ui.table_import_preview.take() else { return false; }; + if preview.candidates.is_empty() { + app.session.record_operation(OperationReport::<()>::failure( + preview.report.id, + OperationKind::TableImport, + "Table import failed because there are no supported tables to import.", + Diagnostic::new( + Severity::Error, + DiagnosticCode::TableImportFailed, + "The import preview contains no supported table candidates.", + ) + .with_source("app.table_import") + .with_context("stage", "preview_commit"), + )); + return false; + } for candidate in preview.candidates { app.import_table_dataset_typed( candidate.name, @@ -305,7 +328,7 @@ pub(crate) fn commit_table_import_preview(app: &mut PlotxApp) -> bool { // through the status line. Recording the import first keeps that // diagnostic: `record_operation` replaces the status unconditionally. if let Some(path) = preview.recent_path { - app.note_recent_file(&path); + note_recent_file(app, &path); } true } @@ -321,8 +344,12 @@ pub(crate) fn load_and_note(app: &mut PlotxApp, path: &std::path::Path) { pub(crate) fn open_file(app: &mut PlotxApp) { if let Some(paths) = rfd::FileDialog::new() .add_filter( - "All supported data (*.spm, *.pfc, *.abf, *.jdf, fid, ser, *.zip)", - &["spm", "pfc", "abf", "jdf", "fid", "ser", "zip"], + "All supported data (*.spm, *.pfc, *.abf, *.jdf, fid, ser, *.zip, *.opj)", + origin::OPEN_FILE_FILTER_EXTENSIONS, + ) + .add_filter( + origin::ORIGIN_PROJECT_FILTER_LABEL, + origin::ORIGIN_PROJECT_FILTER_EXTENSIONS, ) .add_filter("Bruker NanoScope AFM (*.spm, *.pfc)", &["spm", "pfc"]) .add_filter("Axon Binary Format 2 (*.abf)", &["abf"]) @@ -334,7 +361,7 @@ pub(crate) fn open_file(app: &mut PlotxApp) { .pick_files() { for path in paths { - load_and_note(app, &path); + open_recent_path(app, &path); } } } diff --git a/crates/app/src/ui/file_dialogs/origin.rs b/crates/app/src/ui/file_dialogs/origin.rs new file mode 100644 index 0000000..8592d62 --- /dev/null +++ b/crates/app/src/ui/file_dialogs/origin.rs @@ -0,0 +1,527 @@ +use std::io::{Read, Seek, SeekFrom}; +use std::path::Path; +use std::sync::Arc; + +use plotx_core::operation::{ + Diagnostic, DiagnosticCode, OperationId, OperationKind, OperationReport, Severity, +}; +use plotx_core::origin::{ + ImportedOriginWorksheet, ORIGIN_IMPORT_OPERATION, OriginImportError, import_origin_project, +}; +use plotx_core::state::{ + PlotxApp, TableImportCandidate, TableImportPreviewState, TableImportSource, TypedTableState, +}; +use plotx_io::origin::{ + OriginDiagnostic, OriginDiagnosticSeverity, OriginError, OriginLimits, OriginProject, + probe_origin, read_origin, +}; + +pub(super) const IMPORT_TABLE_FILTER_EXTENSIONS: &[&str] = &["csv", "tsv", "txt", "xlsx", "opj"]; +pub(super) const ORIGIN_PROJECT_FILTER_LABEL: &str = + "Origin projects (experimental: OPJ import; OPJU recognition only)"; +pub(super) const ORIGIN_PROJECT_FILTER_EXTENSIONS: &[&str] = &["opj", "opju"]; +pub(super) const OPEN_FILE_FILTER_EXTENSIONS: &[&str] = + &["spm", "pfc", "abf", "jdf", "fid", "ser", "zip", "opj"]; + +const ORIGIN_MEDIA_TYPE: &str = "application/x-origin-project"; +const ORIGIN_READ_CHUNK_BYTES: usize = 16 * 1024; + +#[derive(Debug)] +pub(super) struct OpenOriginSource { + file: std::fs::File, + metadata_len: u64, +} + +impl OpenOriginSource { + pub(super) fn new(file: std::fs::File, metadata_len: u64) -> Self { + Self { file, metadata_len } + } + + fn into_parts(self) -> (std::fs::File, u64) { + (self.file, self.metadata_len) + } +} + +struct OriginFailure { + stage: &'static str, + message: String, + detail: String, +} + +#[derive(Clone, Copy)] +struct SourceByteLimit { + resource: &'static str, + limit: usize, + maximum: u64, + sentinel: usize, +} + +impl OriginFailure { + fn io(stage: &'static str, message: impl Into, error: impl ToString) -> Self { + Self { + stage, + message: message.into(), + detail: error.to_string(), + } + } + + fn parser(stage: &'static str, error: OriginError) -> Self { + let message = match &error { + OriginError::UnrecognizedFormat => { + "The selected file does not have a recognized Origin project signature. No data was imported." + .to_owned() + } + OriginError::UnsupportedOpjuVariant { message } => message.clone(), + OriginError::NoSupportedWorksheet => { + "The Origin project contains no supported table data. No data was imported." + .to_owned() + } + OriginError::LimitExceeded { .. } + | OriginError::InvalidLimit { .. } + | OriginError::ArithmeticOverflow { .. } + | OriginError::AllocationFailed { .. } => { + format!("The Origin project could not be imported safely: {error}. No data was imported.") + } + _ => format!("The Origin project could not be read: {error}. No data was imported."), + }; + Self { + stage, + message, + detail: error.to_string(), + } + } + + fn core(error: OriginImportError) -> Self { + let message = match &error { + OriginImportError::NoSupportedWorksheet => { + "The Origin project contains no supported table data. No data was imported." + .to_owned() + } + _ => format!( + "The Origin project could not be converted into PlotX tables: {error}. No data was imported." + ), + }; + Self { + stage: "convert", + message, + detail: error.to_string(), + } + } +} + +pub(super) fn import_origin_project_source( + app: &mut PlotxApp, + path: &Path, + source: OpenOriginSource, +) { + let limits = OriginLimits::default(); + let result = read_origin_source(source, limits).and_then(|source_bytes| { + probe_origin(&source_bytes).map_err(|error| OriginFailure::parser("probe", error))?; + let project = read_origin(&source_bytes, limits) + .map_err(|error| OriginFailure::parser("parse", error))?; + Ok((source_bytes, project)) + }); + match result { + Ok((source_bytes, project)) => { + import_origin_project_model(app, path, source_bytes, project, limits); + } + Err(error) => { + let operation_id = app.session.begin_operation(); + install_origin_result(app, operation_id, path, Err(error)); + } + } +} + +pub(super) fn import_origin_project_model( + app: &mut PlotxApp, + path: &Path, + source_bytes: Arc<[u8]>, + project: OriginProject, + limits: OriginLimits, +) { + let operation_id = app.session.begin_operation(); + let result = preview_from_project(operation_id, path, source_bytes, project, limits); + install_origin_result(app, operation_id, path, result); +} + +fn read_origin_source( + source: OpenOriginSource, + limits: OriginLimits, +) -> Result, OriginFailure> { + let (mut file, metadata_len) = source.into_parts(); + read_origin_handle(&mut file, Some(metadata_len), limits) +} + +fn read_origin_handle( + mut reader: R, + metadata_len: Option, + limits: OriginLimits, +) -> Result, OriginFailure> { + let source_limit = checked_source_byte_limit(limits) + .map_err(|error| OriginFailure::io("limits", limit_message(&error), error))?; + if let Some(length) = metadata_len + && length > source_limit.maximum + { + let error = source_too_large(length, source_limit); + return Err(OriginFailure::io("metadata", limit_message(&error), error)); + } + reader.seek(SeekFrom::Start(0)).map_err(|error| { + OriginFailure::io( + "rewind", + "The selected Origin project could not be rewound for import. No data was imported.", + error, + ) + })?; + read_bounded_origin(reader, metadata_len, limits).map_err(|error| OriginFailure { + stage: "read", + message: limit_message(&error), + detail: error, + }) +} + +pub(super) fn read_bounded_origin( + mut reader: R, + metadata_len: Option, + limits: OriginLimits, +) -> Result, String> { + let source_limit = checked_source_byte_limit(limits)?; + if let Some(length) = metadata_len + && length > source_limit.maximum + { + return Err(source_too_large(length, source_limit)); + } + + let mut bytes = Vec::new(); + let mut chunk = [0_u8; ORIGIN_READ_CHUNK_BYTES]; + loop { + let remaining = source_limit + .sentinel + .checked_sub(bytes.len()) + .ok_or_else(|| { + OriginError::ArithmeticOverflow { + resource: "Origin source sentinel bytes", + } + .to_string() + })?; + let request = remaining.min(chunk.len()); + let read = match reader.read(&mut chunk[..request]) { + Ok(0) => break, + Ok(read) if read <= request => read, + Ok(read) => { + return Err(format!( + "the bounded Origin project read failed: the reader returned {read} bytes for a {request}-byte buffer" + )); + } + Err(error) if error.kind() == std::io::ErrorKind::Interrupted => continue, + Err(error) => { + return Err(format!("the bounded Origin project read failed: {error}")); + } + }; + let next_len = bytes.len().checked_add(read).ok_or_else(|| { + OriginError::ArithmeticOverflow { + resource: "Origin source bytes", + } + .to_string() + })?; + if next_len > source_limit.limit { + return Err(OriginError::LimitExceeded { + resource: source_limit.resource, + limit: source_limit.limit, + actual: next_len, + } + .to_string()); + } + reserve_source_capacity(&mut bytes, next_len, source_limit, limits)?; + let read_bytes = chunk.get(..read).ok_or_else(|| { + "the bounded Origin project read failed: the reader exceeded its buffer".to_owned() + })?; + bytes.extend_from_slice(read_bytes); + } + + let conversion_peak = bytes.capacity().checked_add(bytes.len()).ok_or_else(|| { + OriginError::ArithmeticOverflow { + resource: "Origin source Arc conversion", + } + .to_string() + })?; + if conversion_peak > limits.max_total_owned_bytes { + return Err(OriginError::LimitExceeded { + resource: "total owned bytes", + limit: limits.max_total_owned_bytes, + actual: conversion_peak, + } + .to_string()); + } + Ok(Arc::<[u8]>::from(bytes)) +} + +fn reserve_source_capacity( + bytes: &mut Vec, + required_len: usize, + source_limit: SourceByteLimit, + limits: OriginLimits, +) -> Result<(), String> { + let old_capacity = bytes.capacity(); + if required_len <= old_capacity { + return Ok(()); + } + let doubled = old_capacity.checked_mul(2).unwrap_or(source_limit.sentinel); + let target_capacity = required_len.max(doubled).min(source_limit.limit); + let additional = target_capacity.checked_sub(bytes.len()).ok_or_else(|| { + OriginError::ArithmeticOverflow { + resource: "Origin source allocation", + } + .to_string() + })?; + bytes.try_reserve_exact(additional).map_err(|_| { + OriginError::AllocationFailed { + resource: "Origin source bytes", + requested: target_capacity, + } + .to_string() + })?; + let actual_capacity = bytes.capacity(); + if actual_capacity > limits.max_total_owned_bytes { + return Err(OriginError::LimitExceeded { + resource: "total owned bytes", + limit: limits.max_total_owned_bytes, + actual: actual_capacity, + } + .to_string()); + } + Ok(()) +} + +fn checked_source_byte_limit(limits: OriginLimits) -> Result { + limits.validate().map_err(|error| error.to_string())?; + let (resource, limit_name, limit) = if limits.max_input_bytes <= limits.max_total_owned_bytes { + ("input bytes", "max_input_bytes", limits.max_input_bytes) + } else { + ( + "total owned bytes", + "max_total_owned_bytes", + limits.max_total_owned_bytes, + ) + }; + let sentinel = limit.checked_add(1).ok_or_else(|| { + invalid_limit( + limit_name, + limit, + "the limit must leave room for an oversize sentinel byte", + ) + .to_string() + })?; + let maximum = u64::try_from(limit).map_err(|_| { + invalid_limit( + limit_name, + limit, + "the source-byte limit cannot be represented by the bounded reader", + ) + .to_string() + })?; + Ok(SourceByteLimit { + resource, + limit, + maximum, + sentinel, + }) +} + +fn invalid_limit(name: &'static str, value: usize, reason: &'static str) -> OriginError { + OriginError::InvalidLimit { + name, + value, + reason, + } +} + +fn source_too_large(actual: u64, source_limit: SourceByteLimit) -> String { + let actual = usize::try_from(actual).unwrap_or(usize::MAX); + OriginError::LimitExceeded { + resource: source_limit.resource, + limit: source_limit.limit, + actual, + } + .to_string() +} + +fn limit_message(detail: impl std::fmt::Display) -> String { + format!("The Origin project could not be imported safely: {detail}. No data was imported.") +} + +fn preview_from_project( + operation_id: OperationId, + path: &Path, + source_bytes: Arc<[u8]>, + project: OriginProject, + limits: OriginLimits, +) -> Result { + let store = Arc::new(plotx_core::data::MemoryBlockStore::default()); + let codecs = plotx_core::data::CodecRegistry::with_arrow_ipc(); + let imported = import_origin_project(project, store.as_ref(), &codecs, limits) + .map_err(OriginFailure::core)?; + preview_from_imported(operation_id, path, source_bytes, store, imported).map_err(|error| { + OriginFailure::io( + "revision", + "The Origin project could not be prepared for preview. No data was imported.", + error, + ) + }) +} + +pub(super) fn preview_from_imported( + operation_id: OperationId, + path: &Path, + source_bytes: Arc<[u8]>, + store: Arc, + imported: Vec, +) -> Result { + ensure_candidate_count(imported.len())?; + let candidate_count = imported.len(); + let project_diagnostics = imported + .first() + .map(|worksheet| worksheet.diagnostics.clone()) + .unwrap_or_default(); + let file_name = path + .file_name() + .map(|name| name.to_string_lossy().into_owned()) + .unwrap_or_else(|| path.display().to_string()); + let mut candidates = Vec::with_capacity(candidate_count); + let mut candidate_diagnostics = Vec::with_capacity(candidate_count); + + for worksheet in imported { + let row_count = worksheet.snapshot.row_count; + let worksheet_name = worksheet.name; + let typed_state = TypedTableState::imported_with_operation( + worksheet.snapshot, + Arc::clone(&store), + ORIGIN_IMPORT_OPERATION, + ) + .map_err(|error| error.to_string())?; + let mut source = TableImportSource::new(Arc::clone(&source_bytes), ORIGIN_MEDIA_TYPE); + source.name = Some(file_name.clone()); + source.metadata = worksheet.source_metadata; + candidates.push(TableImportCandidate { + name: worksheet_name.clone(), + retained_sources: vec![source], + typed_state, + x_binding: None, + series_bindings: Vec::new(), + }); + candidate_diagnostics.push( + Diagnostic::new( + Severity::Info, + DiagnosticCode::TableImportSucceeded, + format!("Prepared Origin table '{worksheet_name}' with {row_count} row(s)."), + ) + .with_source("core.origin") + .with_context("path", path.display().to_string()) + .with_context("table", worksheet_name), + ); + } + + let mut diagnostics = project_diagnostics + .iter() + .map(origin_diagnostic) + .collect::>(); + let warning_count = diagnostics + .iter() + .filter(|diagnostic| diagnostic.severity == Severity::Warning) + .count(); + diagnostics.extend(candidate_diagnostics); + let summary = if warning_count == 0 { + format!("Imported {candidate_count} Origin table(s).") + } else { + format!("Imported {candidate_count} Origin table(s) with {warning_count} warning(s).") + }; + let mut report = if warning_count == 0 { + OperationReport::success(operation_id, OperationKind::TableImport, summary, ()) + } else { + OperationReport::warning(operation_id, OperationKind::TableImport, summary, ()) + }; + for diagnostic in diagnostics { + report = report.with_diagnostic(diagnostic); + } + Ok(TableImportPreviewState { + candidates, + selected: 0, + report, + recent_path: Some(path.to_owned()), + }) +} + +pub(super) fn ensure_candidate_count(count: usize) -> Result<(), String> { + if count == 0 { + Err("the Origin project contains no supported table candidates".to_owned()) + } else { + Ok(()) + } +} + +fn origin_diagnostic(diagnostic: &OriginDiagnostic) -> Diagnostic { + let severity = match diagnostic.severity { + OriginDiagnosticSeverity::Info => Severity::Info, + OriginDiagnosticSeverity::Warning => Severity::Warning, + }; + let mut result = Diagnostic::new( + severity, + if severity == Severity::Warning { + DiagnosticCode::TableImportWarning + } else { + DiagnosticCode::TableImportSucceeded + }, + diagnostic.message.clone(), + ) + .with_source("io.origin") + .with_context("origin_code", format!("{:?}", diagnostic.code)); + if let Some(location) = &diagnostic.location { + if let Some(workbook) = &location.workbook { + result = result.with_context("workbook", workbook.clone()); + } + if let Some(worksheet) = &location.worksheet { + result = result.with_context("table", worksheet.clone()); + } + if let Some(column) = &location.column { + result = result.with_context("column", column.clone()); + } + if let Some(offset) = location.byte_offset { + result = result.with_context("byte_offset", offset.to_string()); + } + } + result +} + +fn install_origin_result( + app: &mut PlotxApp, + operation_id: OperationId, + path: &Path, + result: Result, +) { + match result { + Ok(preview) => app.session.ui.table_import_preview = Some(preview), + Err(error) => { + app.session.record_operation(OperationReport::<()>::failure( + operation_id, + OperationKind::TableImport, + error.message.clone(), + Diagnostic::new( + Severity::Error, + DiagnosticCode::TableImportFailed, + error.message, + ) + .with_source("app.table_import.origin") + .with_context("path", path.display().to_string()) + .with_context("stage", error.stage) + .with_context("error", error.detail), + )); + } + } +} + +#[cfg(test)] +#[path = "origin_allocation_tests.rs"] +mod origin_allocation_tests; + +#[cfg(test)] +#[path = "origin_tests.rs"] +mod origin_tests; diff --git a/crates/app/src/ui/file_dialogs/origin_allocation_tests.rs b/crates/app/src/ui/file_dialogs/origin_allocation_tests.rs new file mode 100644 index 0000000..814613c --- /dev/null +++ b/crates/app/src/ui/file_dialogs/origin_allocation_tests.rs @@ -0,0 +1,114 @@ +use super::*; +use std::cell::Cell; +use std::io::{self, Read}; +use std::rc::Rc; + +const NON_POWER_OF_TWO_BYTES: usize = 5_003; + +struct FiniteCountingReader { + remaining: usize, + consumed: Rc>, +} + +impl FiniteCountingReader { + fn new(remaining: usize, consumed: Rc>) -> Self { + Self { + remaining, + consumed, + } + } +} + +impl Read for FiniteCountingReader { + fn read(&mut self, buffer: &mut [u8]) -> io::Result { + let read = self.remaining.min(buffer.len()); + buffer[..read].fill(0x5a); + self.remaining -= read; + self.consumed.set(self.consumed.get() + read); + Ok(read) + } +} + +struct InfiniteCountingReader { + consumed: Rc>, +} + +impl Read for InfiniteCountingReader { + fn read(&mut self, buffer: &mut [u8]) -> io::Result { + buffer.fill(0x5a); + self.consumed.set(self.consumed.get() + buffer.len()); + Ok(buffer.len()) + } +} + +fn source_limits(max_input_bytes: usize, max_total_owned_bytes: usize) -> OriginLimits { + OriginLimits { + max_input_bytes, + max_total_owned_bytes, + ..OriginLimits::default() + } +} + +#[test] +fn unknown_non_power_of_two_source_rejects_conversion_peak_over_total_limit() { + let total_limit = NON_POWER_OF_TWO_BYTES * 2 - 1; + let consumed = Rc::new(Cell::new(0)); + let reader = FiniteCountingReader::new(NON_POWER_OF_TWO_BYTES, Rc::clone(&consumed)); + + let result = read_bounded_origin( + reader, + None, + source_limits(NON_POWER_OF_TWO_BYTES, total_limit), + ); + let error = match result { + Ok(bytes) => panic!( + "a {}-byte Arc bypassed the {total_limit}-byte conversion-peak budget", + bytes.len() + ), + Err(error) => error, + }; + + assert_eq!(consumed.get(), NON_POWER_OF_TWO_BYTES); + assert!(error.contains("total owned bytes"), "{error}"); + assert!(error.contains(&total_limit.to_string()), "{error}"); +} + +#[test] +fn unknown_non_power_of_two_source_succeeds_at_exact_conversion_peak() { + let total_limit = NON_POWER_OF_TWO_BYTES * 2; + let consumed = Rc::new(Cell::new(0)); + let reader = FiniteCountingReader::new(NON_POWER_OF_TWO_BYTES, Rc::clone(&consumed)); + + let bytes = read_bounded_origin( + reader, + None, + source_limits(NON_POWER_OF_TWO_BYTES, total_limit), + ) + .expect("an exact source allocation plus Arc copy must fit"); + + assert_eq!(bytes.len(), NON_POWER_OF_TWO_BYTES); + assert_eq!(consumed.get(), NON_POWER_OF_TWO_BYTES); +} + +#[test] +fn unknown_source_consumes_only_the_one_byte_oversize_sentinel() { + let input_limit = NON_POWER_OF_TWO_BYTES; + let consumed = Rc::new(Cell::new(0)); + let reader = InfiniteCountingReader { + consumed: Rc::clone(&consumed), + }; + + let error = read_bounded_origin(reader, None, source_limits(input_limit, input_limit * 3)) + .expect_err("an unknown-length source must stop at one byte over the input limit"); + + assert_eq!(consumed.get(), input_limit + 1); + assert_eq!( + error, + OriginError::LimitExceeded { + resource: "input bytes", + limit: input_limit, + actual: input_limit + 1, + } + .to_string() + ); +} diff --git a/crates/app/src/ui/file_dialogs/origin_tests.rs b/crates/app/src/ui/file_dialogs/origin_tests.rs new file mode 100644 index 0000000..39c9208 --- /dev/null +++ b/crates/app/src/ui/file_dialogs/origin_tests.rs @@ -0,0 +1,781 @@ +use super::*; +use crate::ui::file_dialogs::recent::{ + OPEN_HEADER_BYTES, OpenPathEntryType, classify_open_path, classify_open_path_with_header, +}; +#[cfg(unix)] +use crate::ui::file_dialogs::recent::{ + classify_open_handle, dispatch_classified_path, open_file_for_classification, +}; +use crate::ui::file_dialogs::{RecentOpenKind, open_recent_path}; +use plotx_core::operation::{OperationId, OperationOutcome, Severity}; +use plotx_core::origin::{ImportedOriginWorksheet, ORIGIN_IMPORT_OPERATION}; +use plotx_core::state::PlotxApp; +use plotx_io::origin::OriginLimits; +use std::cell::Cell; +use std::io::{self, Read, Seek, SeekFrom}; +use std::path::{Path, PathBuf}; +use std::rc::Rc; +use std::sync::Arc; + +const OPENOPJ_FIXTURE: &[u8] = + include_bytes!("../../../../io/tests/fixtures/origin/test-origin-7.0552.opj"); + +struct PanicOnRead; + +impl Read for PanicOnRead { + fn read(&mut self, _buffer: &mut [u8]) -> io::Result { + panic!("bounded reader must reject before reading") + } +} + +struct SeekFailure; + +impl Read for SeekFailure { + fn read(&mut self, _buffer: &mut [u8]) -> io::Result { + panic!("a failed rewind must stop before reading") + } +} + +impl Seek for SeekFailure { + fn seek(&mut self, _position: SeekFrom) -> io::Result { + Err(io::Error::other("injected rewind failure")) + } +} + +struct ReadFailure; + +impl Read for ReadFailure { + fn read(&mut self, _buffer: &mut [u8]) -> io::Result { + Err(io::Error::other("injected full-read failure")) + } +} + +impl Seek for ReadFailure { + fn seek(&mut self, _position: SeekFrom) -> io::Result { + Ok(0) + } +} + +struct CountingRepeat { + bytes_read: Rc>, +} + +impl Read for CountingRepeat { + fn read(&mut self, buffer: &mut [u8]) -> io::Result { + buffer.fill(0); + self.bytes_read + .set(self.bytes_read.get().saturating_add(buffer.len())); + Ok(buffer.len()) + } +} + +fn temp_origin_file(extension: &str, bytes: &[u8]) -> PathBuf { + let path = std::env::temp_dir().join(format!( + "plotx-origin-app-{}.{}", + uuid::Uuid::new_v4(), + extension + )); + std::fs::write(&path, bytes).unwrap(); + path +} + +fn duplicated_fixture_import() -> ( + Arc, + Vec, +) { + let limits = OriginLimits::default(); + let project = plotx_io::origin::read_origin(OPENOPJ_FIXTURE, limits).unwrap(); + let store = Arc::new(plotx_core::data::MemoryBlockStore::default()); + let codecs = plotx_core::data::CodecRegistry::with_arrow_ipc(); + let imported = + plotx_core::origin::import_origin_project(project, store.as_ref(), &codecs, limits) + .unwrap(); + let first = imported.into_iter().next().unwrap(); + let second = ImportedOriginWorksheet { + name: format!("{} copy", first.name), + snapshot: first.snapshot.clone(), + source_metadata: first.source_metadata.clone(), + diagnostics: first.diagnostics.clone(), + resource_usage: first.resource_usage.clone(), + }; + (store, vec![first, second]) +} + +#[test] +fn origin_import_filter_retains_tables_and_adds_experimental_projects() { + assert_eq!( + IMPORT_TABLE_FILTER_EXTENSIONS, + &["csv", "tsv", "txt", "xlsx", "opj"] + ); + assert_eq!( + ORIGIN_PROJECT_FILTER_LABEL, + "Origin projects (experimental: OPJ import; OPJU recognition only)" + ); + assert_eq!(ORIGIN_PROJECT_FILTER_EXTENSIONS, &["opj", "opju"]); +} + +#[test] +fn origin_supported_file_filter_excludes_recognition_only_opju() { + assert!(OPEN_FILE_FILTER_EXTENSIONS.contains(&"opj")); + assert!(!OPEN_FILE_FILTER_EXTENSIONS.contains(&"opju")); +} + +#[test] +fn origin_routing_uses_signature_before_extension() { + let root = std::env::temp_dir().join(format!("plotx-origin-route-{}", uuid::Uuid::new_v4())); + std::fs::create_dir(&root).unwrap(); + let disguised = root.join("project.dat"); + std::fs::write(&disguised, OPENOPJ_FIXTURE).unwrap(); + + let kind = classify_open_path(&disguised).unwrap(); + + std::fs::remove_dir_all(root).unwrap(); + assert_eq!(kind.kind(), RecentOpenKind::OriginProject); +} + +#[test] +fn origin_extension_routes_signature_mismatch_to_origin_adapter() { + let path = PathBuf::from("not-origin.opj"); + let kind = classify_open_path_with_header(&path, OpenPathEntryType::RegularFile, || { + Ok(([0_u8; OPEN_HEADER_BYTES], 0)) + }) + .unwrap(); + assert_eq!(format!("{kind:?}"), "OriginProject"); +} + +#[test] +fn origin_pending_preview_rejects_a_second_table_path_without_replacement() { + let first = temp_origin_file("opj", OPENOPJ_FIXTURE); + let second = temp_origin_file("csv", b"time,value\n0,1\n"); + let mut app = PlotxApp::new_with_settings(plotx_core::settings::Settings::default()); + + open_recent_path(&mut app, &first); + let first_preview_path = app + .session + .ui + .table_import_preview + .as_ref() + .expect("the first table path should create a preview") + .recent_path + .clone(); + open_recent_path(&mut app, &second); + + std::fs::remove_file(first).unwrap(); + std::fs::remove_file(second).unwrap(); + let preview = app + .session + .ui + .table_import_preview + .as_ref() + .expect("the first preview must remain pending"); + assert_eq!(preview.recent_path, first_preview_path); + assert!(app.doc.datasets.is_empty()); + assert!(app.session.recent_files.is_empty()); + let report = app + .session + .operation_history + .operations() + .next_back() + .expect("the rejected second import should be reported"); + assert_eq!(report.outcome, OperationOutcome::Failure); + assert!( + report + .summary + .to_ascii_lowercase() + .contains("finish or cancel"), + "{report:?}" + ); +} + +#[cfg(unix)] +#[test] +fn origin_dispatch_reuses_the_classified_handle_after_path_replacement() { + use std::os::unix::net::UnixListener; + + let id = uuid::Uuid::new_v4(); + let path = PathBuf::from("/tmp").join(format!("px-{id}.opj")); + let original_path = PathBuf::from("/tmp").join(format!("px-{id}.saved")); + std::fs::write(&path, OPENOPJ_FIXTURE).unwrap(); + let classified = classify_open_path(&path).unwrap(); + assert_eq!(classified.kind(), RecentOpenKind::OriginProject); + std::fs::rename(&path, &original_path).unwrap(); + let replacement = UnixListener::bind(&path).unwrap(); + let mut app = PlotxApp::new_with_settings(plotx_core::settings::Settings::default()); + + dispatch_classified_path(&mut app, &path, classified); + + drop(replacement); + std::fs::remove_file(&path).unwrap(); + std::fs::remove_file(original_path).unwrap(); + let preview = app + .session + .ui + .table_import_preview + .as_ref() + .expect("dispatch must consume the original classified file handle"); + assert_eq!(preview.recent_path.as_deref(), Some(path.as_path())); + assert!(!preview.candidates.is_empty()); + assert!(app.doc.datasets.is_empty()); + assert!(app.session.recent_files.is_empty()); +} + +#[cfg(unix)] +#[test] +fn origin_classification_rejects_non_regular_handle_metadata() { + let device = std::fs::File::open("/dev/null").unwrap(); + + let error = classify_open_handle(Path::new("device.opj"), device) + .expect_err("a character-device handle must be rejected before header reads"); + + assert!(error.to_string().contains("regular file"), "{error}"); +} + +#[cfg(unix)] +#[test] +fn origin_classification_opens_paths_in_nonblocking_mode() { + use std::os::fd::AsRawFd; + + let path = temp_origin_file("opj", OPENOPJ_FIXTURE); + let file = open_file_for_classification(&path).unwrap(); + let flags = unsafe { libc::fcntl(file.as_raw_fd(), libc::F_GETFL) }; + std::fs::remove_file(path).unwrap(); + + assert_ne!(flags, -1, "F_GETFL must succeed"); + assert_ne!(flags & libc::O_NONBLOCK, 0); +} + +#[test] +fn origin_rewind_and_full_read_errors_are_propagated() { + let limits = OriginLimits::default(); + let rewind_error = read_origin_handle(&mut SeekFailure, Some(0), limits) + .expect_err("rewind errors must stop the import"); + assert_eq!(rewind_error.stage, "rewind"); + assert!( + rewind_error.detail.contains("injected rewind failure"), + "{}", + rewind_error.detail + ); + + let read_error = read_origin_handle(&mut ReadFailure, Some(0), limits) + .expect_err("full-read errors must stop the import"); + assert_eq!(read_error.stage, "read"); + assert!( + read_error.detail.contains("injected full-read failure"), + "{}", + read_error.detail + ); +} + +#[test] +fn origin_oversized_metadata_is_rejected_before_rewind() { + let limits = OriginLimits { + max_input_bytes: 4, + max_total_owned_bytes: 8, + ..OriginLimits::default() + }; + let oversized = 5; + + let error = read_origin_handle(&mut SeekFailure, Some(oversized), limits) + .expect_err("known oversized input must be rejected before rewinding"); + + assert_eq!(error.stage, "metadata"); + assert_eq!( + error.detail, + OriginError::LimitExceeded { + resource: "input bytes", + limit: 4, + actual: 5, + } + .to_string() + ); +} + +#[test] +fn origin_lower_total_owned_metadata_limit_is_rejected_before_rewind() { + let limits = OriginLimits { + max_input_bytes: 8, + max_total_owned_bytes: 4, + ..OriginLimits::default() + }; + + let error = read_origin_handle(&mut SeekFailure, Some(5), limits) + .expect_err("known cumulative oversize must be rejected before rewinding"); + + assert_eq!(error.stage, "metadata"); + assert_eq!( + error.detail, + OriginError::LimitExceeded { + resource: "total owned bytes", + limit: 4, + actual: 5, + } + .to_string() + ); +} + +#[test] +fn origin_unknown_length_stops_at_the_lower_total_owned_sentinel() { + let limits = OriginLimits { + max_input_bytes: 8, + max_total_owned_bytes: 4, + ..OriginLimits::default() + }; + let bytes_read = Rc::new(Cell::new(0)); + let reader = CountingRepeat { + bytes_read: Rc::clone(&bytes_read), + }; + + let error = read_bounded_origin(reader, None, limits).unwrap_err(); + + assert_eq!(bytes_read.get(), 5); + assert_eq!( + error, + OriginError::LimitExceeded { + resource: "total owned bytes", + limit: 4, + actual: 5, + } + .to_string() + ); +} + +#[cfg(unix)] +#[test] +fn origin_routing_metadata_errors_are_not_silently_discarded() { + use std::os::unix::fs::symlink; + + let path = std::env::temp_dir().join(format!( + "plotx-origin-metadata-loop-{}.opj", + uuid::Uuid::new_v4() + )); + symlink(&path, &path).unwrap(); + + let result = classify_open_path(&path); + let mut app = PlotxApp::new_with_settings(plotx_core::settings::Settings::default()); + open_recent_path(&mut app, &path); + + std::fs::remove_file(path).unwrap(); + assert!( + result.is_err(), + "metadata errors must be propagated: {result:?}" + ); + let report = app + .session + .operation_history + .operations() + .next_back() + .expect("routing metadata errors must be user-visible"); + assert_eq!(report.outcome, OperationOutcome::Failure); + assert!( + report + .diagnostics + .iter() + .any(|diagnostic| diagnostic.source.as_deref() == Some("app.open_path")), + "{report:?}" + ); +} + +#[test] +fn origin_header_read_errors_are_propagated_by_the_shared_classifier() { + let result = classify_open_path_with_header( + Path::new("unreadable.bin"), + OpenPathEntryType::RegularFile, + || { + Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "injected header read failure", + )) + }, + ); + + let error = result.expect_err("header read errors must not fall back to an extension route"); + assert!(error.to_string().contains("header"), "{error}"); + assert!( + error.to_string().contains("injected header read failure"), + "{error}" + ); +} + +#[test] +fn origin_non_regular_file_classification_never_reads_a_header() { + let result = + classify_open_path_with_header(Path::new("stream.opj"), OpenPathEntryType::Other, || { + panic!("non-regular paths must be rejected before header reads") + }); + + let error = result.expect_err("non-regular paths must be rejected"); + assert!(error.to_string().contains("regular file"), "{error}"); +} + +#[cfg(unix)] +#[test] +fn origin_routing_rejects_non_regular_files_without_opening_them() { + use std::os::unix::net::UnixListener; + + let path = PathBuf::from("/tmp").join(format!("px-{}.opj", uuid::Uuid::new_v4())); + let listener = UnixListener::bind(&path).unwrap(); + + let result = classify_open_path(&path); + + drop(listener); + std::fs::remove_file(path).unwrap(); + assert!( + result.is_err(), + "non-regular files must be rejected before opening: {result:?}" + ); +} + +#[test] +fn origin_opj_extension_rejects_an_opju_signature() { + let path = temp_origin_file("opj", b"CPYUA 4.3668 178\n"); + let mut app = PlotxApp::new_with_settings(plotx_core::settings::Settings::default()); + + open_recent_path(&mut app, &path); + + std::fs::remove_file(path).unwrap(); + assert!(app.session.ui.table_import_preview.is_none()); + assert!(app.session.recent_files.is_empty()); + let report = app + .session + .operation_history + .operations() + .next_back() + .expect("the extension/signature mismatch must be reported"); + assert_eq!(report.outcome, OperationOutcome::Failure); + assert!(report.summary.contains("does not match"), "{report:?}"); +} + +#[test] +fn origin_opju_extension_rejects_an_opj_signature() { + let path = temp_origin_file("opju", OPENOPJ_FIXTURE); + let mut app = PlotxApp::new_with_settings(plotx_core::settings::Settings::default()); + + open_recent_path(&mut app, &path); + + std::fs::remove_file(path).unwrap(); + assert!(app.session.ui.table_import_preview.is_none()); + assert!(app.session.recent_files.is_empty()); + let report = app + .session + .operation_history + .operations() + .next_back() + .expect("the extension/signature mismatch must be reported"); + assert_eq!(report.outcome, OperationOutcome::Failure); + assert!(report.summary.contains("does not match"), "{report:?}"); +} + +#[test] +fn origin_default_limit_accepts_exactly_128_mib_and_rejects_one_more_byte() { + let limits = OriginLimits::default(); + assert_eq!(limits.max_input_bytes, 128 * 1024 * 1024); + + let exact = read_bounded_origin( + io::repeat(0).take(limits.max_input_bytes as u64), + Some(limits.max_input_bytes as u64), + limits, + ) + .expect("the exact default limit must be accepted"); + assert_eq!(exact.len(), limits.max_input_bytes); + drop(exact); + + let error = read_bounded_origin(PanicOnRead, Some(limits.max_input_bytes as u64 + 1), limits) + .expect_err("one byte beyond the default limit must be rejected"); + assert!(error.contains("exceeding"), "{error}"); +} + +#[test] +fn origin_usize_max_input_limit_is_rejected_before_reading() { + let limits = OriginLimits { + max_input_bytes: usize::MAX, + ..OriginLimits::default() + }; + let error = read_bounded_origin(PanicOnRead, None, limits).unwrap_err(); + assert!( + error.contains("invalid Origin limit max_input_bytes"), + "{error}" + ); +} + +#[test] +fn recent_entries_route_to_origin_project_import() { + let classify = |path: &Path| { + classify_open_path_with_header(path, OpenPathEntryType::RegularFile, || { + Ok(([0_u8; OPEN_HEADER_BYTES], 0)) + }) + .unwrap() + }; + assert_eq!( + format!("{:?}", classify(Path::new("project.OPJU"))), + "OriginProject" + ); + assert_ne!(classify(Path::new("project.opj")), RecentOpenKind::DataFile); +} + +#[test] +fn origin_signature_mismatch_becomes_a_user_visible_failure_report() { + let path = temp_origin_file("opj", b"not an Origin project"); + let mut app = PlotxApp::new_with_settings(plotx_core::settings::Settings::default()); + + open_recent_path(&mut app, &path); + + std::fs::remove_file(path).unwrap(); + assert!(app.session.ui.table_import_preview.is_none()); + assert!(app.session.recent_files.is_empty()); + let report = app + .session + .operation_history + .operations() + .next_back() + .expect("a user-visible failure report"); + assert_eq!(report.outcome, OperationOutcome::Failure); + assert!( + report.diagnostics.iter().any(|diagnostic| diagnostic + .message + .to_ascii_lowercase() + .contains("signature")), + "{report:?}" + ); +} + +#[test] +fn origin_opju_is_unsupported_without_preview_or_recent_entry() { + let path = temp_origin_file("opju", b"CPYUA 4.3668 178\n"); + let mut app = PlotxApp::new_with_settings(plotx_core::settings::Settings::default()); + + open_recent_path(&mut app, &path); + + std::fs::remove_file(path).unwrap(); + assert!(app.session.ui.table_import_preview.is_none()); + assert!(app.session.recent_files.is_empty()); + let report = app + .session + .operation_history + .operations() + .next_back() + .expect("OPJU must produce a failure report"); + assert_eq!(report.outcome, OperationOutcome::Failure); + assert!(report.summary.contains("OPJU"), "{report:?}"); + assert!( + report.summary.contains("No data was imported"), + "{report:?}" + ); +} + +#[test] +fn unsupported_classic_opj_reports_its_version_and_supported_profile() { + let path = temp_origin_file("opj", b"CPYA 4.3224 220 W64 #\n"); + let mut app = PlotxApp::new_with_settings(plotx_core::settings::Settings::default()); + + open_recent_path(&mut app, &path); + + std::fs::remove_file(path).unwrap(); + assert!(app.session.ui.table_import_preview.is_none()); + assert!(app.session.recent_files.is_empty()); + let report = app + .session + .operation_history + .operations() + .next_back() + .expect("unsupported OPJ must produce a failure report"); + assert_eq!(report.outcome, OperationOutcome::Failure); + assert!(report.summary.contains("4.3224 220 W64"), "{report:?}"); + assert!(report.summary.contains("Origin 7.0552"), "{report:?}"); + assert!( + report.summary.contains("Origin 9.51 build 195 W64"), + "{report:?}" + ); + assert!( + report.summary.contains("No data was imported"), + "{report:?}" + ); +} + +#[test] +fn origin_core_failure_becomes_a_user_visible_operation_report() { + let probe = plotx_io::origin::probe_origin(OPENOPJ_FIXTURE).unwrap(); + let project = plotx_io::origin::OriginProject { + probe, + parameters: Vec::new(), + notes: Vec::new(), + workbooks: Vec::new(), + diagnostics: Vec::new(), + unsupported_objects: Vec::new(), + resource_usage: plotx_io::origin::OriginResourceUsage::default(), + }; + let mut app = PlotxApp::new_with_settings(plotx_core::settings::Settings::default()); + + import_origin_project_model( + &mut app, + Path::new("empty.opj"), + Arc::<[u8]>::from(OPENOPJ_FIXTURE), + project, + OriginLimits::default(), + ); + + let report = app + .session + .operation_history + .operations() + .next_back() + .expect("core failures must be recorded"); + assert_eq!(report.outcome, OperationOutcome::Failure); + assert!( + report + .diagnostics + .iter() + .any(|diagnostic| diagnostic.message.contains("no supported")), + "{report:?}" + ); +} + +#[test] +fn origin_recent_file_is_recorded_only_after_confirmed_full_success() { + let path = temp_origin_file("opj", OPENOPJ_FIXTURE); + let mut app = PlotxApp::new_with_settings(plotx_core::settings::Settings::default()); + let mut noted_paths = Vec::new(); + + open_recent_path(&mut app, &path); + + assert!(app.doc.datasets.is_empty()); + assert!(app.session.recent_files.is_empty()); + assert_eq!(app.session.operation_history.operation_count(), 0); + let candidate_count = app + .session + .ui + .table_import_preview + .as_ref() + .expect("valid OPJ should produce a preview") + .candidates + .len(); + assert!(candidate_count > 0); + + assert!( + crate::ui::file_dialogs::commit_table_import_preview_with_recent(&mut app, |app, path| { + let path = std::path::absolute(path).unwrap(); + noted_paths.push(path.clone()); + app.session.recent_files.push(path); + },) + ); + assert_eq!(app.doc.datasets.len(), candidate_count); + assert_eq!(app.session.recent_files.len(), 1); + assert_eq!( + app.session.recent_files[0], + std::path::absolute(&path).unwrap() + ); + assert_eq!(noted_paths, app.session.recent_files); + std::fs::remove_file(path).unwrap(); +} + +#[test] +fn origin_cancel_leaves_tables_and_recent_files_unchanged() { + let path = temp_origin_file("opj", OPENOPJ_FIXTURE); + let mut app = PlotxApp::new_with_settings(plotx_core::settings::Settings::default()); + open_recent_path(&mut app, &path); + assert!(app.session.ui.table_import_preview.take().is_some()); + + std::fs::remove_file(path).unwrap(); + assert!(app.doc.datasets.is_empty()); + assert!(app.session.recent_files.is_empty()); + assert_eq!(app.session.operation_history.operation_count(), 0); +} + +#[test] +fn origin_candidates_share_source_allocation_and_stable_operation() { + let (store, imported) = duplicated_fixture_import(); + let source_bytes = Arc::<[u8]>::from(OPENOPJ_FIXTURE); + let source_pointer = source_bytes.as_ptr(); + let preview = preview_from_imported( + OperationId(41), + Path::new("selected-project.opj"), + source_bytes, + store, + imported, + ) + .unwrap(); + + assert_eq!(preview.candidates.len(), 2); + for candidate in &preview.candidates { + let source = &candidate.retained_sources[0]; + assert_eq!(source.bytes().as_ptr(), source_pointer); + assert_eq!(source.media_type, "application/x-origin-project"); + assert_eq!(source.name.as_deref(), Some("selected-project.opj")); + assert!( + source + .metadata + .contains_key("space.nmrtist.plotx.import.origin.resource_usage") + ); + assert_eq!( + candidate.typed_state.envelope.revision.operation.name, + ORIGIN_IMPORT_OPERATION + ); + } +} + +#[test] +fn origin_preview_reports_each_parser_warning_once() { + let (store, imported) = duplicated_fixture_import(); + let expected_warnings = imported[0] + .diagnostics + .iter() + .filter(|diagnostic| diagnostic.severity == OriginDiagnosticSeverity::Warning) + .count(); + assert!(expected_warnings > 0); + + let preview = preview_from_imported( + OperationId(43), + Path::new("selected-project.opj"), + Arc::<[u8]>::from(OPENOPJ_FIXTURE), + store, + imported, + ) + .unwrap(); + let actual_warnings = preview + .report + .diagnostics + .iter() + .filter(|diagnostic| diagnostic.severity == Severity::Warning) + .count(); + + assert_eq!(actual_warnings, expected_warnings); +} + +#[test] +fn origin_zero_candidates_fails_without_indexing_candidate_zero() { + let result = std::panic::catch_unwind(|| ensure_candidate_count(0)); + let error = result.expect("zero candidates must not panic").unwrap_err(); + assert!(error.contains("no supported"), "{error}"); +} + +#[test] +fn origin_selector_changes_preview_only_and_confirmation_imports_all_tables() { + assert_eq!( + crate::ui::file_dialogs::preview::candidate_selector_label(), + "Table" + ); + assert_eq!( + crate::ui::file_dialogs::preview::all_candidate_import_summary(2), + "All 2 candidate tables will be imported." + ); + let (store, imported) = duplicated_fixture_import(); + let mut preview = preview_from_imported( + OperationId(42), + Path::new("selected-project.opj"), + Arc::<[u8]>::from(OPENOPJ_FIXTURE), + store, + imported, + ) + .unwrap(); + preview.selected = 1; + preview.recent_path = None; + let mut app = PlotxApp::new_with_settings(plotx_core::settings::Settings::default()); + app.session.ui.table_import_preview = Some(preview); + + assert!( + crate::ui::file_dialogs::commit_table_import_preview_with_recent(&mut app, |_, _| panic!( + "a preview without a recent path must not persist settings" + ),) + ); + assert_eq!(app.doc.datasets.len(), 2); + assert!(app.session.recent_files.is_empty()); +} diff --git a/crates/app/src/ui/file_dialogs/preview.rs b/crates/app/src/ui/file_dialogs/preview.rs index e4cce5e..760ad32 100644 --- a/crates/app/src/ui/file_dialogs/preview.rs +++ b/crates/app/src/ui/file_dialogs/preview.rs @@ -3,10 +3,31 @@ use plotx_core::state::{PlotxApp, TableImportPreviewState}; use super::commit_table_import_preview; +pub(super) fn candidate_selector_label() -> &'static str { + "Table" +} + +pub(super) fn all_candidate_import_summary(count: usize) -> String { + if count == 1 { + "The candidate table will be imported.".to_owned() + } else { + format!("All {count} candidate tables will be imported.") + } +} + pub(crate) fn table_import_preview_window(app: &mut PlotxApp, ctx: &egui::Context) { let Some(mut preview) = app.session.ui.table_import_preview.take() else { return; }; + if preview.candidates.is_empty() { + app.session.ui.table_import_preview = Some(preview); + let committed = commit_table_import_preview(app); + debug_assert!(!committed); + return; + } + if preview.selected >= preview.candidates.len() { + preview.selected = 0; + } let mut import = false; let mut cancel = false; let modal = super::super::modal( @@ -20,7 +41,7 @@ pub(crate) fn table_import_preview_window(app: &mut PlotxApp, ctx: &egui::Contex ui.label("Confirm the inferred schema before the table is added to the project."); ui.separator(); if preview.candidates.len() > 1 { - egui::ComboBox::from_label("Worksheet") + egui::ComboBox::from_label(candidate_selector_label()) .selected_text(&preview.candidates[preview.selected].name) .show_ui(ui, |ui| { for (index, candidate) in preview.candidates.iter().enumerate() { @@ -64,12 +85,7 @@ fn import_summary(ui: &mut egui::Ui, preview: &TableImportPreviewState) { let candidate = &preview.candidates[preview.selected]; let snapshot = &candidate.typed_state.envelope.revision.snapshot; ui.label(format!("Name: {}", candidate.name)); - if preview.candidates.len() > 1 { - ui.label(format!( - "{} worksheet(s) will be imported", - preview.candidates.len() - )); - } + ui.label(all_candidate_import_summary(preview.candidates.len())); ui.label(format!( "{} row(s), {} column(s)", snapshot.row_count, @@ -85,6 +101,7 @@ fn import_summary(ui: &mut egui::Ui, preview: &TableImportPreviewState) { fn schema_table(ui: &mut egui::Ui, preview: &TableImportPreviewState) { ui.strong("Inferred schema"); egui::ScrollArea::vertical() + .id_salt("table_import_schema_scroll") .max_height(170.0) .show(ui, |ui| { egui::Grid::new("table_import_schema_grid") @@ -133,6 +150,7 @@ fn value_preview(ui: &mut egui::Ui, preview: &TableImportPreviewState) { return; }; egui::ScrollArea::horizontal() + .id_salt("table_import_value_scroll") .max_height(210.0) .show(ui, |ui| { egui::Grid::new("table_import_value_grid") diff --git a/crates/app/src/ui/file_dialogs/recent.rs b/crates/app/src/ui/file_dialogs/recent.rs index 6595cf1..12e834e 100644 --- a/crates/app/src/ui/file_dialogs/recent.rs +++ b/crates/app/src/ui/file_dialogs/recent.rs @@ -1,42 +1,393 @@ use super::{ PlotxApp, import_delimited_table_path, import_xlsx_table_path, load_and_note, open_folder_path, + origin, }; +use plotx_core::operation::{Diagnostic, DiagnosticCode, OperationKind, OperationReport, Severity}; +use plotx_io::origin::{OriginError, OriginFormat}; +use std::fmt; +use std::fs::{File, OpenOptions}; +use std::io::{self, Read}; +use std::path::Path; + +pub(super) const OPEN_HEADER_BYTES: usize = plotx_io::origin::MAX_PROBE_BYTES; +type OpenHeader = ([u8; OPEN_HEADER_BYTES], usize); #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum RecentOpenKind { Project, DelimitedTable, XlsxTable, + OriginProject, + Folder, + DataFile, +} + +impl RecentOpenKind { + fn is_table_import(self) -> bool { + matches!( + self, + Self::DelimitedTable | Self::XlsxTable | Self::OriginProject + ) + } +} + +#[derive(Debug)] +pub(crate) enum ClassifiedOpenPath { + Project, + DelimitedTable, + XlsxTable, + OriginProject(origin::OpenOriginSource), Folder, DataFile, } -pub(crate) fn recent_open_kind(path: &std::path::Path) -> RecentOpenKind { +impl ClassifiedOpenPath { + pub(crate) fn kind(&self) -> RecentOpenKind { + match self { + Self::Project => RecentOpenKind::Project, + Self::DelimitedTable => RecentOpenKind::DelimitedTable, + Self::XlsxTable => RecentOpenKind::XlsxTable, + Self::OriginProject(_) => RecentOpenKind::OriginProject, + Self::Folder => RecentOpenKind::Folder, + Self::DataFile => RecentOpenKind::DataFile, + } + } + + fn is_table_import(&self) -> bool { + self.kind().is_table_import() + } +} + +#[cfg(test)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum OpenPathEntryType { + Directory, + RegularFile, + Other, +} + +#[derive(Debug)] +pub(crate) enum OpenPathError { + Io { + stage: &'static str, + error: io::Error, + }, + OriginProbe(OriginError), + OriginFamilyMismatch { + extension: &'static str, + detected: &'static str, + }, + NonRegularFile, +} + +impl OpenPathError { + fn io(stage: &'static str, error: io::Error) -> Self { + Self::Io { stage, error } + } + + fn stage(&self) -> &'static str { + match self { + Self::Io { stage, .. } => stage, + Self::OriginProbe(_) => "origin_probe", + Self::OriginFamilyMismatch { .. } => "origin_family", + Self::NonRegularFile => "file_type", + } + } +} + +impl fmt::Display for OpenPathError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Io { + stage: "metadata", + error, + } => write!( + formatter, + "the selected path could not be inspected: {error}" + ), + Self::Io { + stage: "open", + error, + } => write!(formatter, "the selected file could not be opened: {error}"), + Self::Io { + stage: "handle_metadata", + error, + } => write!(formatter, "the opened file could not be inspected: {error}"), + Self::Io { error, .. } => { + write!( + formatter, + "the selected file header could not be read: {error}" + ) + } + Self::OriginProbe(OriginError::UnsupportedVersion { raw_version }) => write!( + formatter, + "PlotX recognized classic Origin project version {raw_version}, but this build imports only the verified Origin 7.0552 and Origin 9.51 build 195 W64 OPJ profiles. No data was imported" + ), + Self::OriginProbe(error) => { + write!(formatter, "Origin project detection failed: {error}") + } + Self::OriginFamilyMismatch { + extension, + detected, + } => write!( + formatter, + "the .{extension} extension does not match the detected {detected} project signature; no data was imported" + ), + Self::NonRegularFile => write!( + formatter, + "the selected path is neither a regular file nor a directory" + ), + } + } +} + +impl std::error::Error for OpenPathError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Self::Io { error, .. } => Some(error), + Self::OriginProbe(error) => Some(error), + Self::OriginFamilyMismatch { .. } | Self::NonRegularFile => None, + } + } +} + +pub(crate) fn classify_open_path(path: &Path) -> Result { + let metadata = std::fs::metadata(path).map_err(|error| OpenPathError::io("metadata", error))?; + if metadata.is_dir() { + return Ok(ClassifiedOpenPath::Folder); + } + if !metadata.is_file() { + return Err(OpenPathError::NonRegularFile); + } + let file = + open_file_for_classification(path).map_err(|error| OpenPathError::io("open", error))?; + classify_open_handle(path, file) +} + +pub(crate) fn open_file_for_classification(path: &Path) -> io::Result { + let mut options = OpenOptions::new(); + options.read(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.custom_flags(libc::O_NONBLOCK); + } + options.open(path) +} + +pub(crate) fn classify_open_handle( + path: &Path, + mut file: File, +) -> Result { + let metadata = file + .metadata() + .map_err(|error| OpenPathError::io("handle_metadata", error))?; + if !metadata.is_file() { + return Err(OpenPathError::NonRegularFile); + } + let (header, length) = + read_open_header(&mut file).map_err(|error| OpenPathError::io("header", error))?; + let kind = classify_open_header(path, &header[..length])?; + Ok(match kind { + RecentOpenKind::Project => ClassifiedOpenPath::Project, + RecentOpenKind::DelimitedTable => ClassifiedOpenPath::DelimitedTable, + RecentOpenKind::XlsxTable => ClassifiedOpenPath::XlsxTable, + RecentOpenKind::OriginProject => { + ClassifiedOpenPath::OriginProject(origin::OpenOriginSource::new(file, metadata.len())) + } + RecentOpenKind::Folder => ClassifiedOpenPath::Folder, + RecentOpenKind::DataFile => ClassifiedOpenPath::DataFile, + }) +} + +#[cfg(test)] +pub(crate) fn classify_open_path_with_header( + path: &Path, + entry_type: OpenPathEntryType, + read_header: F, +) -> Result +where + F: FnOnce() -> io::Result, +{ + match entry_type { + OpenPathEntryType::Directory => return Ok(RecentOpenKind::Folder), + OpenPathEntryType::Other => return Err(OpenPathError::NonRegularFile), + OpenPathEntryType::RegularFile => {} + } + + let (header, length) = read_header().map_err(|error| OpenPathError::io("header", error))?; + classify_open_header(path, &header[..length]) +} + +fn classify_open_header(path: &Path, header: &[u8]) -> Result { + if header.starts_with(b"CPYA") || header.starts_with(b"CPYUA") { + let probe = plotx_io::origin::probe_origin(header).map_err(OpenPathError::OriginProbe)?; + reject_origin_family_mismatch(path, probe.format)?; + return Ok(RecentOpenKind::OriginProject); + } + + Ok(extension_open_kind(path)) +} + +fn reject_origin_family_mismatch(path: &Path, detected: OriginFormat) -> Result<(), OpenPathError> { + let extension = path.extension().and_then(|extension| extension.to_str()); + let expected = match extension { + Some(extension) if extension.eq_ignore_ascii_case("opj") => Some(OriginFormat::Opj), + Some(extension) if extension.eq_ignore_ascii_case("opju") => Some(OriginFormat::Opju), + _ => None, + }; + if let Some(expected) = expected + && expected != detected + { + return Err(OpenPathError::OriginFamilyMismatch { + extension: match expected { + OriginFormat::Opj => "opj", + OriginFormat::Opju => "opju", + }, + detected: match detected { + OriginFormat::Opj => "OPJ", + OriginFormat::Opju => "OPJU", + }, + }); + } + Ok(()) +} + +fn extension_open_kind(path: &Path) -> RecentOpenKind { let has_extension = |target: &str| { path.extension() .is_some_and(|extension| extension.eq_ignore_ascii_case(target)) }; - if path.is_dir() { - RecentOpenKind::Folder - } else if has_extension("plotx") { + if has_extension("plotx") { RecentOpenKind::Project } else if has_extension("csv") || has_extension("tsv") || has_extension("txt") { RecentOpenKind::DelimitedTable } else if has_extension("xlsx") { RecentOpenKind::XlsxTable + } else if has_extension("opj") || has_extension("opju") { + RecentOpenKind::OriginProject } else { RecentOpenKind::DataFile } } -pub(crate) fn open_recent_path(app: &mut PlotxApp, path: &std::path::Path) { - match recent_open_kind(path) { - RecentOpenKind::Project => app.request_project_transition( +pub(crate) fn recent_open_kind(path: &Path) -> RecentOpenKind { + if path.is_dir() { + RecentOpenKind::Folder + } else { + extension_open_kind(path) + } +} + +fn read_open_header(mut reader: R) -> io::Result { + let mut header = [0_u8; OPEN_HEADER_BYTES]; + let mut length = 0; + while length < header.len() { + match reader.read(&mut header[length..]) { + Ok(0) => break, + Ok(read) => length += read, + Err(error) if error.kind() == io::ErrorKind::Interrupted => continue, + Err(error) => return Err(error), + } + } + Ok((header, length)) +} + +pub(crate) fn open_recent_path(app: &mut PlotxApp, path: &Path) { + let classified = match classify_open_path(path) { + Ok(classified) => classified, + Err(error) => { + record_open_path_failure(app, path, error); + return; + } + }; + dispatch_classified_path(app, path, classified); +} + +pub(crate) fn dispatch_classified_path( + app: &mut PlotxApp, + path: &Path, + classified: ClassifiedOpenPath, +) { + if classified.is_table_import() && app.session.ui.table_import_preview.is_some() { + record_pending_table_import(app, path); + return; + } + match classified { + ClassifiedOpenPath::Project => app.request_project_transition( plotx_core::state::ProjectTransition::Open(path.to_owned()), ), - RecentOpenKind::DelimitedTable => import_delimited_table_path(app, path), - RecentOpenKind::XlsxTable => import_xlsx_table_path(app, path), - RecentOpenKind::Folder => open_folder_path(app, path), - RecentOpenKind::DataFile => load_and_note(app, path), + ClassifiedOpenPath::DelimitedTable => import_delimited_table_path(app, path), + ClassifiedOpenPath::XlsxTable => import_xlsx_table_path(app, path), + ClassifiedOpenPath::OriginProject(source) => { + origin::import_origin_project_source(app, path, source); + } + ClassifiedOpenPath::Folder => open_folder_path(app, path), + ClassifiedOpenPath::DataFile => load_and_note(app, path), } } + +fn record_open_path_failure(app: &mut PlotxApp, path: &Path, error: OpenPathError) { + let operation_id = app.session.begin_operation(); + let message = error.to_string(); + let (operation_kind, diagnostic_code) = open_path_failure_classification(path, &error); + app.session.record_operation(OperationReport::<()>::failure( + operation_id, + operation_kind, + format!("The selected path could not be opened: {message}."), + Diagnostic::new(Severity::Error, diagnostic_code, message) + .with_source("app.open_path") + .with_context("path", path.display().to_string()) + .with_context("stage", error.stage()) + .with_context("error", error.to_string()), + )); +} + +fn open_path_failure_classification( + path: &Path, + error: &OpenPathError, +) -> (OperationKind, DiagnosticCode) { + let table_like_extension = path + .extension() + .and_then(|extension| extension.to_str()) + .is_some_and(|extension| { + ["csv", "tsv", "txt", "xlsx", "opj", "opju"] + .iter() + .any(|candidate| extension.eq_ignore_ascii_case(candidate)) + }); + if table_like_extension + || matches!( + error, + OpenPathError::OriginProbe(_) | OpenPathError::OriginFamilyMismatch { .. } + ) + { + ( + OperationKind::TableImport, + DiagnosticCode::TableImportFailed, + ) + } else { + ( + OperationKind::DatasetLoad, + DiagnosticCode::DatasetLoadFailed, + ) + } +} + +fn record_pending_table_import(app: &mut PlotxApp, path: &Path) { + let operation_id = app.session.begin_operation(); + let message = + "Finish or cancel the current table import preview before importing another table."; + app.session.record_operation(OperationReport::<()>::failure( + operation_id, + OperationKind::TableImport, + message, + Diagnostic::new(Severity::Error, DiagnosticCode::TableImportFailed, message) + .with_source("app.table_import") + .with_context("path", path.display().to_string()) + .with_context("stage", "preview_pending"), + )); +} + +#[cfg(test)] +#[path = "recent_report_tests.rs"] +mod recent_report_tests; diff --git a/crates/app/src/ui/file_dialogs/recent_report_tests.rs b/crates/app/src/ui/file_dialogs/recent_report_tests.rs new file mode 100644 index 0000000..7895a36 --- /dev/null +++ b/crates/app/src/ui/file_dialogs/recent_report_tests.rs @@ -0,0 +1,117 @@ +use super::*; +use plotx_core::operation::{DiagnosticCode, OperationKind}; +use plotx_core::settings::Settings; +use std::path::{Path, PathBuf}; + +fn unique_path(extension: &str) -> PathBuf { + std::env::temp_dir().join(format!( + "plotx-recent-report-{}.{}", + uuid::Uuid::new_v4(), + extension + )) +} + +fn write_temp(extension: &str, bytes: &[u8]) -> PathBuf { + let path = unique_path(extension); + std::fs::write(&path, bytes).unwrap(); + path +} + +fn assert_latest_failure( + app: &PlotxApp, + expected_kind: OperationKind, + expected_code: DiagnosticCode, +) { + let report = app + .session + .operation_history + .operations() + .next_back() + .expect("the open failure must be reported"); + assert_eq!(report.kind, expected_kind, "{report:?}"); + assert!( + report + .diagnostics + .iter() + .any(|diagnostic| diagnostic.code == expected_code), + "{report:?}" + ); +} + +#[test] +fn missing_table_like_paths_report_table_import_failures() { + for extension in ["csv", "tsv", "txt", "xlsx", "opj", "opju"] { + let path = unique_path(extension); + assert!(!path.exists()); + let mut app = PlotxApp::new_with_settings(Settings::default()); + + open_recent_path(&mut app, &path); + + assert_latest_failure( + &app, + OperationKind::TableImport, + DiagnosticCode::TableImportFailed, + ); + } +} + +#[test] +fn origin_probe_errors_report_table_import_even_without_an_origin_extension() { + let path = write_temp("bin", b"CPYA invalid\n"); + let mut app = PlotxApp::new_with_settings(Settings::default()); + + open_recent_path(&mut app, &path); + + std::fs::remove_file(path).unwrap(); + assert_latest_failure( + &app, + OperationKind::TableImport, + DiagnosticCode::TableImportFailed, + ); +} + +#[test] +fn origin_family_mismatches_report_table_import_failures() { + let path = write_temp("opj", b"CPYUA 4.3668 178\n"); + let mut app = PlotxApp::new_with_settings(Settings::default()); + + open_recent_path(&mut app, &path); + + std::fs::remove_file(path).unwrap(); + assert_latest_failure( + &app, + OperationKind::TableImport, + DiagnosticCode::TableImportFailed, + ); +} + +#[test] +fn missing_dataset_and_project_paths_keep_dataset_load_failures() { + for path in [unique_path("abf"), unique_path("plotx")] { + assert!(!path.exists()); + let mut app = PlotxApp::new_with_settings(Settings::default()); + + open_recent_path(&mut app, Path::new(&path)); + + assert_latest_failure( + &app, + OperationKind::DatasetLoad, + DiagnosticCode::DatasetLoadFailed, + ); + } +} + +#[test] +fn later_origin_failures_remain_table_import_failures() { + let path = write_temp("opju", b"CPYUA 4.3668 178\n"); + let mut app = PlotxApp::new_with_settings(Settings::default()); + + open_recent_path(&mut app, &path); + + std::fs::remove_file(path).unwrap(); + assert_latest_failure( + &app, + OperationKind::TableImport, + DiagnosticCode::TableImportFailed, + ); +} diff --git a/crates/app/src/ui/file_dialogs/tests.rs b/crates/app/src/ui/file_dialogs/tests.rs index 3f648be..94b0a0f 100644 --- a/crates/app/src/ui/file_dialogs/tests.rs +++ b/crates/app/src/ui/file_dialogs/tests.rs @@ -1,6 +1,49 @@ use super::*; use std::path::PathBuf; +fn origin_9_initial_structure() -> Vec { + const HEADER_LEN: usize = 115; + let mut bytes = b"CPYA 4.3268 195 W64 #\n".to_vec(); + bytes.extend_from_slice(&(HEADER_LEN as u32).to_le_bytes()); + bytes.push(b'\n'); + let mut header = [0_u8; HEADER_LEN]; + header[0x1b..0x23].copy_from_slice(&9.510195_f64.to_le_bytes()); + bytes.extend_from_slice(&header); + bytes.push(b'\n'); + bytes.extend_from_slice(&[0, 0, 0, 0, b'\n']); + bytes +} + +#[test] +fn table_import_preview_uses_unique_scroll_area_ids() { + let mut app = PlotxApp::new_with_settings(plotx_core::settings::Settings::default()); + import_delimited_text( + &mut app, + "time,value\n0,1\n", + DelimitedTableSource::Clipboard, + ); + let ctx = egui::Context::default(); + + let output = ctx.run_ui(egui::RawInput::default(), |ui| { + table_import_preview_window(&mut app, ui.ctx()); + }); + let painted = format!("{:?}", output.shapes); + + assert!(!painted.contains("use of ScrollArea ID"), "{painted}"); +} + +#[test] +fn recent_path_classification_reads_enough_bytes_for_origin_9() { + let path = + std::env::temp_dir().join(format!("plotx-origin-9-probe-{}.opj", uuid::Uuid::new_v4())); + std::fs::write(&path, origin_9_initial_structure()).unwrap(); + + let classified = recent::classify_open_path(&path); + + std::fs::remove_file(path).unwrap(); + assert_eq!(classified.unwrap().kind(), RecentOpenKind::OriginProject); +} + #[test] fn mixed_columns_import_as_typed_text_without_being_discarded() { let mut app = PlotxApp::new(); @@ -139,30 +182,43 @@ fn clipboard_schema_restores_typed_contract_and_is_retained() { #[test] fn recent_entries_route_to_their_import_path() { let file = |name: &str| PathBuf::from(format!("C:/data/{name}")); + let regular = |path: &std::path::Path| { + recent::classify_open_path_with_header(path, recent::OpenPathEntryType::RegularFile, || { + Ok(([0_u8; recent::OPEN_HEADER_BYTES], 0)) + }) + .unwrap() + }; + assert_eq!(regular(&file("session.PLOTX")), RecentOpenKind::Project); assert_eq!( - recent_open_kind(&file("session.PLOTX")), - RecentOpenKind::Project + regular(&file("results.csv")), + RecentOpenKind::DelimitedTable ); assert_eq!( - recent_open_kind(&file("results.csv")), + regular(&file("results.tsv")), RecentOpenKind::DelimitedTable ); assert_eq!( - recent_open_kind(&file("results.tsv")), + regular(&file("results.txt")), RecentOpenKind::DelimitedTable ); + assert_eq!(regular(&file("results.XLSX")), RecentOpenKind::XlsxTable); + assert_eq!(regular(&file("run.abf")), RecentOpenKind::DataFile); + assert_eq!(regular(&file("fid")), RecentOpenKind::DataFile); assert_eq!( - recent_open_kind(&file("results.txt")), - RecentOpenKind::DelimitedTable + format!("{:?}", regular(&file("project.opj"))), + "OriginProject" ); assert_eq!( - recent_open_kind(&file("results.XLSX")), - RecentOpenKind::XlsxTable + format!("{:?}", regular(&file("project.OPJU"))), + "OriginProject" ); - assert_eq!(recent_open_kind(&file("run.abf")), RecentOpenKind::DataFile); - assert_eq!(recent_open_kind(&file("fid")), RecentOpenKind::DataFile); assert_eq!( - recent_open_kind(&std::env::temp_dir()), + recent::classify_open_path_with_header( + &std::env::temp_dir(), + recent::OpenPathEntryType::Directory, + || panic!("directories must not be opened for header reads"), + ) + .unwrap(), RecentOpenKind::Folder ); @@ -179,9 +235,32 @@ fn recent_entries_route_to_their_import_path() { std::fs::create_dir(&csv_directory).expect("create CSV-named directory"); std::fs::create_dir(&plotx_directory).expect("create PlotX-named directory"); let kinds = ( - recent_open_kind(&csv_directory), - recent_open_kind(&plotx_directory), + recent::classify_open_path(&csv_directory).unwrap().kind(), + recent::classify_open_path(&plotx_directory).unwrap().kind(), ); std::fs::remove_dir_all(&root).expect("remove recent-open test directory"); assert_eq!(kinds, (RecentOpenKind::Folder, RecentOpenKind::Folder)); } + +#[test] +fn project_dispatch_preserves_dirty_document_confirmation() { + let mut app = PlotxApp::new_with_settings(plotx_core::settings::Settings::default()); + app.mark_document_dirty(); + let path = PathBuf::from("pending-project.plotx"); + + recent::dispatch_classified_path(&mut app, &path, recent::ClassifiedOpenPath::Project); + + let pending = app + .session + .ui + .project_transition + .expect("a dirty document must defer the project open"); + assert_eq!( + pending.target, + plotx_core::state::ProjectTransition::Open(path) + ); + assert_eq!( + pending.phase, + plotx_core::state::ProjectTransitionPhase::NeedsConfirmation + ); +} diff --git a/crates/app/src/ui/file_dialogs/xlsx.rs b/crates/app/src/ui/file_dialogs/xlsx.rs index d0ca78c..014a317 100644 --- a/crates/app/src/ui/file_dialogs/xlsx.rs +++ b/crates/app/src/ui/file_dialogs/xlsx.rs @@ -45,8 +45,8 @@ pub(super) fn import_xlsx_table_path(app: &mut PlotxApp, path: &std::path::Path) operation_id, path, "worksheet_selection", - "The workbook has no visible data worksheets.", - "no visible non-empty worksheet".into(), + "The workbook has no visible data tables.", + "no visible non-empty table".into(), ); return; } @@ -68,7 +68,7 @@ pub(super) fn import_xlsx_table_path(app: &mut PlotxApp, path: &std::path::Path) let mut report = OperationReport::success( operation_id, OperationKind::TableImport, - format!("Imported {sheet_count} worksheet(s) from an XLSX workbook."), + format!("Imported {sheet_count} table(s) from an XLSX workbook."), (), ); let mut candidates = Vec::with_capacity(sheet_count); @@ -125,10 +125,7 @@ pub(super) fn import_xlsx_table_path(app: &mut PlotxApp, path: &std::path::Path) Diagnostic::new( Severity::Info, DiagnosticCode::TableImportSucceeded, - format!( - "Imported worksheet '{}' with {row_count} row(s).", - sheet.name - ), + format!("Imported table '{}' with {row_count} row(s).", sheet.name), ) .with_source("core.xlsx") .with_context("path", path.display().to_string()) @@ -150,7 +147,7 @@ pub(super) fn import_xlsx_table_path(app: &mut PlotxApp, path: &std::path::Path) if warning_count > 0 { report.outcome = plotx_core::operation::OperationOutcome::Warning; report.summary = - format!("Imported {sheet_count} XLSX worksheet(s) with {warning_count} warning(s)."); + format!("Imported {sheet_count} XLSX table(s) with {warning_count} warning(s)."); } app.session.ui.table_import_preview = Some(TableImportPreviewState { candidates, diff --git a/crates/app/src/ui/shortcuts.rs b/crates/app/src/ui/shortcuts.rs index e21e270..0f4371c 100644 --- a/crates/app/src/ui/shortcuts.rs +++ b/crates/app/src/ui/shortcuts.rs @@ -520,7 +520,7 @@ pub(super) fn handle_file_drop(app: &mut PlotxApp, ctx: &egui::Context) { painter.text( rect.center(), egui::Align2::CENTER_CENTER, - "Drop a .plotx project, a CSV/TSV table, an .abf/.jdf file, a Waters .raw or Bruker folder, or a .zip archive", + "Drop a PlotX project, table, Origin project, data file, data folder, or archive", egui::FontId::proportional(24.0), Color32::WHITE, ); diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index dd29a69..1273008 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -8,6 +8,13 @@ pub mod export; pub mod fit_model_library; pub mod layout; pub mod operation; +pub mod origin; +#[cfg(test)] +#[path = "origin_hardening_tests.rs"] +mod origin_hardening_tests; +#[cfg(test)] +#[path = "origin_tests.rs"] +mod origin_tests; pub mod project; pub mod properties; pub mod settings; diff --git a/crates/core/src/origin.rs b/crates/core/src/origin.rs new file mode 100644 index 0000000..cfd54a4 --- /dev/null +++ b/crates/core/src/origin.rs @@ -0,0 +1,720 @@ +//! Conversion from the bounded Origin transport model to PlotX tables. + +use std::{collections::BTreeMap, mem::size_of}; + +use plotx_data::{ + BlockStore, CodecRegistry, ColumnChunk, ColumnSchema, ColumnValues, LogicalType, RowId, + SnapshotBuilder, TableId, TableSchema, TableSnapshot, Validity, +}; +use plotx_io::origin::{ + OriginCell, OriginColumn, OriginColumnType, OriginDiagnostic, OriginDiagnosticCode, + OriginDiagnosticSeverity, OriginError, OriginLimits, OriginMetadataEntry, OriginNote, + OriginProject, OriginResourceUsage, OriginUnsupportedObjectSummary, OriginWorksheet, +}; +use serde_json::{Map, Value}; + +mod names; +mod preflight; + +/// Stable operation identifier stored in revisions created from Origin imports. +pub const ORIGIN_IMPORT_OPERATION: &str = "plotx.import.origin.v1"; + +const CHUNK_ROWS: usize = 65_536; + +const FORMAT_KEY: &str = "space.nmrtist.plotx.import.format"; +const VERSION_KEY: &str = "space.nmrtist.plotx.import.origin.producer_version"; +const WORKBOOK_KEY: &str = "space.nmrtist.plotx.import.origin.workbook"; +const WORKSHEET_KEY: &str = "space.nmrtist.plotx.import.origin.worksheet"; +const PARAMETERS_KEY: &str = "space.nmrtist.plotx.import.origin.parameters"; +const NOTES_KEY: &str = "space.nmrtist.plotx.import.origin.notes"; +const DIAGNOSTICS_KEY: &str = "space.nmrtist.plotx.import.origin.diagnostics"; +const UNSUPPORTED_KEY: &str = "space.nmrtist.plotx.import.origin.unsupported_objects"; +const USAGE_KEY: &str = "space.nmrtist.plotx.import.origin.resource_usage"; +const COLUMNS_KEY: &str = "space.nmrtist.plotx.import.origin.columns"; +const WORKSHEET_METADATA_KEY: &str = "space.nmrtist.plotx.import.origin.worksheet_metadata"; +const ORIGINAL_NAME_KEY: &str = "space.nmrtist.plotx.import.origin.original_name"; +const LONG_NAME_KEY: &str = "space.nmrtist.plotx.import.origin.long_name"; +const ROLE_KEY: &str = "space.nmrtist.plotx.import.origin.role"; +const UNITS_KEY: &str = "space.nmrtist.plotx.import.origin.units"; +const COMMENTS_KEY: &str = "space.nmrtist.plotx.import.origin.comments"; + +/// One worksheet ready for the application's existing import-preview flow. +#[derive(Debug)] +pub struct ImportedOriginWorksheet { + /// Human-readable candidate label preserving workbook and worksheet identity. + pub name: String, + /// Typed, validity-aware table snapshot. + pub snapshot: TableSnapshot, + /// Bounded metadata suitable for direct attachment to `TableImportSource`. + pub source_metadata: BTreeMap, + /// Recoverable parser diagnostics retained for the operation report. + pub diagnostics: Vec, + /// Shared cumulative parser and conversion allocation estimate. + pub resource_usage: OriginResourceUsage, +} + +struct PreparedOriginWorksheet { + name: String, + worksheet: OriginWorksheet, + schema: TableSchema, + source_metadata: BTreeMap, + snapshot_metadata: BTreeMap, + diagnostics: Vec, + resource_usage: OriginResourceUsage, +} + +/// Errors raised while validating or converting an engine-neutral Origin model. +#[derive(Debug, thiserror::Error)] +pub enum OriginImportError { + #[error(transparent)] + Origin(#[from] OriginError), + + #[error(transparent)] + Data(#[from] plotx_data::DataError), + + #[error("the Origin project contains no supported worksheet data")] + NoSupportedWorksheet, + + #[error("invalid Origin project model: {detail}")] + InvalidModel { detail: String }, + + #[error( + "Origin column {column:?} row {row} contains {actual}, but its declared type is {expected:?}" + )] + InvalidCellType { + column: String, + row: usize, + expected: OriginColumnType, + actual: &'static str, + }, + + #[error("Origin table conversion size calculation overflowed for {resource}")] + ArithmeticOverflow { resource: &'static str }, + + #[error( + "Origin table conversion {resource} is {actual}, exceeding the configured limit of {limit}" + )] + LimitExceeded { + resource: &'static str, + limit: usize, + actual: usize, + }, + + #[error("Origin table conversion could not reserve {requested} bytes for {resource}")] + AllocationFailed { + resource: &'static str, + requested: usize, + }, +} + +/// Converts every nonempty Origin worksheet into an independent typed snapshot. +/// +/// The complete neutral model is validated and conservatively charged against +/// `max_total_owned_bytes` before a snapshot builder or block-store write is +/// created. Source cell vectors are then drained batch by batch, so text cell +/// storage is moved rather than cloned. +pub fn import_origin_project( + project: OriginProject, + store: &dyn BlockStore, + codecs: &CodecRegistry, + limits: OriginLimits, +) -> Result, OriginImportError> { + limits.validate()?; + let preflight = preflight::validate(&project, &limits)?; + if preflight.worksheets.is_empty() { + return Err(OriginImportError::NoSupportedWorksheet); + } + + let OriginProject { + probe, + parameters, + notes, + workbooks, + diagnostics, + unsupported_objects, + mut resource_usage, + } = project; + resource_usage.total_owned_bytes = preflight.total_owned_bytes; + let candidate_count = preflight.worksheets.len(); + let mut worksheet_preflights = preflight.worksheets.into_iter(); + let mut prepared = Vec::new(); + try_reserve(&mut prepared, candidate_count, "prepared Origin worksheets")?; + + for workbook in workbooks { + let workbook_name = workbook.name; + for worksheet in workbook.worksheets { + if worksheet.row_count == 0 || worksheet.columns.is_empty() { + continue; + } + let worksheet_preflight = + worksheet_preflights + .next() + .ok_or_else(|| OriginImportError::InvalidModel { + detail: "worksheet preflight count does not match the retained model" + .to_owned(), + })?; + let source_metadata = source_metadata( + &probe.raw_version, + ¶meters, + ¬es, + &diagnostics, + &unsupported_objects, + &resource_usage, + &workbook_name, + &worksheet, + &worksheet_preflight.imported_names, + )?; + let snapshot_metadata = snapshot_metadata(&source_metadata)?; + let name = candidate_name(&workbook_name, &worksheet.name)?; + let schema = build_schema(&worksheet.columns, worksheet_preflight.imported_names)?; + prepared.push(PreparedOriginWorksheet { + name, + worksheet, + schema, + source_metadata, + snapshot_metadata, + diagnostics: diagnostics.clone(), + resource_usage: resource_usage.clone(), + }); + } + } + if worksheet_preflights.next().is_some() { + return Err(OriginImportError::InvalidModel { + detail: "worksheet preflight count does not match the retained model".to_owned(), + }); + } + + let mut imported = Vec::new(); + try_reserve( + &mut imported, + candidate_count, + "Origin worksheet candidates", + )?; + for prepared in prepared { + let snapshot = build_snapshot( + prepared.worksheet, + prepared.schema, + prepared.snapshot_metadata, + store, + codecs, + )?; + imported.push(ImportedOriginWorksheet { + name: prepared.name, + snapshot, + source_metadata: prepared.source_metadata, + diagnostics: prepared.diagnostics, + resource_usage: prepared.resource_usage, + }); + } + Ok(imported) +} + +fn build_snapshot( + mut worksheet: OriginWorksheet, + schema: TableSchema, + metadata: BTreeMap, + store: &dyn BlockStore, + codecs: &CodecRegistry, +) -> Result { + let mut builder = + SnapshotBuilder::new(TableId::new(), schema, store, codecs)?.with_trusted_row_identity(); + *builder.metadata_mut() = metadata; + + let mut row_start = 0_usize; + while row_start < worksheet.row_count { + let row_count = (worksheet.row_count - row_start).min(CHUNK_ROWS); + let mut chunks = Vec::new(); + try_reserve(&mut chunks, worksheet.columns.len(), "Origin column chunks")?; + for column in &mut worksheet.columns { + chunks.push(drain_column_chunk(column, row_start, row_count)?); + } + let mut row_ids = Vec::new(); + try_reserve(&mut row_ids, row_count, "Origin row identities")?; + row_ids.extend((0..row_count).map(|_| RowId::new())); + builder.push_batch(&row_ids, &chunks)?; + row_start = checked_add(row_start, row_count, "worksheet row offset")?; + } + Ok(builder.finish()?) +} + +fn build_schema( + columns: &[OriginColumn], + imported_names: Vec, +) -> Result { + let mut schemas = Vec::new(); + try_reserve(&mut schemas, columns.len(), "Origin column schemas")?; + for (column, name) in columns.iter().zip(imported_names) { + let logical_type = match column.column_type { + OriginColumnType::Float => LogicalType::Float64, + OriginColumnType::Integer => LogicalType::Int64, + OriginColumnType::Text | OriginColumnType::Mixed => LogicalType::Utf8, + }; + let changed = name != column.name; + let mut schema = ColumnSchema::new(name, logical_type); + if changed { + insert_text(&mut schema.metadata, ORIGINAL_NAME_KEY, &column.name)?; + } + for (key, value) in [ + (LONG_NAME_KEY, column.long_name.as_deref()), + (ROLE_KEY, column.role.as_deref()), + (UNITS_KEY, column.units.as_deref()), + (COMMENTS_KEY, column.comments.as_deref()), + ] { + if let Some(value) = value { + insert_text(&mut schema.metadata, key, value)?; + } + } + schemas.push(schema); + } + Ok(TableSchema::new(schemas)?) +} + +fn drain_column_chunk( + column: &mut OriginColumn, + row_start: usize, + row_count: usize, +) -> Result { + let take = row_count.min(column.cells.len()); + let column_name = column.name.as_str(); + let column_type = column.column_type; + let mut validity = Vec::new(); + try_reserve(&mut validity, row_count, "Origin validity input")?; + let values = match column.column_type { + OriginColumnType::Float => { + let mut values = Vec::new(); + try_reserve(&mut values, row_count, "Origin Float64 values")?; + for (index, cell) in column.cells.drain(..take).enumerate() { + match cell { + OriginCell::Float(value) => { + values.push(value); + validity.push(true); + } + OriginCell::Null => { + values.push(0.0); + validity.push(false); + } + other => { + return cell_type_error( + column_name, + column_type, + checked_add(row_start, index, "worksheet row index")?, + &other, + ); + } + } + } + values.resize(row_count, 0.0); + ColumnValues::Float64(values) + } + OriginColumnType::Integer => { + let mut values = Vec::new(); + try_reserve(&mut values, row_count, "Origin Int64 values")?; + for (index, cell) in column.cells.drain(..take).enumerate() { + match cell { + OriginCell::Integer(value) => { + values.push(value); + validity.push(true); + } + OriginCell::Null => { + values.push(0); + validity.push(false); + } + other => { + return cell_type_error( + column_name, + column_type, + checked_add(row_start, index, "worksheet row index")?, + &other, + ); + } + } + } + values.resize(row_count, 0); + ColumnValues::Int64(values) + } + OriginColumnType::Text | OriginColumnType::Mixed => { + let mut values = Vec::new(); + try_reserve(&mut values, row_count, "Origin UTF-8 values")?; + for (index, cell) in column.cells.drain(..take).enumerate() { + match cell { + OriginCell::Text(value) => { + values.push(value); + validity.push(true); + } + OriginCell::Float(value) if column.column_type == OriginColumnType::Mixed => { + values.push(value.to_string()); + validity.push(true); + } + OriginCell::Integer(value) if column.column_type == OriginColumnType::Mixed => { + values.push(value.to_string()); + validity.push(true); + } + OriginCell::Null => { + values.push(String::new()); + validity.push(false); + } + other => { + return cell_type_error( + column_name, + column_type, + checked_add(row_start, index, "worksheet row index")?, + &other, + ); + } + } + } + values.resize_with(row_count, String::new); + ColumnValues::Utf8(values) + } + }; + validity.resize(row_count, false); + Ok(ColumnChunk::new(values, Validity::from_valid(validity))?) +} + +fn cell_type_error( + column: &str, + expected: OriginColumnType, + row: usize, + cell: &OriginCell, +) -> Result { + Err(OriginImportError::InvalidCellType { + column: copy_text(column, "Origin column name")?, + row, + expected, + actual: cell_kind(cell), + }) +} + +#[allow(clippy::too_many_arguments)] +fn source_metadata( + version: &str, + parameters: &[OriginMetadataEntry], + notes: &[OriginNote], + diagnostics: &[OriginDiagnostic], + unsupported: &[OriginUnsupportedObjectSummary], + usage: &OriginResourceUsage, + workbook: &str, + worksheet: &OriginWorksheet, + imported_names: &[String], +) -> Result, OriginImportError> { + let mut metadata = BTreeMap::new(); + insert_text(&mut metadata, FORMAT_KEY, "opj")?; + insert_text(&mut metadata, VERSION_KEY, version)?; + insert_text(&mut metadata, WORKBOOK_KEY, workbook)?; + insert_text(&mut metadata, WORKSHEET_KEY, &worksheet.name)?; + metadata.insert(PARAMETERS_KEY.to_owned(), entries_json(parameters)?); + metadata.insert(NOTES_KEY.to_owned(), notes_json(notes)?); + metadata.insert(DIAGNOSTICS_KEY.to_owned(), diagnostics_json(diagnostics)?); + metadata.insert(UNSUPPORTED_KEY.to_owned(), unsupported_json(unsupported)?); + metadata.insert(USAGE_KEY.to_owned(), usage_json(usage)); + metadata.insert( + COLUMNS_KEY.to_owned(), + columns_json(&worksheet.columns, imported_names)?, + ); + metadata.insert( + WORKSHEET_METADATA_KEY.to_owned(), + entries_json(&worksheet.metadata)?, + ); + Ok(metadata) +} + +fn snapshot_metadata( + source: &BTreeMap, +) -> Result, OriginImportError> { + let mut metadata = BTreeMap::new(); + for key in [ + FORMAT_KEY, + VERSION_KEY, + WORKBOOK_KEY, + WORKSHEET_KEY, + DIAGNOSTICS_KEY, + ] { + let value = source + .get(key) + .ok_or_else(|| OriginImportError::InvalidModel { + detail: format!("source metadata is missing {key}"), + })?; + metadata.insert(key.to_owned(), value.clone()); + } + Ok(metadata) +} + +fn entries_json(entries: &[OriginMetadataEntry]) -> Result { + let mut values = Vec::new(); + try_reserve(&mut values, entries.len(), "Origin metadata JSON")?; + for entry in entries { + values.push(Value::Object(Map::from_iter([ + ( + "key".to_owned(), + Value::String(copy_text(&entry.key, "metadata key")?), + ), + ( + "value".to_owned(), + Value::String(copy_text(&entry.value, "metadata value")?), + ), + ]))); + } + Ok(Value::Array(values)) +} + +fn notes_json(notes: &[OriginNote]) -> Result { + let mut values = Vec::new(); + try_reserve(&mut values, notes.len(), "Origin notes JSON")?; + for note in notes { + values.push(Value::Object(Map::from_iter([ + ( + "name".to_owned(), + Value::String(copy_text(¬e.name, "note name")?), + ), + ( + "content".to_owned(), + Value::String(copy_text(¬e.content, "note content")?), + ), + ]))); + } + Ok(Value::Array(values)) +} + +fn diagnostics_json(diagnostics: &[OriginDiagnostic]) -> Result { + let mut values = Vec::new(); + try_reserve(&mut values, diagnostics.len(), "Origin diagnostics JSON")?; + for diagnostic in diagnostics { + let location = diagnostic.location.as_ref().map(|location| { + Value::Object(Map::from_iter([ + ( + "workbook".to_owned(), + option_text(location.workbook.as_deref()), + ), + ( + "worksheet".to_owned(), + option_text(location.worksheet.as_deref()), + ), + ("column".to_owned(), option_text(location.column.as_deref())), + ("byte_offset".to_owned(), usize_value(location.byte_offset)), + ])) + }); + values.push(Value::Object(Map::from_iter([ + ( + "code".to_owned(), + Value::String(diagnostic_code(diagnostic.code).to_owned()), + ), + ( + "severity".to_owned(), + Value::String(diagnostic_severity(diagnostic.severity).to_owned()), + ), + ("location".to_owned(), location.unwrap_or(Value::Null)), + ( + "message".to_owned(), + Value::String(copy_text(&diagnostic.message, "diagnostic message")?), + ), + ]))); + } + Ok(Value::Array(values)) +} + +fn unsupported_json( + unsupported: &[OriginUnsupportedObjectSummary], +) -> Result { + let mut values = Vec::new(); + try_reserve( + &mut values, + unsupported.len(), + "unsupported Origin object JSON", + )?; + for summary in unsupported { + values.push(Value::Object(Map::from_iter([ + ( + "kind".to_owned(), + Value::String(copy_text(&summary.kind, "object kind")?), + ), + ("count".to_owned(), Value::from(summary.count)), + ]))); + } + Ok(Value::Array(values)) +} + +fn columns_json( + columns: &[OriginColumn], + imported_names: &[String], +) -> Result { + let mut values = Vec::new(); + try_reserve(&mut values, columns.len(), "Origin column metadata JSON")?; + for (index, (column, imported_name)) in columns.iter().zip(imported_names).enumerate() { + let mut value = Map::new(); + value.insert("index".to_owned(), Value::from(index)); + value.insert( + "source_name".to_owned(), + Value::String(copy_text(&column.name, "column source name")?), + ); + value.insert( + "imported_name".to_owned(), + Value::String(copy_text(imported_name, "column imported name")?), + ); + value.insert( + "long_name".to_owned(), + option_text(column.long_name.as_deref()), + ); + value.insert("role".to_owned(), option_text(column.role.as_deref())); + value.insert("units".to_owned(), option_text(column.units.as_deref())); + value.insert( + "comments".to_owned(), + option_text(column.comments.as_deref()), + ); + values.push(Value::Object(value)); + } + Ok(Value::Array(values)) +} + +fn usage_json(usage: &OriginResourceUsage) -> Value { + Value::Object(Map::from_iter([ + ("input_bytes".to_owned(), Value::from(usage.input_bytes)), + ("parser_bytes".to_owned(), Value::from(usage.parser_bytes)), + ( + "decoded_text_bytes".to_owned(), + Value::from(usage.decoded_text_bytes), + ), + ( + "total_owned_bytes".to_owned(), + Value::from(usage.total_owned_bytes), + ), + ("workbooks".to_owned(), Value::from(usage.workbooks)), + ("worksheets".to_owned(), Value::from(usage.worksheets)), + ("columns".to_owned(), Value::from(usage.columns)), + ("cells".to_owned(), Value::from(usage.cells)), + ( + "metadata_records".to_owned(), + Value::from(usage.metadata_records), + ), + ])) +} + +fn candidate_name(workbook: &str, worksheet: &str) -> Result { + let capacity = checked_add( + checked_add(workbook.len(), worksheet.len(), "candidate name")?, + 3, + "candidate name", + )?; + let mut name = String::new(); + name.try_reserve_exact(capacity) + .map_err(|_| OriginImportError::AllocationFailed { + resource: "Origin candidate name", + requested: capacity, + })?; + name.push_str(workbook); + name.push_str(" / "); + name.push_str(worksheet); + Ok(name) +} + +fn insert_text( + metadata: &mut BTreeMap, + key: &str, + value: &str, +) -> Result<(), OriginImportError> { + metadata.insert( + key.to_owned(), + Value::String(copy_text(value, "Origin metadata text")?), + ); + Ok(()) +} + +fn option_text(value: Option<&str>) -> Value { + value.map_or(Value::Null, |value| Value::String(value.to_owned())) +} + +fn usize_value(value: Option) -> Value { + value.map_or(Value::Null, Value::from) +} + +fn diagnostic_code(code: OriginDiagnosticCode) -> &'static str { + match code { + OriginDiagnosticCode::UnsupportedObjectSkipped => "unsupported_object_skipped", + OriginDiagnosticCode::UnsupportedColumnSkipped => "unsupported_column_skipped", + OriginDiagnosticCode::MetadataSkipped => "metadata_skipped", + OriginDiagnosticCode::DecodingWarning => "decoding_warning", + } +} + +fn diagnostic_severity(severity: OriginDiagnosticSeverity) -> &'static str { + match severity { + OriginDiagnosticSeverity::Info => "info", + OriginDiagnosticSeverity::Warning => "warning", + } +} + +fn cell_matches(column_type: OriginColumnType, cell: &OriginCell) -> bool { + matches!( + (column_type, cell), + (_, OriginCell::Null) + | (OriginColumnType::Float, OriginCell::Float(_)) + | (OriginColumnType::Integer, OriginCell::Integer(_)) + | (OriginColumnType::Text, OriginCell::Text(_)) + | ( + OriginColumnType::Mixed, + OriginCell::Float(_) | OriginCell::Integer(_) | OriginCell::Text(_), + ) + ) +} + +fn cell_kind(cell: &OriginCell) -> &'static str { + match cell { + OriginCell::Null => "null", + OriginCell::Float(_) => "a floating-point value", + OriginCell::Integer(_) => "an integer", + OriginCell::Text(_) => "text", + } +} + +fn copy_text(text: &str, resource: &'static str) -> Result { + let mut copy = String::new(); + copy.try_reserve_exact(text.len()) + .map_err(|_| OriginImportError::AllocationFailed { + resource, + requested: text.len(), + })?; + copy.push_str(text); + Ok(copy) +} + +fn try_reserve( + values: &mut Vec, + additional: usize, + resource: &'static str, +) -> Result<(), OriginImportError> { + let requested = checked_mul(additional, size_of::(), resource)?; + values + .try_reserve_exact(additional) + .map_err(|_| OriginImportError::AllocationFailed { + resource, + requested, + }) +} + +fn enforce(resource: &'static str, actual: usize, limit: usize) -> Result<(), OriginImportError> { + if actual > limit { + return Err(OriginImportError::LimitExceeded { + resource, + limit, + actual, + }); + } + Ok(()) +} + +fn checked_add( + left: usize, + right: usize, + resource: &'static str, +) -> Result { + left.checked_add(right) + .ok_or(OriginImportError::ArithmeticOverflow { resource }) +} + +fn checked_mul( + left: usize, + right: usize, + resource: &'static str, +) -> Result { + left.checked_mul(right) + .ok_or(OriginImportError::ArithmeticOverflow { resource }) +} diff --git a/crates/core/src/origin/names.rs b/crates/core/src/origin/names.rs new file mode 100644 index 0000000..aa8ed9c --- /dev/null +++ b/crates/core/src/origin/names.rs @@ -0,0 +1,100 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use plotx_io::origin::{OriginColumn, OriginLimits}; + +use super::{OriginImportError, checked_add, copy_text, enforce, try_reserve}; + +pub(super) fn normalize( + columns: &[OriginColumn], + limits: &OriginLimits, +) -> Result, OriginImportError> { + let mut reserved = BTreeSet::new(); + for column in columns { + if !column.name.trim().is_empty() { + reserved.insert(copy_text(&column.name, "Origin column name")?); + } + } + + let mut names = Vec::new(); + try_reserve(&mut names, columns.len(), "Origin column names")?; + let mut used = BTreeSet::new(); + let mut next_suffix = BTreeMap::new(); + for (index, column) in columns.iter().enumerate() { + let source_name = !column.name.trim().is_empty(); + let base = if source_name { + copy_text(&column.name, "Origin column name")? + } else { + generated_column_name(index, limits)? + }; + enforce("string bytes", base.len(), limits.max_string_bytes)?; + let can_use_base = !used.contains(&base) && (source_name || !reserved.contains(&base)); + let candidate = if can_use_base { + base + } else { + unique_suffixed_name(&base, &reserved, &used, &mut next_suffix, limits)? + }; + if !used.insert(copy_text(&candidate, "Origin column name")?) { + return Err(OriginImportError::InvalidModel { + detail: "column name normalization produced a duplicate".to_owned(), + }); + } + names.push(candidate); + } + Ok(names) +} + +fn unique_suffixed_name( + base: &str, + reserved: &BTreeSet, + used: &BTreeSet, + next_suffix: &mut BTreeMap, + limits: &OriginLimits, +) -> Result { + let mut suffix = next_suffix.get(base).copied().unwrap_or(2); + let candidate = loop { + let candidate = suffixed_name(base, suffix, limits)?; + suffix = checked_add(suffix, 1, "column name suffix")?; + if !reserved.contains(&candidate) && !used.contains(&candidate) { + break candidate; + } + }; + next_suffix.insert(copy_text(base, "Origin column name")?, suffix); + Ok(candidate) +} + +fn generated_column_name(index: usize, limits: &OriginLimits) -> Result { + let number = checked_add(index, 1, "generated column number")?.to_string(); + joined_name("Column ", "", &number, limits) +} + +fn suffixed_name( + base: &str, + suffix: usize, + limits: &OriginLimits, +) -> Result { + joined_name(base, " (", &format!("{suffix})"), limits) +} + +fn joined_name( + left: &str, + separator: &str, + right: &str, + limits: &OriginLimits, +) -> Result { + let length = checked_add( + checked_add(left.len(), separator.len(), "column name")?, + right.len(), + "column name", + )?; + enforce("string bytes", length, limits.max_string_bytes)?; + let mut name = String::new(); + name.try_reserve_exact(length) + .map_err(|_| OriginImportError::AllocationFailed { + resource: "Origin column name", + requested: length, + })?; + name.push_str(left); + name.push_str(separator); + name.push_str(right); + Ok(name) +} diff --git a/crates/core/src/origin/preflight.rs b/crates/core/src/origin/preflight.rs new file mode 100644 index 0000000..4578c65 --- /dev/null +++ b/crates/core/src/origin/preflight.rs @@ -0,0 +1,682 @@ +use std::mem::size_of; + +use plotx_data::{ + ChunkDescriptor, ColumnChunk, ColumnManifest, ColumnSchema, RowId, TableSnapshot, +}; +use plotx_io::origin::{ + OriginByteOrder, OriginCell, OriginColumn, OriginColumnType, OriginDiagnostic, OriginFormat, + OriginHeaderVersion, OriginLimits, OriginMetadataEntry, OriginProfile, OriginProject, + OriginResourceUsage, OriginSupport, OriginWorksheet, +}; + +use super::{ + CHUNK_ROWS, ImportedOriginWorksheet, OriginImportError, PreparedOriginWorksheet, cell_kind, + cell_matches, checked_add, checked_mul, copy_text, enforce, names, try_reserve, +}; + +const ARROW_BLOCK_OVERHEAD: usize = 4_096; +const BTREE_ENTRY_OVERHEAD: usize = 64; +const IPC_OFFSET_BYTES: usize = 4; +const FINGERPRINT_LENGTH_BYTES: usize = 8; +const UUID_TEXT_BYTES: usize = 36; +const JSON_ENTRY_OVERHEAD: usize = 256; +const MIXED_FLOAT_TEXT_MAX: usize = 32; +const MIXED_INTEGER_TEXT_MAX: usize = 20; + +mod model; + +pub(super) struct Preflight { + pub(super) worksheets: Vec, + pub(super) total_owned_bytes: usize, +} + +pub(super) struct WorksheetPreflight { + pub(super) imported_names: Vec, +} + +pub(super) fn validate( + project: &OriginProject, + limits: &OriginLimits, +) -> Result { + validate_probe(project)?; + validate_reported_usage(&project.resource_usage, limits)?; + enforce("workbooks", project.workbooks.len(), limits.max_workbooks)?; + let retained_model = checked_add( + project.resource_usage.input_bytes, + model::owned_lower_bound(project)?, + "retained Origin model", + )?; + let mut estimated_total = project.resource_usage.total_owned_bytes.max(retained_model); + enforce( + "total owned bytes", + estimated_total, + limits.max_total_owned_bytes, + )?; + + let mut text_bytes = 0_usize; + charge_text(&mut text_bytes, &project.probe.raw_version, limits)?; + let mut metadata_records = 0_usize; + validate_entries( + &project.parameters, + &mut text_bytes, + &mut metadata_records, + limits, + )?; + for note in &project.notes { + add_record(&mut metadata_records, limits)?; + charge_text(&mut text_bytes, ¬e.name, limits)?; + charge_text(&mut text_bytes, ¬e.content, limits)?; + } + validate_diagnostics( + &project.diagnostics, + &mut text_bytes, + &mut metadata_records, + limits, + )?; + for summary in &project.unsupported_objects { + add_record(&mut metadata_records, limits)?; + charge_text(&mut text_bytes, &summary.kind, limits)?; + } + + let mut total_columns = 0_usize; + let mut total_cells = 0_usize; + let mut total_worksheets = 0_usize; + let mut retained_metadata_records = checked_add( + project.parameters.len(), + project.notes.len(), + "retained metadata records", + )?; + let mut worksheets = Vec::new(); + for workbook in &project.workbooks { + charge_text(&mut text_bytes, &workbook.name, limits)?; + enforce( + "worksheets per workbook", + workbook.worksheets.len(), + limits.max_worksheets_per_workbook, + )?; + for worksheet in &workbook.worksheets { + total_worksheets = checked_add(total_worksheets, 1, "worksheets")?; + retained_metadata_records = checked_add( + retained_metadata_records, + worksheet.metadata.len(), + "retained metadata records", + )?; + charge_text(&mut text_bytes, &worksheet.name, limits)?; + validate_entries( + &worksheet.metadata, + &mut text_bytes, + &mut metadata_records, + limits, + )?; + total_columns = checked_add(total_columns, worksheet.columns.len(), "columns")?; + enforce("columns", total_columns, limits.max_columns)?; + enforce( + "rows per column", + worksheet.row_count, + limits.max_rows_per_column, + )?; + for column in &worksheet.columns { + validate_column( + column, + worksheet.row_count, + &mut text_bytes, + &mut total_cells, + limits, + )?; + } + if worksheet.row_count > 0 && !worksheet.columns.is_empty() { + let imported_names = names::normalize(&worksheet.columns, limits)?; + estimate_worksheet( + &mut estimated_total, + project, + &workbook.name, + worksheet, + &imported_names, + limits, + )?; + try_reserve(&mut worksheets, 1, "Origin worksheet preflight")?; + worksheets.push(WorksheetPreflight { imported_names }); + } + } + } + enforce( + "decoded text bytes", + text_bytes, + limits.max_decoded_text_bytes, + )?; + enforce( + "metadata records", + metadata_records, + limits.max_metadata_records, + )?; + enforce("cells", total_cells, limits.max_cells)?; + validate_retained_counts( + &project.resource_usage, + project.workbooks.len(), + total_worksheets, + total_columns, + total_cells, + retained_metadata_records, + )?; + enforce( + "total owned bytes", + estimated_total, + limits.max_total_owned_bytes, + )?; + Ok(Preflight { + worksheets, + total_owned_bytes: estimated_total, + }) +} + +fn validate_probe(project: &OriginProject) -> Result<(), OriginImportError> { + const ORIGIN_7_VERSION: OriginHeaderVersion = OriginHeaderVersion { + major: 4, + minor: 2673, + build: 552, + }; + const ORIGIN_9_VERSION: OriginHeaderVersion = OriginHeaderVersion { + major: 4, + minor: 3268, + build: 195, + }; + let exact_profile = match project.probe.profile { + Some(OriginProfile::Origin7V552) => { + project.probe.raw_version == "4.2673 552" && project.probe.version == ORIGIN_7_VERSION + } + Some(OriginProfile::Origin9V951) => { + project.probe.raw_version == "4.3268 195 W64" + && project.probe.version == ORIGIN_9_VERSION + } + None => false, + }; + if project.probe.format != OriginFormat::Opj + || project.probe.support != OriginSupport::Supported + || project.probe.byte_order != OriginByteOrder::LittleEndian + || !exact_profile + { + return Err(OriginImportError::InvalidModel { + detail: + "only the exact verified little-endian Origin7V552 or Origin9V951 OPJ profile can be converted" + .to_owned(), + }); + } + Ok(()) +} + +fn validate_reported_usage( + usage: &OriginResourceUsage, + limits: &OriginLimits, +) -> Result<(), OriginImportError> { + enforce("input bytes", usage.input_bytes, limits.max_input_bytes)?; + enforce("parser bytes", usage.parser_bytes, limits.max_parser_bytes)?; + enforce( + "decoded text bytes", + usage.decoded_text_bytes, + limits.max_decoded_text_bytes, + )?; + enforce( + "total owned bytes", + usage.total_owned_bytes, + limits.max_total_owned_bytes, + )?; + enforce("workbooks", usage.workbooks, limits.max_workbooks)?; + enforce("columns", usage.columns, limits.max_columns)?; + enforce("cells", usage.cells, limits.max_cells)?; + enforce( + "metadata records", + usage.metadata_records, + limits.max_metadata_records, + )?; + let parser_minimum = checked_add(usage.input_bytes, usage.parser_bytes, "parser ownership")?; + if usage.total_owned_bytes < parser_minimum { + return Err(OriginImportError::InvalidModel { + detail: "resource usage total is smaller than input plus parser ownership".to_owned(), + }); + } + if usage.decoded_text_bytes > usage.parser_bytes { + return Err(OriginImportError::InvalidModel { + detail: "decoded text accounting exceeds parser ownership".to_owned(), + }); + } + Ok(()) +} + +fn validate_retained_counts( + usage: &OriginResourceUsage, + workbooks: usize, + worksheets: usize, + columns: usize, + cells: usize, + metadata_records: usize, +) -> Result<(), OriginImportError> { + for (resource, reported, retained, exact) in [ + ("workbooks", usage.workbooks, workbooks, true), + ("worksheets", usage.worksheets, worksheets, true), + ("columns", usage.columns, columns, false), + ("cells", usage.cells, cells, false), + ( + "metadata records", + usage.metadata_records, + metadata_records, + false, + ), + ] { + if (exact && reported != retained) || (!exact && reported < retained) { + let relation = if exact { "equal" } else { "cover" }; + return Err(OriginImportError::InvalidModel { + detail: format!( + "reported {resource} count {reported} does not {relation} the retained count {retained}" + ), + }); + } + } + Ok(()) +} + +fn validate_entries( + entries: &[OriginMetadataEntry], + text_bytes: &mut usize, + records: &mut usize, + limits: &OriginLimits, +) -> Result<(), OriginImportError> { + for entry in entries { + add_record(records, limits)?; + charge_text(text_bytes, &entry.key, limits)?; + charge_text(text_bytes, &entry.value, limits)?; + } + Ok(()) +} + +fn validate_diagnostics( + diagnostics: &[OriginDiagnostic], + text_bytes: &mut usize, + records: &mut usize, + limits: &OriginLimits, +) -> Result<(), OriginImportError> { + for diagnostic in diagnostics { + add_record(records, limits)?; + charge_text(text_bytes, &diagnostic.message, limits)?; + if let Some(location) = &diagnostic.location { + for value in [ + location.workbook.as_deref(), + location.worksheet.as_deref(), + location.column.as_deref(), + ] + .into_iter() + .flatten() + { + charge_text(text_bytes, value, limits)?; + } + } + } + Ok(()) +} + +fn validate_column( + column: &OriginColumn, + row_count: usize, + text_bytes: &mut usize, + total_cells: &mut usize, + limits: &OriginLimits, +) -> Result<(), OriginImportError> { + for text in [ + Some(column.name.as_str()), + column.long_name.as_deref(), + column.role.as_deref(), + column.units.as_deref(), + column.comments.as_deref(), + ] + .into_iter() + .flatten() + { + charge_text(text_bytes, text, limits)?; + } + enforce( + "rows per column", + column.cells.len(), + limits.max_rows_per_column, + )?; + if column.cells.len() > row_count { + return Err(OriginImportError::InvalidModel { + detail: format!( + "column {:?} has {} cells but worksheet row_count is {}", + column.name, + column.cells.len(), + row_count + ), + }); + } + *total_cells = checked_add(*total_cells, column.cells.len(), "cells")?; + enforce("cells", *total_cells, limits.max_cells)?; + for (row, cell) in column.cells.iter().enumerate() { + if let OriginCell::Text(text) = cell { + charge_text(text_bytes, text, limits)?; + } + if !cell_matches(column.column_type, cell) { + return Err(OriginImportError::InvalidCellType { + column: copy_text(&column.name, "Origin column name")?, + row, + expected: column.column_type, + actual: cell_kind(cell), + }); + } + } + Ok(()) +} + +fn estimate_worksheet( + total: &mut usize, + project: &OriginProject, + workbook_name: &str, + worksheet: &OriginWorksheet, + imported_names: &[String], + limits: &OriginLimits, +) -> Result<(), OriginImportError> { + let rows = worksheet.row_count; + let columns = worksheet.columns.len(); + let batches = checked_add((rows - 1) / CHUNK_ROWS, 1, "snapshot batches")?; + estimate_add(total, size_of::(), limits)?; + estimate_add(total, size_of::(), limits)?; + estimate_add(total, size_of::(), limits)?; + estimate_add(total, size_of::(), limits)?; + estimate_mul(total, columns, size_of::(), limits)?; + estimate_mul(total, columns, size_of::(), limits)?; + estimate_mul(total, batches, size_of::(), limits)?; + estimate_mul( + total, + checked_mul(columns, batches, "column descriptors")?, + size_of::(), + limits, + )?; + estimate_mul(total, rows, size_of::(), limits)?; + // SnapshotBuilder validates every batch in a BTreeSet before formatting + // UUID row ids as strings and sending them through the UTF-8 codec path. + estimate_mul( + total, + rows, + checked_add(size_of::(), BTREE_ENTRY_OVERHEAD, "row identity set")?, + limits, + )?; + let row_identity_text = checked_mul(rows, UUID_TEXT_BYTES, "row identity text")?; + estimate_utf8_conversion(total, rows, row_identity_text, row_identity_text, limits)?; + estimate_mul( + total, + checked_mul(columns, batches, "column chunks")?, + size_of::(), + limits, + )?; + estimate_mul(total, columns, size_of::(), limits)?; + let imported_name_bytes = imported_names.iter().try_fold(0_usize, |total, name| { + checked_add(total, name.len(), "imported column names") + })?; + // Normalization retains the final names and temporarily owns reserved and + // used-name set copies while protecting genuine source names. + estimate_mul(total, imported_name_bytes, 3, limits)?; + estimate_mul(total, columns, BTREE_ENTRY_OVERHEAD * 2, limits)?; + + for column in &worksheet.columns { + match column.column_type { + OriginColumnType::Float | OriginColumnType::Integer => { + estimate_numeric_conversion(total, rows, limits)?; + } + OriginColumnType::Text | OriginColumnType::Mixed => { + let text = estimated_column_text(column)?; + estimate_utf8_conversion( + total, + rows, + text.encoded_bytes, + text.new_target_bytes, + limits, + )?; + } + } + estimate_add( + total, + checked_add( + column.name.len(), + checked_mul(5, JSON_ENTRY_OVERHEAD, "column metadata")?, + "column metadata", + )?, + limits, + )?; + } + estimate_mul( + total, + source_metadata_text_bytes(project, workbook_name, worksheet)?, + 4, + limits, + )?; + let records = checked_add( + checked_add( + project.parameters.len(), + project.notes.len(), + "metadata estimate", + )?, + checked_add( + project.diagnostics.len(), + project.unsupported_objects.len(), + "metadata estimate", + )?, + "metadata estimate", + )?; + let records = checked_add(records, worksheet.metadata.len(), "metadata estimate")?; + let records = checked_add(records, worksheet.columns.len(), "metadata estimate")?; + estimate_mul( + total, + checked_add(records, 12, "metadata estimate")?, + JSON_ENTRY_OVERHEAD, + limits, + )?; + estimate_mul( + total, + checked_mul( + batches, + checked_add(columns, 1, "encoded blocks")?, + "encoded blocks", + )?, + ARROW_BLOCK_OVERHEAD, + limits, + ) +} + +fn estimate_numeric_conversion( + total: &mut usize, + rows: usize, + limits: &OriginLimits, +) -> Result<(), OriginImportError> { + let validity = bitmap_bytes(rows); + // Conversion target plus the byte-per-row validity input. + estimate_mul(total, rows, size_of::(), limits)?; + estimate_add(total, rows, limits)?; + estimate_add(total, validity, limits)?; + // The Arrow codec first materializes Vec>, then Arrow value and + // validity buffers. IPC retains another value representation in the block + // store, and logical_fingerprint builds a separate canonical byte vector. + estimate_mul(total, rows, size_of::>(), limits)?; + estimate_numeric_buffers(total, rows, validity, limits)?; + estimate_numeric_buffers(total, rows, validity, limits)?; + estimate_numeric_buffers(total, rows, validity, limits) +} + +fn estimate_numeric_buffers( + total: &mut usize, + rows: usize, + validity: usize, + limits: &OriginLimits, +) -> Result<(), OriginImportError> { + estimate_mul(total, rows, size_of::(), limits)?; + estimate_add(total, validity, limits) +} + +fn estimate_utf8_conversion( + total: &mut usize, + rows: usize, + text_bytes: usize, + new_target_text_bytes: usize, + limits: &OriginLimits, +) -> Result<(), OriginImportError> { + let validity = bitmap_bytes(rows); + // Conversion target plus the byte-per-row validity input and the retained + // ColumnChunk bitmap. + estimate_mul(total, rows, size_of::(), limits)?; + estimate_add(total, new_target_text_bytes, limits)?; + estimate_add(total, rows, limits)?; + estimate_add(total, validity, limits)?; + // StringArray::from first collects borrowed values into Vec>. + estimate_mul(total, rows, size_of::>(), limits)?; + // Arrow, retained IPC, and the canonical fingerprint each own a separate + // row-scaled representation. The fingerprint uses u64 lengths rather than + // Arrow's i32 offsets. + estimate_utf8_buffers(total, rows, text_bytes, validity, IPC_OFFSET_BYTES, limits)?; + estimate_utf8_buffers(total, rows, text_bytes, validity, IPC_OFFSET_BYTES, limits)?; + estimate_utf8_buffers( + total, + rows, + text_bytes, + validity, + FINGERPRINT_LENGTH_BYTES, + limits, + ) +} + +fn estimate_utf8_buffers( + total: &mut usize, + rows: usize, + text_bytes: usize, + validity: usize, + offset_width: usize, + limits: &OriginLimits, +) -> Result<(), OriginImportError> { + estimate_mul( + total, + checked_add(rows, 1, "UTF-8 offsets")?, + offset_width, + limits, + )?; + estimate_add(total, text_bytes, limits)?; + estimate_add(total, validity, limits) +} + +struct EstimatedColumnText { + encoded_bytes: usize, + new_target_bytes: usize, +} + +fn estimated_column_text(column: &OriginColumn) -> Result { + let mut encoded_bytes = 0_usize; + let mut new_target_bytes = 0_usize; + for cell in &column.cells { + let (encoded, newly_allocated) = match cell { + OriginCell::Text(value) => (value.len(), 0), + OriginCell::Float(_) => (MIXED_FLOAT_TEXT_MAX, MIXED_FLOAT_TEXT_MAX), + OriginCell::Integer(_) => (MIXED_INTEGER_TEXT_MAX, MIXED_INTEGER_TEXT_MAX), + OriginCell::Null => (0, 0), + }; + encoded_bytes = checked_add(encoded_bytes, encoded, "UTF-8 cell data")?; + new_target_bytes = checked_add( + new_target_bytes, + newly_allocated, + "converted UTF-8 cell data", + )?; + } + Ok(EstimatedColumnText { + encoded_bytes, + new_target_bytes, + }) +} + +fn source_metadata_text_bytes( + project: &OriginProject, + workbook_name: &str, + worksheet: &OriginWorksheet, +) -> Result { + let mut bytes = checked_add( + project.probe.raw_version.len(), + workbook_name.len(), + "metadata", + )?; + bytes = checked_add(bytes, worksheet.name.len(), "metadata")?; + for entry in project.parameters.iter().chain(&worksheet.metadata) { + bytes = checked_add(bytes, entry.key.len(), "metadata")?; + bytes = checked_add(bytes, entry.value.len(), "metadata")?; + } + for note in &project.notes { + bytes = checked_add(bytes, note.name.len(), "metadata")?; + bytes = checked_add(bytes, note.content.len(), "metadata")?; + } + for diagnostic in &project.diagnostics { + bytes = checked_add(bytes, diagnostic.message.len(), "metadata")?; + if let Some(location) = &diagnostic.location { + for text in [ + location.workbook.as_deref(), + location.worksheet.as_deref(), + location.column.as_deref(), + ] + .into_iter() + .flatten() + { + bytes = checked_add(bytes, text.len(), "metadata")?; + } + } + } + for summary in &project.unsupported_objects { + bytes = checked_add(bytes, summary.kind.len(), "metadata")?; + } + for column in &worksheet.columns { + for text in [ + Some(column.name.as_str()), + column.long_name.as_deref(), + column.role.as_deref(), + column.units.as_deref(), + column.comments.as_deref(), + ] + .into_iter() + .flatten() + { + bytes = checked_add(bytes, text.len(), "metadata")?; + } + } + Ok(bytes) +} + +fn add_record(records: &mut usize, limits: &OriginLimits) -> Result<(), OriginImportError> { + *records = checked_add(*records, 1, "metadata records")?; + enforce("metadata records", *records, limits.max_metadata_records) +} + +fn charge_text( + total: &mut usize, + text: &str, + limits: &OriginLimits, +) -> Result<(), OriginImportError> { + enforce("string bytes", text.len(), limits.max_string_bytes)?; + *total = checked_add(*total, text.len(), "decoded text bytes")?; + enforce("decoded text bytes", *total, limits.max_decoded_text_bytes) +} + +fn estimate_add( + total: &mut usize, + bytes: usize, + limits: &OriginLimits, +) -> Result<(), OriginImportError> { + *total = checked_add(*total, bytes, "total owned bytes")?; + enforce("total owned bytes", *total, limits.max_total_owned_bytes) +} + +fn estimate_mul( + total: &mut usize, + count: usize, + width: usize, + limits: &OriginLimits, +) -> Result<(), OriginImportError> { + estimate_add( + total, + checked_mul(count, width, "total owned bytes")?, + limits, + ) +} + +fn bitmap_bytes(rows: usize) -> usize { + rows / 8 + usize::from(!rows.is_multiple_of(8)) +} diff --git a/crates/core/src/origin/preflight/model.rs b/crates/core/src/origin/preflight/model.rs new file mode 100644 index 0000000..b02bf55 --- /dev/null +++ b/crates/core/src/origin/preflight/model.rs @@ -0,0 +1,109 @@ +use std::mem::size_of; + +use plotx_io::origin::{ + OriginCell, OriginColumn, OriginDiagnostic, OriginMetadataEntry, OriginNote, OriginProject, + OriginUnsupportedObjectSummary, OriginWorkbook, OriginWorksheet, +}; + +use super::super::{OriginImportError, checked_add, checked_mul}; + +pub(super) fn owned_lower_bound(project: &OriginProject) -> Result { + let mut bytes = size_of::(); + add_text_storage(&mut bytes, &project.probe.raw_version)?; + add_vec_storage::(&mut bytes, project.parameters.len())?; + for entry in &project.parameters { + add_entry_storage(&mut bytes, entry)?; + } + add_vec_storage::(&mut bytes, project.notes.len())?; + for note in &project.notes { + add_text_storage(&mut bytes, ¬e.name)?; + add_text_storage(&mut bytes, ¬e.content)?; + } + add_vec_storage::(&mut bytes, project.workbooks.len())?; + for workbook in &project.workbooks { + add_text_storage(&mut bytes, &workbook.name)?; + add_vec_storage::(&mut bytes, workbook.worksheets.len())?; + for worksheet in &workbook.worksheets { + add_worksheet_storage(&mut bytes, worksheet)?; + } + } + add_vec_storage::(&mut bytes, project.diagnostics.len())?; + for diagnostic in &project.diagnostics { + add_text_storage(&mut bytes, &diagnostic.message)?; + if let Some(location) = &diagnostic.location { + for text in [ + location.workbook.as_ref(), + location.worksheet.as_ref(), + location.column.as_ref(), + ] + .into_iter() + .flatten() + { + add_text_storage(&mut bytes, text)?; + } + } + } + add_vec_storage::( + &mut bytes, + project.unsupported_objects.len(), + )?; + for summary in &project.unsupported_objects { + add_text_storage(&mut bytes, &summary.kind)?; + } + Ok(bytes) +} + +fn add_worksheet_storage( + bytes: &mut usize, + worksheet: &OriginWorksheet, +) -> Result<(), OriginImportError> { + add_text_storage(bytes, &worksheet.name)?; + add_vec_storage::(bytes, worksheet.metadata.len())?; + for entry in &worksheet.metadata { + add_entry_storage(bytes, entry)?; + } + add_vec_storage::(bytes, worksheet.columns.len())?; + for column in &worksheet.columns { + for text in [ + Some(&column.name), + column.long_name.as_ref(), + column.role.as_ref(), + column.units.as_ref(), + column.comments.as_ref(), + ] + .into_iter() + .flatten() + { + add_text_storage(bytes, text)?; + } + add_vec_storage::(bytes, column.cells.len())?; + for cell in &column.cells { + if let OriginCell::Text(text) = cell { + add_text_storage(bytes, text)?; + } + } + } + Ok(()) +} + +fn add_entry_storage( + bytes: &mut usize, + entry: &OriginMetadataEntry, +) -> Result<(), OriginImportError> { + add_text_storage(bytes, &entry.key)?; + add_text_storage(bytes, &entry.value) +} + +fn add_text_storage(bytes: &mut usize, text: &str) -> Result<(), OriginImportError> { + *bytes = checked_add(*bytes, text.len(), "retained Origin model")?; + Ok(()) +} + +fn add_vec_storage(bytes: &mut usize, len: usize) -> Result<(), OriginImportError> { + *bytes = checked_add( + *bytes, + checked_mul(len, size_of::(), "retained Origin model")?, + "retained Origin model", + )?; + Ok(()) +} diff --git a/crates/core/src/origin_hardening_tests.rs b/crates/core/src/origin_hardening_tests.rs new file mode 100644 index 0000000..c86d141 --- /dev/null +++ b/crates/core/src/origin_hardening_tests.rs @@ -0,0 +1,286 @@ +use plotx_data::{CodecRegistry, MemoryBlockStore, ScalarValue, SnapshotReader}; +use plotx_io::origin::{ + OriginByteOrder, OriginCell, OriginColumn, OriginColumnType, OriginFormat, OriginHeaderVersion, + OriginLimits, OriginProbe, OriginProfile, OriginProject, OriginResourceUsage, OriginSupport, + OriginWorkbook, OriginWorksheet, +}; + +use crate::origin::{OriginImportError, import_origin_project}; + +const ORIGINAL_NAME_KEY: &str = "space.nmrtist.plotx.import.origin.original_name"; + +fn project(worksheets: Vec) -> OriginProject { + let mut project = OriginProject { + probe: OriginProbe { + format: OriginFormat::Opj, + raw_version: "4.2673 552".to_owned(), + version: OriginHeaderVersion { + major: 4, + minor: 2673, + build: 552, + }, + byte_order: OriginByteOrder::LittleEndian, + profile: Some(OriginProfile::Origin7V552), + support: OriginSupport::Supported, + }, + parameters: Vec::new(), + notes: Vec::new(), + workbooks: vec![OriginWorkbook { + name: "Book".to_owned(), + worksheets, + }], + diagnostics: Vec::new(), + unsupported_objects: Vec::new(), + resource_usage: OriginResourceUsage::default(), + }; + sync_resource_counts(&mut project); + project +} + +fn sync_resource_counts(project: &mut OriginProject) { + project.resource_usage.workbooks = project.workbooks.len(); + project.resource_usage.worksheets = project + .workbooks + .iter() + .map(|workbook| workbook.worksheets.len()) + .sum(); + project.resource_usage.columns = project + .workbooks + .iter() + .flat_map(|workbook| &workbook.worksheets) + .map(|worksheet| worksheet.columns.len()) + .sum(); + project.resource_usage.cells = project + .workbooks + .iter() + .flat_map(|workbook| &workbook.worksheets) + .flat_map(|worksheet| &worksheet.columns) + .map(|column| column.cells.len()) + .sum(); +} + +fn worksheet(name: &str, columns: Vec) -> OriginWorksheet { + OriginWorksheet { + name: name.to_owned(), + columns, + row_count: 1, + metadata: Vec::new(), + } +} + +fn columns(names: &[&str]) -> Vec { + names + .iter() + .map(|name| OriginColumn { + name: (*name).to_owned(), + long_name: None, + role: None, + units: None, + comments: None, + column_type: OriginColumnType::Integer, + cells: vec![OriginCell::Integer(1)], + }) + .collect() +} + +fn imported_names(names: &[&str]) -> Vec { + let imported = import_origin_project( + project(vec![worksheet("Sheet", columns(names))]), + &MemoryBlockStore::default(), + &CodecRegistry::with_arrow_ipc(), + OriginLimits::default(), + ) + .expect("test project should convert"); + imported[0] + .snapshot + .schema + .columns + .iter() + .map(|column| column.name.clone()) + .collect() +} + +#[test] +fn converts_the_exact_verified_origin_9_profile() { + let mut project = project(vec![worksheet("Sheet", columns(&["value"]))]); + project.probe = OriginProbe { + format: OriginFormat::Opj, + raw_version: "4.3268 195 W64".to_owned(), + version: OriginHeaderVersion { + major: 4, + minor: 3268, + build: 195, + }, + byte_order: OriginByteOrder::LittleEndian, + profile: Some(OriginProfile::Origin9V951), + support: OriginSupport::Supported, + }; + + let imported = import_origin_project( + project, + &MemoryBlockStore::default(), + &CodecRegistry::with_arrow_ipc(), + OriginLimits::default(), + ) + .expect("the verified Origin 9.51 profile should convert"); + + assert_eq!(imported[0].snapshot.schema.columns[0].name, "value"); +} + +#[test] +fn generated_duplicate_names_do_not_steal_later_source_names() { + assert_eq!( + imported_names(&["name", "name", "name (2)"]), + ["name", "name (3)", "name (2)"] + ); +} + +#[test] +fn generated_blank_names_do_not_steal_real_position_names() { + assert_eq!( + imported_names(&["", "Column 1"]), + ["Column 1 (2)", "Column 1"] + ); + assert_eq!(imported_names(&["Column 1", ""]), ["Column 1", "Column 2"]); +} + +#[test] +fn every_changed_name_retains_its_source_name() { + let imported = import_origin_project( + project(vec![worksheet( + "Sheet", + columns(&["name", "name", "name (2)", ""]), + )]), + &MemoryBlockStore::default(), + &CodecRegistry::with_arrow_ipc(), + OriginLimits::default(), + ) + .expect("test project should convert"); + let schemas = &imported[0].snapshot.schema.columns; + + assert_eq!(schemas[1].metadata[ORIGINAL_NAME_KEY], "name"); + assert_eq!(schemas[3].metadata[ORIGINAL_NAME_KEY], ""); + assert!(!schemas[0].metadata.contains_key(ORIGINAL_NAME_KEY)); + assert!(!schemas[2].metadata.contains_key(ORIGINAL_NAME_KEY)); +} + +#[test] +fn prepares_every_candidate_before_writing_any_blocks() { + let project = project(vec![ + worksheet("First", columns(&["x"])), + worksheet("Second", columns(&["1234567890123456", "1234567890123456"])), + ]); + let store = MemoryBlockStore::default(); + let limits = OriginLimits { + max_string_bytes: 16, + ..OriginLimits::default() + }; + + let error = import_origin_project(project, &store, &CodecRegistry::with_arrow_ipc(), limits) + .expect_err("all candidate names must be prepared before block writes"); + + assert!(matches!( + error, + OriginImportError::LimitExceeded { + resource: "string bytes", + .. + } + )); + assert_eq!(store.block_count(), 0); +} + +#[test] +fn pads_short_columns_across_the_first_batch_and_into_the_second() { + const ROWS: usize = 65_537; + let mut sheet = worksheet( + "Sheet", + vec![ + OriginColumn { + name: "anchor".to_owned(), + long_name: None, + role: None, + units: None, + comments: None, + column_type: OriginColumnType::Integer, + cells: (0..ROWS) + .map(|value| OriginCell::Integer(value as i64)) + .collect(), + }, + OriginColumn { + name: "short".to_owned(), + long_name: None, + role: None, + units: None, + comments: None, + column_type: OriginColumnType::Integer, + cells: vec![OriginCell::Integer(10), OriginCell::Integer(20)], + }, + ], + ); + sheet.row_count = ROWS; + let store = MemoryBlockStore::default(); + let codecs = CodecRegistry::with_arrow_ipc(); + let imported = import_origin_project( + project(vec![sheet]), + &store, + &codecs, + OriginLimits::default(), + ) + .expect("short columns should be padded with nulls"); + let reader = SnapshotReader::new(&imported[0].snapshot, &store, &codecs).unwrap(); + let first = reader.read_batch(0, &[]).unwrap(); + let second = reader.read_batch(1, &[]).unwrap(); + + assert_eq!(first.columns[1].1.value(1), Some(ScalarValue::Int64(20))); + assert_eq!(first.columns[1].1.value(2), Some(ScalarValue::Null)); + assert_eq!(first.columns[1].1.value(65_535), Some(ScalarValue::Null)); + assert_eq!(second.columns[1].1.value(0), Some(ScalarValue::Null)); +} + +#[test] +fn pads_a_short_column_that_ends_exactly_on_a_batch_boundary() { + const ROWS: usize = 65_537; + let mut sheet = worksheet( + "Sheet", + vec![ + OriginColumn { + name: "anchor".to_owned(), + long_name: None, + role: None, + units: None, + comments: None, + column_type: OriginColumnType::Integer, + cells: (0..ROWS) + .map(|value| OriginCell::Integer(value as i64)) + .collect(), + }, + OriginColumn { + name: "short".to_owned(), + long_name: None, + role: None, + units: None, + comments: None, + column_type: OriginColumnType::Integer, + cells: (0..65_536) + .map(|value| OriginCell::Integer(value as i64)) + .collect(), + }, + ], + ); + sheet.row_count = ROWS; + let store = MemoryBlockStore::default(); + let codecs = CodecRegistry::with_arrow_ipc(); + let imported = import_origin_project( + project(vec![sheet]), + &store, + &codecs, + OriginLimits::default(), + ) + .expect("a batch-boundary short column should be padded with nulls"); + let second = SnapshotReader::new(&imported[0].snapshot, &store, &codecs) + .unwrap() + .read_batch(1, &[]) + .unwrap(); + + assert_eq!(second.columns[1].1.value(0), Some(ScalarValue::Null)); +} diff --git a/crates/core/src/origin_tests.rs b/crates/core/src/origin_tests.rs new file mode 100644 index 0000000..a992508 --- /dev/null +++ b/crates/core/src/origin_tests.rs @@ -0,0 +1,772 @@ +use std::collections::BTreeMap; + +use plotx_data::{CodecRegistry, LogicalType, MemoryBlockStore, ScalarValue, SnapshotReader}; +use plotx_io::origin::{ + OriginByteOrder, OriginCell, OriginColumn, OriginColumnType, OriginDiagnostic, + OriginDiagnosticCode, OriginDiagnosticSeverity, OriginFormat, OriginHeaderVersion, + OriginLimits, OriginMetadataEntry, OriginNote, OriginObjectLocation, OriginProbe, + OriginProfile, OriginProject, OriginResourceUsage, OriginSupport, + OriginUnsupportedObjectSummary, OriginWorkbook, OriginWorksheet, +}; + +use crate::origin::{ORIGIN_IMPORT_OPERATION, OriginImportError, import_origin_project}; + +fn probe() -> OriginProbe { + OriginProbe { + format: OriginFormat::Opj, + raw_version: "4.2673 552".to_owned(), + version: OriginHeaderVersion { + major: 4, + minor: 2673, + build: 552, + }, + byte_order: OriginByteOrder::LittleEndian, + profile: Some(OriginProfile::Origin7V552), + support: OriginSupport::Supported, + } +} + +fn column(name: &str, column_type: OriginColumnType, cells: Vec) -> OriginColumn { + OriginColumn { + name: name.to_owned(), + long_name: None, + role: None, + units: None, + comments: None, + column_type, + cells, + } +} + +fn worksheet(name: &str, row_count: usize, columns: Vec) -> OriginWorksheet { + OriginWorksheet { + name: name.to_owned(), + columns, + row_count, + metadata: Vec::new(), + } +} + +fn project(workbooks: Vec) -> OriginProject { + let mut project = OriginProject { + probe: probe(), + parameters: Vec::new(), + notes: Vec::new(), + workbooks, + diagnostics: Vec::new(), + unsupported_objects: Vec::new(), + resource_usage: OriginResourceUsage::default(), + }; + sync_resource_counts(&mut project); + project +} + +fn sync_resource_counts(project: &mut OriginProject) { + project.resource_usage.workbooks = project.workbooks.len(); + project.resource_usage.worksheets = project + .workbooks + .iter() + .map(|workbook| workbook.worksheets.len()) + .sum(); + project.resource_usage.columns = project + .workbooks + .iter() + .flat_map(|workbook| &workbook.worksheets) + .map(|worksheet| worksheet.columns.len()) + .sum(); + project.resource_usage.cells = project + .workbooks + .iter() + .flat_map(|workbook| &workbook.worksheets) + .flat_map(|worksheet| &worksheet.columns) + .map(|column| column.cells.len()) + .sum(); + project.resource_usage.metadata_records = project.parameters.len() + + project.notes.len() + + project + .workbooks + .iter() + .flat_map(|workbook| &workbook.worksheets) + .map(|worksheet| worksheet.metadata.len()) + .sum::(); +} + +fn workbook(name: &str, worksheets: Vec) -> OriginWorkbook { + OriginWorkbook { + name: name.to_owned(), + worksheets, + } +} + +fn imported( + mut project: OriginProject, +) -> ( + Vec, + MemoryBlockStore, + CodecRegistry, +) { + sync_resource_counts(&mut project); + let store = MemoryBlockStore::default(); + let codecs = CodecRegistry::with_arrow_ipc(); + let imported = import_origin_project(project, &store, &codecs, OriginLimits::default()) + .expect("test project should convert"); + (imported, store, codecs) +} + +#[test] +fn exposes_the_stable_origin_import_operation() { + assert_eq!(ORIGIN_IMPORT_OPERATION, "plotx.import.origin.v1"); +} + +#[test] +fn converts_supported_types_and_pads_short_columns_with_nulls() { + let columns = vec![ + column( + "double", + OriginColumnType::Float, + vec![OriginCell::Float(1.25), OriginCell::Null], + ), + column( + "float", + OriginColumnType::Float, + vec![OriginCell::Float(f32::from_bits(0x43ac_cccd) as f64)], + ), + column( + "integer", + OriginColumnType::Integer, + vec![OriginCell::Integer(-1000), OriginCell::Integer(34)], + ), + column( + "text", + OriginColumnType::Text, + vec![OriginCell::Text("alpha".to_owned()), OriginCell::Null], + ), + ]; + let (imported, store, codecs) = imported(project(vec![workbook( + "Book1", + vec![worksheet("Sheet1", 2, columns)], + )])); + + assert_eq!(imported.len(), 1); + assert_eq!(imported[0].name, "Book1 / Sheet1"); + assert_eq!(imported[0].snapshot.row_count, 2); + assert_eq!( + imported[0] + .snapshot + .schema + .columns + .iter() + .map(|column| column.logical_type.clone()) + .collect::>(), + vec![ + LogicalType::Float64, + LogicalType::Float64, + LogicalType::Int64, + LogicalType::Utf8, + ] + ); + assert!( + imported[0].snapshot.schema.columns[3].unit.is_none(), + "text columns must never acquire a numeric unit" + ); + + let batch = SnapshotReader::new(&imported[0].snapshot, &store, &codecs) + .unwrap() + .read_batch(0, &[]) + .unwrap(); + assert_eq!( + batch.columns[0].1.value(0), + Some(ScalarValue::Float64(1.25)) + ); + assert_eq!(batch.columns[0].1.value(1), Some(ScalarValue::Null)); + assert_eq!( + batch.columns[1].1.value(0), + Some(ScalarValue::Float64(f32::from_bits(0x43ac_cccd) as f64)) + ); + assert_eq!(batch.columns[1].1.value(1), Some(ScalarValue::Null)); + assert_eq!(batch.columns[2].1.value(0), Some(ScalarValue::Int64(-1000))); + assert_eq!( + batch.columns[3].1.value(0), + Some(ScalarValue::Utf8("alpha".to_owned())) + ); + assert_eq!(batch.columns[3].1.value(1), Some(ScalarValue::Null)); +} + +#[test] +fn converts_mixed_cells_without_dropping_numbers() { + let mixed_number = "3.14".parse::().unwrap(); + let mixed = column( + "mixed", + OriginColumnType::Mixed, + vec![ + OriginCell::Text("text".to_owned()), + OriginCell::Float(mixed_number), + OriginCell::Integer(-7), + OriginCell::Null, + ], + ); + let (imported, store, codecs) = imported(project(vec![workbook( + "Book", + vec![worksheet("Sheet", 4, vec![mixed])], + )])); + + assert_eq!( + imported[0].snapshot.schema.columns[0].logical_type, + LogicalType::Utf8 + ); + let batch = SnapshotReader::new(&imported[0].snapshot, &store, &codecs) + .unwrap() + .read_batch(0, &[]) + .unwrap(); + assert_eq!( + (0..4) + .map(|row| batch.columns[0].1.value(row)) + .collect::>(), + vec![ + Some(ScalarValue::Utf8("text".to_owned())), + Some(ScalarValue::Utf8("3.14".to_owned())), + Some(ScalarValue::Utf8("-7".to_owned())), + Some(ScalarValue::Null), + ] + ); +} + +#[test] +fn generates_empty_names_and_disambiguates_exact_duplicates_case_sensitively() { + let names = ["", "", "name", "name", "name", "Name"]; + let columns = names + .into_iter() + .map(|name| { + column( + name, + OriginColumnType::Integer, + vec![OriginCell::Integer(1)], + ) + }) + .collect(); + let (imported, _, _) = imported(project(vec![workbook( + "Book", + vec![worksheet("Sheet", 1, columns)], + )])); + + assert_eq!( + imported[0] + .snapshot + .schema + .columns + .iter() + .map(|column| column.name.as_str()) + .collect::>(), + [ + "Column 1", "Column 2", "name", "name (2)", "name (3)", "Name" + ] + ); + let source_columns = imported[0].source_metadata["space.nmrtist.plotx.import.origin.columns"] + .as_array() + .unwrap(); + assert_eq!(source_columns[0]["source_name"], ""); + assert_eq!(source_columns[0]["imported_name"], "Column 1"); + assert_eq!(source_columns[3]["source_name"], "name"); + assert_eq!(source_columns[3]["imported_name"], "name (2)"); + assert_eq!( + imported[0].snapshot.schema.columns[3].metadata["space.nmrtist.plotx.import.origin.original_name"], + "name" + ); +} + +#[test] +fn creates_one_candidate_for_each_nonempty_worksheet() { + let value = || column("x", OriginColumnType::Float, vec![OriginCell::Float(1.0)]); + let project = project(vec![ + workbook( + "Book1", + vec![ + worksheet("Empty", 0, Vec::new()), + worksheet("Data", 1, vec![value()]), + ], + ), + workbook("Book2", vec![worksheet("More", 1, vec![value()])]), + ]); + let (imported, _, _) = imported(project); + + assert_eq!( + imported + .iter() + .map(|item| item.name.as_str()) + .collect::>(), + ["Book1 / Data", "Book2 / More"] + ); + assert_eq!( + imported[0].resource_usage.total_owned_bytes, + imported[1].resource_usage.total_owned_bytes + ); + assert_eq!( + imported[0].source_metadata["space.nmrtist.plotx.import.origin.resource_usage"]["total_owned_bytes"], + imported[1].resource_usage.total_owned_bytes + ); +} + +#[test] +fn preserves_project_column_and_parser_metadata() { + let mut data = column( + "signal", + OriginColumnType::Float, + vec![OriginCell::Float(2.0)], + ); + data.long_name = Some("Detector signal".to_owned()); + data.role = Some("Y".to_owned()); + data.units = Some("mV".to_owned()); + data.comments = Some("calibrated".to_owned()); + let mut sheet = worksheet("Sheet1", 1, vec![data]); + sheet.metadata = vec![OriginMetadataEntry { + key: "layer".to_owned(), + value: "raw".to_owned(), + }]; + let mut project = project(vec![workbook("Book1", vec![sheet])]); + project.parameters = vec![OriginMetadataEntry { + key: "temperature".to_owned(), + value: "298".to_owned(), + }]; + project.notes = vec![OriginNote { + name: "Methods".to_owned(), + content: "Prepared under nitrogen.".to_owned(), + }]; + project.diagnostics = vec![OriginDiagnostic { + code: OriginDiagnosticCode::UnsupportedObjectSkipped, + severity: OriginDiagnosticSeverity::Warning, + location: Some(OriginObjectLocation { + workbook: Some("Book1".to_owned()), + worksheet: Some("Sheet1".to_owned()), + column: None, + byte_offset: Some(42), + }), + message: "PlotX skipped a graph.".to_owned(), + }]; + project.unsupported_objects = vec![OriginUnsupportedObjectSummary { + kind: "graphs".to_owned(), + count: 1, + }]; + project.resource_usage = OriginResourceUsage { + input_bytes: 100, + parser_bytes: 50, + decoded_text_bytes: 20, + total_owned_bytes: 150, + workbooks: 1, + worksheets: 1, + columns: 1, + cells: 1, + metadata_records: 3, + }; + let (imported, _, _) = imported(project); + let item = &imported[0]; + + assert_eq!(item.diagnostics.len(), 1); + assert!(item.resource_usage.total_owned_bytes > 150); + assert_eq!( + item.source_metadata["space.nmrtist.plotx.import.format"], + "opj" + ); + assert_eq!( + item.source_metadata["space.nmrtist.plotx.import.origin.producer_version"], + "4.2673 552" + ); + assert_eq!( + item.source_metadata["space.nmrtist.plotx.import.origin.parameters"][0]["key"], + "temperature" + ); + assert_eq!( + item.source_metadata["space.nmrtist.plotx.import.origin.notes"][0]["content"], + "Prepared under nitrogen." + ); + assert_eq!( + item.source_metadata["space.nmrtist.plotx.import.origin.unsupported_objects"][0]["kind"], + "graphs" + ); + assert_eq!( + item.source_metadata["space.nmrtist.plotx.import.origin.resource_usage"]["total_owned_bytes"], + item.resource_usage.total_owned_bytes + ); + assert_eq!( + item.source_metadata["space.nmrtist.plotx.import.origin.diagnostics"][0]["message"], + "PlotX skipped a graph." + ); + assert_eq!( + item.snapshot.metadata["space.nmrtist.plotx.import.origin.workbook"], + "Book1" + ); + assert_eq!( + item.snapshot.metadata["space.nmrtist.plotx.import.origin.worksheet"], + "Sheet1" + ); + assert_eq!( + item.snapshot.metadata["space.nmrtist.plotx.import.origin.diagnostics"][0]["message"], + "PlotX skipped a graph." + ); + let schema_metadata = &item.snapshot.schema.columns[0].metadata; + assert_eq!( + schema_metadata["space.nmrtist.plotx.import.origin.long_name"], + "Detector signal" + ); + assert_eq!( + schema_metadata["space.nmrtist.plotx.import.origin.role"], + "Y" + ); + assert_eq!( + schema_metadata["space.nmrtist.plotx.import.origin.units"], + "mV" + ); + assert_eq!( + schema_metadata["space.nmrtist.plotx.import.origin.comments"], + "calibrated" + ); +} + +#[test] +fn rejects_empty_projects_and_projects_with_only_empty_worksheets() { + let store = MemoryBlockStore::default(); + let codecs = CodecRegistry::with_arrow_ipc(); + for project in [ + project(Vec::new()), + project(vec![workbook( + "Book", + vec![worksheet("Empty", 0, Vec::new())], + )]), + ] { + assert!(matches!( + import_origin_project(project, &store, &codecs, OriginLimits::default()), + Err(OriginImportError::NoSupportedWorksheet) + )); + } +} + +#[test] +fn rejects_snapshot_capacity_before_writing_any_blocks() { + let mut project = project(vec![workbook( + "Book", + vec![worksheet( + "Sheet", + 1, + vec![column( + "value", + OriginColumnType::Float, + vec![OriginCell::Float(1.0)], + )], + )], + )]); + project.resource_usage.input_bytes = 1; + project.resource_usage.total_owned_bytes = 1; + let limits = OriginLimits { + max_total_owned_bytes: 1, + ..OriginLimits::default() + }; + let store = MemoryBlockStore::default(); + let codecs = CodecRegistry::with_arrow_ipc(); + + let error = import_origin_project(project, &store, &codecs, limits).unwrap_err(); + + assert!(matches!( + error, + OriginImportError::LimitExceeded { + resource: "total owned bytes", + limit: 1, + actual: _, + } + )); + assert_eq!(store.block_count(), 0); +} + +#[test] +fn rejects_nonempty_models_with_zero_reported_resource_usage() { + let mut project = project(vec![workbook( + "Book", + vec![worksheet( + "Sheet", + 1, + vec![column( + "value", + OriginColumnType::Integer, + vec![OriginCell::Integer(1)], + )], + )], + )]); + project.resource_usage = OriginResourceUsage::default(); + let store = MemoryBlockStore::default(); + + let error = import_origin_project( + project, + &store, + &CodecRegistry::with_arrow_ipc(), + OriginLimits::default(), + ) + .expect_err("a nonempty model cannot report zero decoded objects"); + + assert!(matches!(error, OriginImportError::InvalidModel { .. })); + assert_eq!(store.block_count(), 0); +} + +#[test] +fn rejects_resource_counts_smaller_than_the_retained_model() { + let mut project = project(vec![workbook( + "Book", + vec![worksheet( + "Sheet", + 1, + vec![column( + "value", + OriginColumnType::Integer, + vec![OriginCell::Integer(1)], + )], + )], + )]); + project.resource_usage.columns = 0; + project.resource_usage.cells = 0; + let store = MemoryBlockStore::default(); + + let error = import_origin_project( + project, + &store, + &CodecRegistry::with_arrow_ipc(), + OriginLimits::default(), + ) + .expect_err("reported counts must cover retained objects"); + + assert!(matches!(error, OriginImportError::InvalidModel { .. })); + assert_eq!(store.block_count(), 0); +} + +#[test] +fn rejects_supported_profile_claims_with_inconsistent_version_fields() { + let base = project(vec![workbook( + "Book", + vec![worksheet( + "Sheet", + 1, + vec![column( + "value", + OriginColumnType::Integer, + vec![OriginCell::Integer(1)], + )], + )], + )]); + let mut projects = vec![base.clone(), base]; + projects[0].probe.raw_version = "4.2673 553".to_owned(); + projects[1].probe.version.build = 553; + + for project in projects { + let store = MemoryBlockStore::default(); + let error = import_origin_project( + project, + &store, + &CodecRegistry::with_arrow_ipc(), + OriginLimits::default(), + ) + .expect_err("a supported profile must match its exact verified header"); + + assert!(matches!(error, OriginImportError::InvalidModel { .. })); + assert_eq!(store.block_count(), 0); + } +} + +#[test] +fn rejects_declared_cell_type_mismatches_without_panicking() { + let invalid = project(vec![workbook( + "Book", + vec![worksheet( + "Sheet", + 1, + vec![column( + "value", + OriginColumnType::Float, + vec![OriginCell::Text("not a float".to_owned())], + )], + )], + )]); + let store = MemoryBlockStore::default(); + let codecs = CodecRegistry::with_arrow_ipc(); + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + import_origin_project(invalid, &store, &codecs, OriginLimits::default()) + })); + + assert!(result.is_ok()); + assert!(matches!( + result.unwrap(), + Err(OriginImportError::InvalidCellType { + column, + row: 0, + expected: OriginColumnType::Float, + .. + }) if column == "value" + )); +} + +#[test] +fn streams_large_worksheets_in_fixed_size_batches() { + const ROWS: usize = 65_537; + let values = (0..ROWS) + .map(|value| OriginCell::Integer(value as i64)) + .collect(); + let (imported, store, codecs) = imported(project(vec![workbook( + "Book", + vec![worksheet( + "Sheet", + ROWS, + vec![column("index", OriginColumnType::Integer, values)], + )], + )])); + + assert_eq!(imported[0].snapshot.batch_count(), 2); + let second = SnapshotReader::new(&imported[0].snapshot, &store, &codecs) + .unwrap() + .read_batch(1, &[]) + .unwrap(); + assert_eq!(second.row_ids.len(), 1); + assert_eq!( + second.columns[0].1.value(0), + Some(ScalarValue::Int64(65_536)) + ); +} + +#[test] +fn generated_names_skip_existing_generated_and_suffixed_names() { + let names = ["name", "name (2)", "name", "", "Column 4"]; + let columns = names + .into_iter() + .map(|name| { + column( + name, + OriginColumnType::Integer, + vec![OriginCell::Integer(1)], + ) + }) + .collect(); + let (imported, _, _) = imported(project(vec![workbook( + "Book", + vec![worksheet("Sheet", 1, columns)], + )])); + + assert_eq!( + imported[0] + .snapshot + .schema + .columns + .iter() + .map(|column| column.name.as_str()) + .collect::>(), + ["name", "name (2)", "name (3)", "Column 4 (2)", "Column 4"] + ); +} + +#[test] +fn all_blank_column_names_receive_position_based_names() { + let columns = ["", " ", "\t"] + .into_iter() + .map(|name| column(name, OriginColumnType::Text, vec![OriginCell::Null])) + .collect(); + let (imported, _, _) = imported(project(vec![workbook( + "Book", + vec![worksheet("Sheet", 1, columns)], + )])); + + assert_eq!( + imported[0] + .snapshot + .schema + .columns + .iter() + .map(|column| column.name.as_str()) + .collect::>(), + ["Column 1", "Column 2", "Column 3"] + ); +} + +#[test] +fn source_metadata_is_a_direct_table_import_source_map() { + let (imported, _, _) = imported(project(vec![workbook( + "Book", + vec![worksheet( + "Sheet", + 1, + vec![column( + "value", + OriginColumnType::Integer, + vec![OriginCell::Integer(1)], + )], + )], + )])); + + let _: &BTreeMap = &imported[0].source_metadata; +} + +#[test] +fn rejects_large_numeric_conversion_before_writing_blocks() { + const ROWS: usize = 65_536; + let mut project = project(vec![workbook( + "Book", + vec![worksheet( + "Sheet", + ROWS, + vec![column( + "value", + OriginColumnType::Float, + (0..ROWS) + .map(|value| OriginCell::Float(value as f64)) + .collect(), + )], + )], + )]); + sync_resource_counts(&mut project); + let store = MemoryBlockStore::default(); + let limits = OriginLimits { + max_total_owned_bytes: 20_000_000, + ..OriginLimits::default() + }; + + let error = import_origin_project(project, &store, &CodecRegistry::with_arrow_ipc(), limits) + .expect_err("all row-scaled conversion allocations must be preflighted"); + + assert!(matches!( + error, + OriginImportError::LimitExceeded { + resource: "total owned bytes", + .. + } + )); + assert_eq!(store.block_count(), 0); +} + +#[test] +fn rejects_large_utf8_conversion_before_writing_blocks() { + const ROWS: usize = 65_536; + let mut project = project(vec![workbook( + "Book", + vec![worksheet( + "Sheet", + ROWS, + vec![column( + "value", + OriginColumnType::Text, + (0..ROWS) + .map(|_| OriginCell::Text("0123456789abcdef".to_owned())) + .collect(), + )], + )], + )]); + sync_resource_counts(&mut project); + let store = MemoryBlockStore::default(); + let limits = OriginLimits { + max_total_owned_bytes: 24_000_000, + ..OriginLimits::default() + }; + + let error = import_origin_project(project, &store, &CodecRegistry::with_arrow_ipc(), limits) + .expect_err("all row-scaled conversion allocations must be preflighted"); + + assert!(matches!( + error, + OriginImportError::LimitExceeded { + resource: "total owned bytes", + .. + } + )); + assert_eq!(store.block_count(), 0); +} diff --git a/crates/io/src/lib.rs b/crates/io/src/lib.rs index 9bf1795..4947d44 100644 --- a/crates/io/src/lib.rs +++ b/crates/io/src/lib.rs @@ -8,6 +8,7 @@ pub mod jcamp_dx; pub mod jeol; mod mass_spec; pub mod nanoscope; +pub mod origin; pub mod waters; pub mod xlsx; diff --git a/crates/io/src/origin.rs b/crates/io/src/origin.rs new file mode 100644 index 0000000..b49e1fa --- /dev/null +++ b/crates/io/src/origin.rs @@ -0,0 +1,772 @@ +//! Bounded detection and engine-neutral transport types for Origin projects. +//! +//! Format detection is based only on the first line of the file. The module +//! does not use filename extensions, and OPJU input is detection-only. + +mod opj; +mod opju; +mod reader; + +const DEFAULT_MAX_HEADER_BYTES: usize = 128; +const MIB: usize = 1024 * 1024; +const OPJ_MAGIC: &[u8] = b"CPYA"; +const OPJU_MAGIC: &[u8] = b"CPYUA"; +const ORIGIN_7_V552_VERSION: &str = "4.2673 552"; +const ORIGIN_9_V951_VERSION: &str = "4.3268 195 W64"; + +/// Maximum leading-byte prefix needed to validate every supported Origin profile. +/// +/// Bounded format dispatchers should read this many bytes, or the complete file +/// when it is shorter, before calling [`probe_origin`]. +pub const MAX_PROBE_BYTES: usize = 152; + +/// Origin project container family detected from its file header. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OriginFormat { + /// Classic Origin project container. + Opj, + /// Newer Unicode Origin project container. + Opju, +} + +/// Whether the detected container can be decoded by this parser profile. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OriginSupport { + /// The exact producer profile is supported. + Supported, + /// The family is recognized but deliberately not decoded. + RecognizedUnsupported, +} + +/// Exact producer profiles with verified parsing rules. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OriginProfile { + /// Classic Origin 7 project header `CPYA 4.2673 552#`. + Origin7V552, + /// 64-bit Origin 9.51 project header `CPYA 4.3268 195 W64 #`. + Origin9V951, +} + +/// Byte order used by a verified Origin profile. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OriginByteOrder { + /// Least-significant byte first. + LittleEndian, +} + +/// Integer components parsed from the producer version header. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct OriginHeaderVersion { + /// Version component before the dot. + pub major: u16, + /// Version component after the dot. + pub minor: u16, + /// Numeric producer build. + pub build: u32, +} + +/// Result of bounded, content-based Origin format detection. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OriginProbe { + /// Detected container family. + pub format: OriginFormat, + /// Version and build text exactly as written in the header. + pub raw_version: String, + /// Parsed integer version components. + pub version: OriginHeaderVersion, + /// Byte order of the detected family. + pub byte_order: OriginByteOrder, + /// Exact supported parser profile, if one is verified. + pub profile: Option, + /// Whether full decoding is supported. + pub support: OriginSupport, +} + +/// A cell decoded from an Origin worksheet column. +#[derive(Debug, Clone, PartialEq)] +pub enum OriginCell { + /// Missing or invalid value. + Null, + /// Floating-point value. + Float(f64), + /// Signed integer value. + Integer(i64), + /// Text value. + Text(String), +} + +/// Logical storage class of an Origin worksheet column. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OriginColumnType { + /// Floating-point values and nulls. + Float, + /// Signed integer values and nulls. + Integer, + /// Text values and nulls. + Text, + /// A verified mixture of numeric and text values. + Mixed, +} + +/// One bounded metadata value retained from an Origin object. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OriginMetadataEntry { + /// Stable or source-derived metadata key. + pub key: String, + /// User-safe decoded value. + pub value: String, +} + +/// One named project note retained without concatenating independent content. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OriginNote { + /// Source note name. + pub name: String, + /// Decoded note content. + pub content: String, +} + +/// One worksheet column in the neutral import model. +#[derive(Debug, Clone, PartialEq)] +pub struct OriginColumn { + /// Source column name. + pub name: String, + /// Optional source long name. + pub long_name: Option, + /// Optional Origin column role such as X or Y. + pub role: Option, + /// Optional source units. + pub units: Option, + /// Optional source comments. + pub comments: Option, + /// Logical class of the decoded cells. + pub column_type: OriginColumnType, + /// Cells in source row order. + pub cells: Vec, +} + +/// One worksheet decoded from an Origin workbook. +#[derive(Debug, Clone, PartialEq)] +pub struct OriginWorksheet { + /// Source worksheet name. + pub name: String, + /// Worksheet columns in source order. + pub columns: Vec, + /// Logical row count, including trailing null rows. + pub row_count: usize, + /// Bounded worksheet-level metadata. + pub metadata: Vec, +} + +/// One workbook decoded from an Origin project. +#[derive(Debug, Clone, PartialEq)] +pub struct OriginWorkbook { + /// Source workbook name. + pub name: String, + /// Worksheets in source order. + pub worksheets: Vec, +} + +/// Severity of a recoverable Origin parsing diagnostic. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OriginDiagnosticSeverity { + /// Informational detail that does not alter imported values. + Info, + /// A bounded object or value was skipped or degraded. + Warning, +} + +/// Stable category for a recoverable Origin parsing diagnostic. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OriginDiagnosticCode { + /// An independently framed unsupported object was skipped. + UnsupportedObjectSkipped, + /// An independently framed unsupported column was skipped. + UnsupportedColumnSkipped, + /// Nonessential metadata could not be retained. + MetadataSkipped, + /// Text or value decoding required a documented fallback. + DecodingWarning, +} + +/// Structured location of a recoverable parsing diagnostic. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OriginObjectLocation { + /// Workbook name, when known. + pub workbook: Option, + /// Worksheet name, when known. + pub worksheet: Option, + /// Column name, when known. + pub column: Option, + /// Source byte offset, when meaningful. + pub byte_offset: Option, +} + +/// User-safe diagnostic emitted while retaining a usable project. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OriginDiagnostic { + /// Stable diagnostic category. + pub code: OriginDiagnosticCode, + /// Diagnostic severity. + pub severity: OriginDiagnosticSeverity, + /// Source object or byte location, when known. + pub location: Option, + /// Plain-language message safe to present to a user. + pub message: String, +} + +/// Count of skipped Origin objects of one source-defined kind. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OriginUnsupportedObjectSummary { + /// Source object kind, such as graph or matrix. + pub kind: String, + /// Number of skipped objects of this kind. + pub count: usize, +} + +/// Conservative resource accounting carried into later conversion stages. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct OriginResourceUsage { + /// Retained source input bytes. + pub input_bytes: usize, + /// Parser-owned decoded and structural bytes. + pub parser_bytes: usize, + /// Decoded text bytes included in parser storage. + pub decoded_text_bytes: usize, + /// Cumulative owned-byte charge across import stages. + pub total_owned_bytes: usize, + /// Decoded workbook count. + pub workbooks: usize, + /// Decoded worksheet count. + pub worksheets: usize, + /// Decoded column count. + pub columns: usize, + /// Decoded cell count. + pub cells: usize, + /// Logical metadata records traversed, excluding list terminators. + pub metadata_records: usize, +} + +/// Complete engine-neutral result of an Origin project read. +#[derive(Debug, Clone, PartialEq)] +pub struct OriginProject { + /// Probe that selected the exact parsing profile. + pub probe: OriginProbe, + /// Bounded project parameters. + pub parameters: Vec, + /// Bounded project notes. + pub notes: Vec, + /// Decoded workbooks in source order. + pub workbooks: Vec, + /// Recoverable parsing diagnostics. + pub diagnostics: Vec, + /// Counts of independently skipped unsupported objects. + pub unsupported_objects: Vec, + /// Conservative parser and allocation accounting. + pub resource_usage: OriginResourceUsage, +} + +/// Resource limits applied to untrusted Origin input. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct OriginLimits { + /// Maximum complete source size. + pub max_input_bytes: usize, + /// Maximum first-line header size, including LF. + pub max_header_bytes: usize, + /// Maximum individual framed block size. + pub max_block_bytes: usize, + /// Maximum individual decoded string size. + pub max_string_bytes: usize, + /// Maximum cumulative decoded text size. + pub max_decoded_text_bytes: usize, + /// Maximum cumulative parser-owned allocation. + pub max_parser_bytes: usize, + /// Maximum cumulative owned allocation across import stages. + pub max_total_owned_bytes: usize, + /// Maximum workbook count. + pub max_workbooks: usize, + /// Maximum source window records retained for worksheet association. + pub max_window_records: usize, + /// Maximum worksheet count per workbook. + pub max_worksheets_per_workbook: usize, + /// Maximum total worksheet data column count. + pub max_columns: usize, + /// Maximum logical metadata records traversed, excluding list terminators. + pub max_metadata_records: usize, + /// Maximum logical rows in one column. + pub max_rows_per_column: usize, + /// Maximum total decoded cell count. + pub max_cells: usize, + /// Maximum metadata nesting depth accepted by readers. + pub max_metadata_depth: usize, +} + +impl Default for OriginLimits { + fn default() -> Self { + Self { + max_input_bytes: 128 * MIB, + max_header_bytes: DEFAULT_MAX_HEADER_BYTES, + max_block_bytes: 32 * MIB, + max_string_bytes: MIB, + max_decoded_text_bytes: 32 * MIB, + max_parser_bytes: 128 * MIB, + max_total_owned_bytes: 384 * MIB, + max_workbooks: 256, + max_window_records: 1024, + max_worksheets_per_workbook: 128, + max_columns: 4096, + max_metadata_records: 65_536, + max_rows_per_column: 1_000_000, + max_cells: 2_000_000, + max_metadata_depth: 32, + } + } +} + +impl OriginLimits { + /// Rejects unusable custom limits before any input is parsed. + pub fn validate(&self) -> Result<(), OriginError> { + let values = [ + ("max_input_bytes", self.max_input_bytes), + ("max_header_bytes", self.max_header_bytes), + ("max_block_bytes", self.max_block_bytes), + ("max_string_bytes", self.max_string_bytes), + ("max_decoded_text_bytes", self.max_decoded_text_bytes), + ("max_parser_bytes", self.max_parser_bytes), + ("max_total_owned_bytes", self.max_total_owned_bytes), + ("max_workbooks", self.max_workbooks), + ("max_window_records", self.max_window_records), + ( + "max_worksheets_per_workbook", + self.max_worksheets_per_workbook, + ), + ("max_columns", self.max_columns), + ("max_metadata_records", self.max_metadata_records), + ("max_rows_per_column", self.max_rows_per_column), + ("max_cells", self.max_cells), + ("max_metadata_depth", self.max_metadata_depth), + ]; + if let Some((name, value)) = values.into_iter().find(|(_, value)| *value == 0) { + return Err(OriginError::InvalidLimit { + name, + value, + reason: "the limit must be greater than zero", + }); + } + if self.max_input_bytes.checked_add(1).is_none() { + return Err(OriginError::InvalidLimit { + name: "max_input_bytes", + value: self.max_input_bytes, + reason: "the limit must leave room for an oversize sentinel byte", + }); + } + if self.max_header_bytes.checked_add(1).is_none() { + return Err(OriginError::InvalidLimit { + name: "max_header_bytes", + value: self.max_header_bytes, + reason: "the limit is too large for bounded header probing", + }); + } + if self.max_string_bytes.checked_add(1).is_none() { + return Err(OriginError::InvalidLimit { + name: "max_string_bytes", + value: self.max_string_bytes, + reason: "the limit must leave room for a metadata-line sentinel byte", + }); + } + Ok(()) + } +} + +/// Errors returned while probing or reading untrusted Origin projects. +#[derive(Debug, thiserror::Error, PartialEq, Eq)] +pub enum OriginError { + /// The leading bytes do not match an Origin project family. + #[error("the file is not a recognized Origin project")] + UnrecognizedFormat, + + /// Input ended before a required bounded field was complete. + #[error("Origin project data is truncated at byte {offset}: need {needed} bytes, have {have}")] + Truncated { + /// Offset of the incomplete field. + offset: usize, + /// Bytes required at the offset. + needed: usize, + /// Bytes available at the offset. + have: usize, + }, + + /// The first line cannot fit within the configured header bound. + #[error("Origin project header exceeds the configured {limit}-byte limit")] + HeaderTooLong { + /// Configured maximum bytes, including LF. + limit: usize, + }, + + /// The family signature is present but the version line is invalid. + #[error("malformed Origin project header: {detail}")] + MalformedHeader { + /// User-safe description of the failed grammar rule. + detail: String, + }, + + /// A valid classic header names a producer profile without verified rules. + #[error("Origin project version {raw_version} is not supported")] + UnsupportedVersion { + /// Version and build text from the validated header. + raw_version: String, + }, + + /// OPJU was recognized, but this release intentionally does not decode it. + #[error("{message}")] + UnsupportedOpjuVariant { + /// Plain user-safe explanation that no data was imported. + message: String, + }, + + /// A caller supplied a limit that cannot be applied safely. + #[error("invalid Origin limit {name}={value}: {reason}")] + InvalidLimit { + /// Public field name of the invalid limit. + name: &'static str, + /// Invalid value. + value: usize, + /// Stable reason the value cannot be used. + reason: &'static str, + }, + + /// A verified resource count exceeds its configured bound. + #[error("Origin import {resource} is {actual}, exceeding the configured limit of {limit}")] + LimitExceeded { + /// Resource being bounded. + resource: &'static str, + /// Configured maximum. + limit: usize, + /// Observed or requested amount. + actual: usize, + }, + + /// Checked arithmetic could not represent a derived size or offset. + #[error("Origin import size calculation overflowed for {resource}")] + ArithmeticOverflow { + /// Resource whose calculation overflowed. + resource: &'static str, + }, + + /// A bounded allocation could not be reserved. + #[error("Origin import could not reserve {requested} bytes for {resource}")] + AllocationFailed { + /// Allocation purpose. + resource: &'static str, + /// Requested capacity in bytes. + requested: usize, + }, + + /// Validated framing contains an impossible or unsupported structure. + #[error("corrupt Origin project structure at byte {offset}: {detail}")] + CorruptStructure { + /// Source byte offset. + offset: usize, + /// User-safe structural description. + detail: String, + }, + + /// A value uses an encoding that cannot be decoded without guessing. + #[error("unsupported Origin text encoding at byte {offset}: {encoding}")] + UnsupportedEncoding { + /// Source byte offset. + offset: usize, + /// Source encoding description. + encoding: String, + }, + + /// A structurally valid project uses an unimplemented independent feature. + #[error("unsupported Origin project feature: {feature}")] + UnsupportedFeature { + /// User-safe feature description. + feature: String, + }, + + /// Declared and decoded row geometry disagree. + #[error("Origin column {column} declares {declared} rows but contains {decoded} decoded rows")] + InconsistentRowCount { + /// Source column name. + column: String, + /// Declared logical row count. + declared: usize, + /// Decoded row count. + decoded: usize, + }, + + /// Parsing completed without any worksheet that can be imported. + #[error("the Origin project contains no supported worksheet data")] + NoSupportedWorksheet, +} + +struct AccountedOriginProbe { + probe: OriginProbe, + retained_parser_bytes: usize, +} + +/// Detects an Origin family and exact supported profile from bounded content. +pub fn probe_origin(bytes: &[u8]) -> Result { + let limits = OriginLimits::default(); + Ok(probe_origin_with_limits(bytes, &limits, 0)?.probe) +} + +/// Reads a complete Origin project under explicit resource limits. +pub fn read_origin(bytes: &[u8], limits: OriginLimits) -> Result { + limits.validate()?; + enforce_limit("input bytes", bytes.len(), limits.max_input_bytes)?; + enforce_limit( + "total owned bytes", + bytes.len(), + limits.max_total_owned_bytes, + )?; + + let accounted = probe_origin_with_limits(bytes, &limits, bytes.len())?; + let probe = accounted.probe; + match probe.format { + OriginFormat::Opju => opju::read(probe), + OriginFormat::Opj => opj::read(bytes, &limits, probe, accounted.retained_parser_bytes), + } +} + +fn enforce_limit(resource: &'static str, actual: usize, limit: usize) -> Result<(), OriginError> { + if actual > limit { + return Err(OriginError::LimitExceeded { + resource, + limit, + actual, + }); + } + Ok(()) +} + +fn probe_origin_with_limits( + bytes: &[u8], + limits: &OriginLimits, + initial_total_owned_bytes: usize, +) -> Result { + limits.validate()?; + let format = identify_family(bytes)?; + let line = bounded_first_line(bytes, limits.max_header_bytes)?; + if !line.iter().all(|byte| matches!(byte, b' '..=b'~')) { + return malformed("the version line must contain printable ASCII only"); + } + + let line = std::str::from_utf8(line) + .map_err(|_| malformed_error("the version line must contain printable ASCII only"))?; + let raw_version = match format { + OriginFormat::Opj => { + let value = line + .strip_prefix("CPYA ") + .and_then(|value| value.strip_suffix('#')) + .ok_or_else(|| { + malformed_error("classic OPJ headers must end with exactly one # before LF") + })?; + value + .strip_suffix(' ') + .filter(|value| value.ends_with(" W64")) + .unwrap_or(value) + } + OriginFormat::Opju => line.strip_prefix("CPYUA ").ok_or_else(|| { + malformed_error("OPJU headers must contain one space after the CPYUA signature") + })?, + }; + let version = parse_version(raw_version, format)?; + let (raw_version, retained_parser_bytes) = + copy_header_version(raw_version, limits, initial_total_owned_bytes)?; + + match format { + OriginFormat::Opj => { + let profile = match raw_version.as_str() { + ORIGIN_7_V552_VERSION => OriginProfile::Origin7V552, + ORIGIN_9_V951_VERSION => OriginProfile::Origin9V951, + _ => return Err(OriginError::UnsupportedVersion { raw_version }), + }; + opj::validate_initial_structure(bytes, profile, limits.max_block_bytes)?; + Ok(AccountedOriginProbe { + probe: OriginProbe { + format, + raw_version, + version, + byte_order: OriginByteOrder::LittleEndian, + profile: Some(profile), + support: OriginSupport::Supported, + }, + retained_parser_bytes, + }) + } + OriginFormat::Opju => Ok(AccountedOriginProbe { + probe: OriginProbe { + format, + raw_version, + version, + byte_order: OriginByteOrder::LittleEndian, + profile: None, + support: OriginSupport::RecognizedUnsupported, + }, + retained_parser_bytes, + }), + } +} + +fn identify_family(bytes: &[u8]) -> Result { + if bytes.starts_with(OPJU_MAGIC) { + return Ok(OriginFormat::Opju); + } + if bytes.starts_with(OPJ_MAGIC) { + return Ok(OriginFormat::Opj); + } + + let needed = if OPJ_MAGIC.starts_with(bytes) { + Some(OPJ_MAGIC.len()) + } else if OPJU_MAGIC.starts_with(bytes) { + Some(OPJU_MAGIC.len()) + } else { + None + }; + match needed { + Some(needed) => Err(OriginError::Truncated { + offset: 0, + needed, + have: bytes.len(), + }), + None => Err(OriginError::UnrecognizedFormat), + } +} + +fn bounded_first_line(bytes: &[u8], max_header_bytes: usize) -> Result<&[u8], OriginError> { + let scan_len = bytes.len().min(max_header_bytes); + let scan = bytes.get(..scan_len).ok_or(OriginError::Truncated { + offset: 0, + needed: scan_len, + have: bytes.len(), + })?; + if let Some(newline) = scan.iter().position(|byte| *byte == b'\n') { + return scan.get(..newline).ok_or(OriginError::Truncated { + offset: 0, + needed: newline, + have: scan.len(), + }); + } + if bytes.len() >= max_header_bytes { + return Err(OriginError::HeaderTooLong { + limit: max_header_bytes, + }); + } + Err(OriginError::Truncated { + offset: bytes.len(), + needed: 1, + have: 0, + }) +} + +fn parse_version( + raw_version: &str, + format: OriginFormat, +) -> Result { + let mut fields = raw_version.split(' '); + let version = fields.next().unwrap_or_default(); + let build = fields.next().unwrap_or_default(); + let platform = fields.next(); + let platform_is_valid = + platform.is_none() || matches!((format, platform), (OriginFormat::Opj, Some("W64"))); + if version.is_empty() || build.is_empty() || !platform_is_valid || fields.next().is_some() { + return malformed( + "the header must contain version and numeric build fields followed only by the optional classic OPJ W64 marker", + ); + } + + let (major, minor) = version + .split_once('.') + .ok_or_else(|| malformed_error("the version token must contain major.minor integers"))?; + if !is_ascii_digits(major) + || !is_ascii_digits(minor) + || minor.contains('.') + || !is_ascii_digits(build) + { + return malformed("the version and build fields must contain decimal digits only"); + } + + let major = major + .parse::() + .map_err(|_| malformed_error("the major version is outside the supported integer range"))?; + let minor = minor + .parse::() + .map_err(|_| malformed_error("the minor version is outside the supported integer range"))?; + let build = build + .parse::() + .map_err(|_| malformed_error("the build is outside the supported integer range"))?; + Ok(OriginHeaderVersion { + major, + minor, + build, + }) +} + +fn is_ascii_digits(value: &str) -> bool { + !value.is_empty() && value.bytes().all(|byte| byte.is_ascii_digit()) +} + +fn copy_header_version( + raw_version: &str, + limits: &OriginLimits, + initial_total_owned_bytes: usize, +) -> Result<(String, usize), OriginError> { + let requested = raw_version.len(); + enforce_limit("string bytes", requested, limits.max_string_bytes)?; + enforce_limit("parser bytes", requested, limits.max_parser_bytes)?; + let preflight_total = + reader::checked_add(initial_total_owned_bytes, requested, "total owned bytes")?; + enforce_limit( + "total owned bytes", + preflight_total, + limits.max_total_owned_bytes, + )?; + + let mut owned = String::new(); + owned + .try_reserve_exact(requested) + .map_err(|_| OriginError::AllocationFailed { + resource: "header version text", + requested, + })?; + let retained_parser_bytes = owned.capacity(); + enforce_limit( + "parser bytes", + retained_parser_bytes, + limits.max_parser_bytes, + )?; + let actual_total = reader::checked_add( + initial_total_owned_bytes, + retained_parser_bytes, + "total owned bytes", + )?; + enforce_limit( + "total owned bytes", + actual_total, + limits.max_total_owned_bytes, + )?; + owned.push_str(raw_version); + Ok((owned, retained_parser_bytes)) +} + +fn malformed(detail: &str) -> Result { + Err(malformed_error(detail)) +} + +fn malformed_error(detail: &str) -> OriginError { + OriginError::MalformedHeader { + detail: detail.to_owned(), + } +} + +#[cfg(test)] +#[path = "origin/reader_tests.rs"] +mod reader_tests; + +#[cfg(test)] +#[path = "origin/tests.rs"] +mod tests; diff --git a/crates/io/src/origin/opj.rs b/crates/io/src/origin/opj.rs new file mode 100644 index 0000000..e998eca --- /dev/null +++ b/crates/io/src/origin/opj.rs @@ -0,0 +1,621 @@ +use std::mem::size_of; + +use super::reader::{FramedBlock, Reader, checked_add}; +use super::{ + OriginColumn, OriginDiagnosticCode, OriginError, OriginLimits, OriginProbe, OriginProfile, + OriginProject, OriginResourceUsage, OriginWorkbook, OriginWorksheet, +}; + +mod metadata; +mod records; + +const ORIGIN_VERSION_OFFSET: usize = 0x1b; +const BLOCK_PREFIX_LEN: usize = 5; + +#[derive(Debug, Clone, Copy)] +enum MetadataPolicy { + Full, + WindowsOnly, +} + +#[derive(Debug, Clone, Copy)] +struct ProfileLayout { + name: &'static str, + raw_version: &'static str, + signature: &'static [u8], + global_header_len: usize, + embedded_version: f64, + data_header_len: usize, + metadata_policy: MetadataPolicy, +} + +const ORIGIN7_LAYOUT: ProfileLayout = ProfileLayout { + name: "Origin7V552", + raw_version: "4.2673 552", + signature: b"CPYA 4.2673 552#\n", + global_header_len: 39, + embedded_version: 7.0552, + data_header_len: 123, + metadata_policy: MetadataPolicy::Full, +}; + +const ORIGIN9_LAYOUT: ProfileLayout = ProfileLayout { + name: "Origin9V951", + raw_version: "4.3268 195 W64", + signature: b"CPYA 4.3268 195 W64 #\n", + global_header_len: 115, + embedded_version: 9.510195, + data_header_len: 147, + metadata_policy: MetadataPolicy::WindowsOnly, +}; + +fn profile_layout(profile: OriginProfile) -> &'static ProfileLayout { + match profile { + OriginProfile::Origin7V552 => &ORIGIN7_LAYOUT, + OriginProfile::Origin9V951 => &ORIGIN9_LAYOUT, + } +} + +#[derive(Debug)] +pub(super) struct RawOpjDataSection<'a> { + pub(super) header: &'a [u8], + pub(super) content: Option<&'a [u8]>, +} + +#[derive(Debug)] +pub(super) struct RawOpjProject<'a> { + pub(super) origin_header: &'a [u8], + pub(super) data_sections: Vec>, + pub(super) remaining: &'a [u8], + pub(super) resource_usage: OriginResourceUsage, +} + +pub(super) fn validate_initial_structure( + bytes: &[u8], + profile: OriginProfile, + max_block_bytes: usize, +) -> Result<(), OriginError> { + let layout = profile_layout(profile); + let initial_structure_len = layout + .signature + .len() + .checked_add(BLOCK_PREFIX_LEN) + .and_then(|length| length.checked_add(layout.global_header_len)) + .and_then(|length| length.checked_add(1 + BLOCK_PREFIX_LEN)) + .ok_or(OriginError::ArithmeticOverflow { + resource: "initial OPJ probe structure", + })?; + let initial_len = bytes.len().min(initial_structure_len); + let initial = bytes + .get(..initial_len) + .ok_or(OriginError::ArithmeticOverflow { + resource: "initial OPJ probe structure", + })?; + let limits = OriginLimits { + max_input_bytes: initial_structure_len, + max_block_bytes, + max_total_owned_bytes: initial_structure_len, + ..OriginLimits::default() + }; + let mut reader = Reader::new(initial, &limits)?; + let signature = reader.read_slice(layout.signature.len())?; + if signature != layout.signature { + return Err(OriginError::CorruptStructure { + offset: 0, + detail: "the classic OPJ signature changed inside its initial framing".to_owned(), + }); + } + let header_block = reader.read_block()?; + let (header_offset, header) = + require_data_block(header_block, "the Origin header must be a data block")?; + require_exact_length( + header_offset, + header, + layout.global_header_len, + "Origin header payload", + )?; + validate_embedded_origin_version(header_offset, header, layout)?; + let terminator_offset = reader.offset(); + let terminator = reader.read_slice(BLOCK_PREFIX_LEN)?; + if terminator != [0, 0, 0, 0, b'\n'] { + return Err(OriginError::CorruptStructure { + offset: terminator_offset, + detail: "the Origin header must end with a null block".to_owned(), + }); + } + Ok(()) +} + +pub(super) fn read( + bytes: &[u8], + limits: &OriginLimits, + probe: OriginProbe, + retained_probe_bytes: usize, +) -> Result { + let profile = probe + .profile + .ok_or_else(|| OriginError::UnsupportedVersion { + raw_version: probe.raw_version.clone(), + })?; + let raw = parse_raw(bytes, limits, profile, retained_probe_bytes)?; + if raw.data_sections.is_empty() && raw.remaining.is_empty() { + return Err(OriginError::NoSupportedWorksheet); + } + + let _validated_header_len = raw.origin_header.len(); + let mut usage = raw.resource_usage; + let metadata_offset = + bytes + .len() + .checked_sub(raw.remaining.len()) + .ok_or(OriginError::ArithmeticOverflow { + resource: "OPJ metadata offset", + })?; + let mut parsed = match profile_layout(profile).metadata_policy { + MetadataPolicy::Full => { + metadata::parse(raw.remaining, metadata_offset, limits, &mut usage)? + } + MetadataPolicy::WindowsOnly => { + metadata::parse_windows_only(raw.remaining, metadata_offset, limits, &mut usage)? + } + }; + let mut unsupported_columns = 0_usize; + + for section in &raw.data_sections { + match records::decode_column_record( + profile, + section.header, + section.content, + limits, + &mut usage, + ) { + Ok(decoded) => { + match associate_dataset(&decoded.dataset_name, &parsed.windows) { + DatasetAssociation::Window { + index, + prefix_bytes, + } => { + let column = make_column(decoded, Some(prefix_bytes))?; + let window = parsed.windows.get_mut(index).ok_or( + OriginError::ArithmeticOverflow { + resource: "associated Origin window", + }, + )?; + metadata::try_reserve( + &mut window.columns, + 1, + "Origin worksheet columns", + limits, + &mut usage, + )?; + window.columns.push(column); + } + DatasetAssociation::ExactWindow => { + unsupported_columns = + checked_add(unsupported_columns, 1, "unsupported Origin columns")?; + } + DatasetAssociation::Fallback => { + // A decoded column is still unsafe to expose when no + // verified window record proves its table grouping. + // Combining unrelated unmatched columns would invent + // row alignment that is not present in the source. + unsupported_columns = + checked_add(unsupported_columns, 1, "unsupported Origin columns")?; + } + } + } + Err(OriginError::UnsupportedFeature { .. }) => { + // The Task 4 decoder emits UnsupportedFeature only while + // classifying a fully framed column header, before it reads or + // interprets that column's payload. The outer data-section + // bounds therefore make this individual skip deterministic. + unsupported_columns = + checked_add(unsupported_columns, 1, "unsupported Origin columns")?; + } + Err(error) => return Err(error), + } + } + + if unsupported_columns > 0 { + metadata::push_summary( + &mut parsed.unsupported_objects, + "worksheet columns", + unsupported_columns, + limits, + &mut usage, + )?; + metadata::push_diagnostic( + &mut parsed.diagnostics, + OriginDiagnosticCode::UnsupportedColumnSkipped, + "PlotX skipped independently framed Origin columns whose value layouts are not verified.", + None, + limits, + &mut usage, + )?; + } + + let unused_windows = parsed + .windows + .iter() + .filter(|window| window.columns.is_empty()) + .count(); + if unused_windows > 0 { + metadata::push_summary( + &mut parsed.unsupported_objects, + "unsupported window records", + unused_windows, + limits, + &mut usage, + )?; + metadata::push_diagnostic( + &mut parsed.diagnostics, + OriginDiagnosticCode::UnsupportedObjectSkipped, + "PlotX skipped Origin window records that had no supported worksheet columns.", + None, + limits, + &mut usage, + )?; + } + + let workbooks = assemble_workbooks(parsed.windows, limits, &mut usage)?; + Ok(OriginProject { + probe, + parameters: parsed.parameters, + notes: parsed.notes, + workbooks, + diagnostics: parsed.diagnostics, + unsupported_objects: parsed.unsupported_objects, + resource_usage: usage, + }) +} + +enum DatasetAssociation { + Window { index: usize, prefix_bytes: usize }, + ExactWindow, + Fallback, +} + +fn associate_dataset(dataset_name: &str, windows: &[metadata::WindowInfo]) -> DatasetAssociation { + let mut best = None; + let mut best_len = 0_usize; + let mut ambiguous = false; + let mut exact = false; + + for (index, window) in windows.iter().enumerate() { + let Some(window_name) = window.name.as_deref() else { + continue; + }; + if dataset_name == window_name { + exact = true; + continue; + } + let Some(suffix) = dataset_name + .strip_prefix(window_name) + .and_then(|rest| rest.strip_prefix('_')) + else { + continue; + }; + if suffix.is_empty() { + continue; + } + + match window_name.len().cmp(&best_len) { + std::cmp::Ordering::Greater => { + best = Some(index); + best_len = window_name.len(); + ambiguous = false; + } + std::cmp::Ordering::Equal => ambiguous = true, + std::cmp::Ordering::Less => {} + } + } + + if exact { + return DatasetAssociation::ExactWindow; + } + match (best, ambiguous) { + (Some(index), false) => DatasetAssociation::Window { + index, + prefix_bytes: best_len, + }, + _ => DatasetAssociation::Fallback, + } +} + +fn make_column( + decoded: records::DecodedColumnRecord, + prefix_bytes: Option, +) -> Result { + let mut name = decoded.dataset_name; + if let Some(prefix_bytes) = prefix_bytes { + if !name.is_char_boundary(prefix_bytes) || name.get(prefix_bytes..).is_none() { + return Err(OriginError::CorruptStructure { + offset: 0, + detail: "an associated Origin dataset prefix is not a text boundary".to_owned(), + }); + } + name.replace_range(..prefix_bytes, ""); + if name.as_bytes().first() == Some(&b'_') { + name.remove(0); + } + } + Ok(OriginColumn { + name, + long_name: None, + role: None, + units: None, + comments: None, + column_type: decoded.column_type, + cells: decoded.cells, + }) +} + +fn assemble_workbooks( + windows: Vec, + limits: &OriginLimits, + usage: &mut OriginResourceUsage, +) -> Result, OriginError> { + let workbook_count = windows + .iter() + .filter(|window| !window.columns.is_empty()) + .count(); + if workbook_count == 0 { + return Err(OriginError::NoSupportedWorksheet); + } + enforce_count("workbooks", workbook_count, limits.max_workbooks)?; + enforce_count( + "worksheets per workbook", + 1, + limits.max_worksheets_per_workbook, + )?; + let has_supported_rows = windows + .iter() + .any(|window| window.columns.iter().any(|column| !column.cells.is_empty())); + if !has_supported_rows { + return Err(OriginError::NoSupportedWorksheet); + } + + let mut workbooks = Vec::new(); + metadata::try_reserve( + &mut workbooks, + workbook_count, + "Origin workbooks", + limits, + usage, + )?; + for window in windows { + if window.columns.is_empty() { + continue; + } + let name = window.name.ok_or(OriginError::CorruptStructure { + offset: 0, + detail: "a supported Origin worksheet lost its validated window name".to_owned(), + })?; + push_workbook(&mut workbooks, name, window.columns, limits, usage)?; + } + usage.workbooks = workbook_count; + usage.worksheets = workbook_count; + Ok(workbooks) +} + +fn push_workbook( + workbooks: &mut Vec, + name: String, + columns: Vec, + limits: &OriginLimits, + usage: &mut OriginResourceUsage, +) -> Result<(), OriginError> { + let row_count = columns + .iter() + .map(|column| column.cells.len()) + .max() + .unwrap_or(0); + let worksheet_name = metadata::copy_generated_text("Sheet1", limits, usage)?; + let mut worksheets = Vec::new(); + metadata::try_reserve(&mut worksheets, 1, "Origin worksheets", limits, usage)?; + worksheets.push(OriginWorksheet { + name: worksheet_name, + columns, + row_count, + metadata: Vec::new(), + }); + workbooks.push(OriginWorkbook { name, worksheets }); + Ok(()) +} + +fn enforce_count(resource: &'static str, actual: usize, limit: usize) -> Result<(), OriginError> { + if actual > limit { + return Err(OriginError::LimitExceeded { + resource, + limit, + actual, + }); + } + Ok(()) +} + +pub(super) fn parse_raw<'a>( + bytes: &'a [u8], + limits: &OriginLimits, + profile: OriginProfile, + initial_parser_bytes: usize, +) -> Result, OriginError> { + let layout = profile_layout(profile); + let mut reader = Reader::new_with_parser_bytes(bytes, limits, initial_parser_bytes)?; + let signature = reader.read_slice(layout.signature.len())?; + if signature != layout.signature { + return Err(OriginError::UnsupportedVersion { + raw_version: format!("classic OPJ signature does not match {}", layout.name), + }); + } + + // OpenOPJ documents the Origin header as one data block followed by a null + // block, with the Origin version f64 at payload offset 0x1b: + // https://github.com/jgonera/openopj/blob/42ddcf1eb3a490744c54fca0a4ed6fe7a5e723ca/docs/opj_format.markdown + // https://github.com/jgonera/openopj/blob/42ddcf1eb3a490744c54fca0a4ed6fe7a5e723ca/lib/OpenOPJ/OPJFile.php + let origin_header_block = reader.read_block()?; + let (origin_header_offset, origin_header) = require_data_block( + origin_header_block, + "the Origin header must be a data block", + )?; + require_exact_length( + origin_header_offset, + origin_header, + layout.global_header_len, + "Origin header payload", + )?; + validate_embedded_origin_version(origin_header_offset, origin_header, layout)?; + + let header_terminator = reader.read_block()?; + require_null_block( + header_terminator, + "the Origin header must end with a null block", + )?; + + let mut data_sections = Vec::new(); + loop { + // Each verified profile's data list is a sequence of + // , + // followed by a consumed list null. Parsing stops at that exact + // bounded boundary. + // https://github.com/jgonera/openopj/blob/42ddcf1eb3a490744c54fca0a4ed6fe7a5e723ca/docs/opj_format.markdown + // https://github.com/jgonera/openopj/blob/42ddcf1eb3a490744c54fca0a4ed6fe7a5e723ca/lib/OpenOPJ/OPJFile.php + let header_block = reader.read_block()?; + let (header_offset, header) = match header_block { + FramedBlock::Null { .. } => break, + FramedBlock::Data { offset, payload } => (offset, payload), + }; + require_exact_length( + header_offset, + header, + layout.data_header_len, + match profile { + OriginProfile::Origin7V552 => "Origin7V552 data-header payload", + OriginProfile::Origin9V951 => "Origin9V951 data-header payload", + }, + )?; + + let content = match reader.read_block()? { + FramedBlock::Null { .. } => None, + FramedBlock::Data { payload, .. } => Some(payload), + }; + let section_terminator = reader.read_block()?; + require_null_block( + section_terminator, + "an Origin data section must end with a null block", + )?; + + let section_count = checked_add(data_sections.len(), 1, "raw OPJ data sections")?; + if section_count > limits.max_columns { + return Err(OriginError::LimitExceeded { + resource: "data sections", + limit: limits.max_columns, + actual: section_count, + }); + } + reader.try_reserve(&mut data_sections, 1, "raw OPJ data sections")?; + data_sections.push(RawOpjDataSection { header, content }); + } + + let remaining = bytes + .get(reader.offset()..) + .ok_or(OriginError::ArithmeticOverflow { + resource: "remaining OPJ structure", + })?; + let resource_usage = reader.into_usage(); + Ok(RawOpjProject { + origin_header, + data_sections, + remaining, + resource_usage, + }) +} + +fn validate_embedded_origin_version( + header_block_offset: usize, + header: &[u8], + layout: &ProfileLayout, +) -> Result<(), OriginError> { + let version_payload_delta = checked_add( + BLOCK_PREFIX_LEN, + ORIGIN_VERSION_OFFSET, + "embedded Origin version offset", + )?; + let version_offset_in_file = checked_add( + header_block_offset, + version_payload_delta, + "embedded Origin version offset", + )?; + let version_end = checked_add( + ORIGIN_VERSION_OFFSET, + size_of::(), + "embedded Origin version range", + )?; + let available_version_bytes = + header + .len() + .checked_sub(ORIGIN_VERSION_OFFSET) + .ok_or(OriginError::ArithmeticOverflow { + resource: "embedded Origin version bytes", + })?; + let version_bytes = + header + .get(ORIGIN_VERSION_OFFSET..version_end) + .ok_or(OriginError::Truncated { + offset: version_offset_in_file, + needed: size_of::(), + have: available_version_bytes, + })?; + let version_array: [u8; 8] = version_bytes + .try_into() + .map_err(|_| OriginError::Truncated { + offset: version_offset_in_file, + needed: size_of::(), + have: version_bytes.len(), + })?; + let version = f64::from_le_bytes(version_array); + if version.to_bits() != layout.embedded_version.to_bits() { + return Err(OriginError::UnsupportedVersion { + raw_version: format!("{} header embeds Origin {version}", layout.raw_version), + }); + } + Ok(()) +} + +fn require_data_block<'a>( + block: FramedBlock<'a>, + detail: &'static str, +) -> Result<(usize, &'a [u8]), OriginError> { + match block { + FramedBlock::Data { offset, payload } => Ok((offset, payload)), + FramedBlock::Null { offset } => Err(OriginError::CorruptStructure { + offset, + detail: detail.to_owned(), + }), + } +} + +fn require_null_block(block: FramedBlock<'_>, detail: &'static str) -> Result<(), OriginError> { + match block { + FramedBlock::Null { .. } => Ok(()), + FramedBlock::Data { .. } => Err(OriginError::CorruptStructure { + offset: block.offset(), + detail: detail.to_owned(), + }), + } +} + +fn require_exact_length( + block_offset: usize, + payload: &[u8], + expected: usize, + field: &'static str, +) -> Result<(), OriginError> { + if payload.len() != expected { + return Err(OriginError::CorruptStructure { + offset: block_offset, + detail: format!("{field} must be exactly {expected} bytes"), + }); + } + Ok(()) +} diff --git a/crates/io/src/origin/opj/metadata.rs b/crates/io/src/origin/opj/metadata.rs new file mode 100644 index 0000000..483a815 --- /dev/null +++ b/crates/io/src/origin/opj/metadata.rs @@ -0,0 +1,775 @@ +use std::fmt::{self, Write as _}; +use std::mem::size_of; + +use crate::origin::{ + OriginColumn, OriginDiagnostic, OriginDiagnosticCode, OriginDiagnosticSeverity, OriginError, + OriginLimits, OriginMetadataEntry, OriginNote, OriginObjectLocation, OriginResourceUsage, + OriginUnsupportedObjectSummary, +}; + +use super::super::reader::{checked_add, checked_mul}; +use cursor::{MetadataBlock, MetadataCursor}; + +mod cursor; +mod tail; + +const BLOCK_PREFIX_LEN: usize = 5; +const WINDOW_NAME_OFFSET: usize = 2; +const WINDOW_NAME_WIDTH: usize = 25; +const WINDOW_HEADER_MIN_LEN: usize = WINDOW_NAME_OFFSET + WINDOW_NAME_WIDTH; +const AXIS_PARAMETER_LISTS: usize = 3; +const FORMATTED_F64_CAPACITY: usize = 32; + +pub(super) struct WindowInfo { + pub(super) name: Option, + pub(super) columns: Vec, +} + +pub(super) struct ParsedMetadata { + pub(super) windows: Vec, + pub(super) parameters: Vec, + pub(super) notes: Vec, + pub(super) diagnostics: Vec, + pub(super) unsupported_objects: Vec, +} + +pub(super) fn parse( + bytes: &[u8], + base_offset: usize, + limits: &OriginLimits, + usage: &mut OriginResourceUsage, +) -> Result { + let mut cursor = MetadataCursor::new(bytes, base_offset, limits); + let mut diagnostics = Vec::new(); + let mut unsupported_objects = Vec::new(); + + let (windows, presentation_records) = parse_windows(&mut cursor, usage, &mut diagnostics)?; + if presentation_records > 0 { + push_summary( + &mut unsupported_objects, + "window presentation records", + presentation_records, + limits, + usage, + )?; + push_diagnostic( + &mut diagnostics, + OriginDiagnosticCode::UnsupportedObjectSkipped, + "PlotX imported worksheet values but skipped bounded Origin window presentation records.", + None, + limits, + usage, + )?; + } + let parameters = parse_parameters(&mut cursor, usage, &mut diagnostics)?; + let (notes, note_properties) = parse_notes(&mut cursor, usage, &mut diagnostics)?; + if note_properties > 0 { + push_summary( + &mut unsupported_objects, + "note properties", + note_properties, + limits, + usage, + )?; + push_diagnostic( + &mut diagnostics, + OriginDiagnosticCode::MetadataSkipped, + "PlotX imported note names and text but skipped bounded Origin note properties.", + None, + limits, + usage, + )?; + } + + tail::parse( + &mut cursor, + &mut diagnostics, + &mut unsupported_objects, + usage, + )?; + usage.metadata_records = cursor.metadata_records(); + + Ok(ParsedMetadata { + windows, + parameters, + notes, + diagnostics, + unsupported_objects, + }) +} + +pub(super) fn parse_windows_only( + bytes: &[u8], + base_offset: usize, + limits: &OriginLimits, + usage: &mut OriginResourceUsage, +) -> Result { + let mut cursor = MetadataCursor::new(bytes, base_offset, limits); + let mut diagnostics = Vec::new(); + let mut unsupported_objects = Vec::new(); + let (windows, presentation_records) = parse_windows(&mut cursor, usage, &mut diagnostics)?; + + if presentation_records > 0 { + push_summary( + &mut unsupported_objects, + "window presentation records", + presentation_records, + limits, + usage, + )?; + push_diagnostic( + &mut diagnostics, + OriginDiagnosticCode::UnsupportedObjectSkipped, + "PlotX imported worksheet values but skipped bounded Origin window presentation records.", + None, + limits, + usage, + )?; + } + + // The Origin9V951 evidence establishes the complete window-list boundary, + // but not the object grammar that follows it. Stop at the consumed null + // terminator and report one opaque tail instead of scanning for markers. + if cursor.remaining() > 0 { + push_summary( + &mut unsupported_objects, + "remaining Origin 9.51 project objects", + 1, + limits, + usage, + )?; + push_diagnostic( + &mut diagnostics, + OriginDiagnosticCode::UnsupportedObjectSkipped, + "PlotX imported worksheet values but did not import the remaining Origin 9.51 project objects.", + Some(cursor.absolute_offset()?), + limits, + usage, + )?; + } + usage.metadata_records = cursor.metadata_records(); + + Ok(ParsedMetadata { + windows, + parameters: Vec::new(), + notes: Vec::new(), + diagnostics, + unsupported_objects, + }) +} + +fn parse_windows( + cursor: &mut MetadataCursor<'_>, + usage: &mut OriginResourceUsage, + diagnostics: &mut Vec, +) -> Result<(Vec, usize), OriginError> { + enforce_depth(1, cursor.limits)?; + let mut windows = Vec::new(); + let mut presentation_records = 0_usize; + + loop { + let header = match cursor.read_block()? { + MetadataBlock::Null { .. } => break, + MetadataBlock::Data { offset, payload } => (offset, payload), + }; + cursor.charge_record()?; + let window_count = checked_add(windows.len(), 1, "window records")?; + enforce_limit( + "window records", + window_count, + cursor.limits.max_window_records, + )?; + + // This exact header-plus-layer-list traversal is reimplemented from + // the pinned MIT OpenOPJ Origin 7.0552 WindowList description: + // https://github.com/jgonera/openopj/blob/42ddcf1eb3a490744c54fca0a4ed6fe7a5e723ca/lib/OpenOPJ/WindowList.php + let name = match decode_window_name(header.1, header.0, cursor.limits, usage) { + Ok(name) if !name.is_empty() => Some(name), + Ok(_) => { + push_diagnostic( + diagnostics, + OriginDiagnosticCode::MetadataSkipped, + "PlotX skipped an empty Origin window name after validating its boundaries.", + Some(header.0), + cursor.limits, + usage, + )?; + None + } + Err(OriginError::UnsupportedEncoding { .. }) => { + push_diagnostic( + diagnostics, + OriginDiagnosticCode::MetadataSkipped, + "PlotX skipped a non-ASCII Origin window name after validating its boundaries.", + Some(header.0), + cursor.limits, + usage, + )?; + None + } + Err(error) => return Err(error), + }; + + presentation_records = checked_add( + presentation_records, + walk_layer_list(cursor, 2)?, + "Origin window presentation records", + )?; + try_reserve( + &mut windows, + 1, + "Origin window records", + cursor.limits, + usage, + )?; + windows.push(WindowInfo { + name, + columns: Vec::new(), + }); + } + Ok((windows, presentation_records)) +} + +fn walk_layer_list(cursor: &mut MetadataCursor<'_>, depth: usize) -> Result { + enforce_depth(depth, cursor.limits)?; + let mut records = 0_usize; + loop { + match cursor.read_block()? { + MetadataBlock::Null { .. } => break, + MetadataBlock::Data { .. } => cursor.charge_record()?, + } + records = checked_add(records, 1, "Origin window presentation records")?; + + records = checked_add( + records, + walk_fixed_block_list(cursor, checked_add(depth, 1, "metadata nesting depth")?, 4)?, + "Origin window presentation records", + )?; + records = checked_add( + records, + walk_fixed_block_list(cursor, checked_add(depth, 1, "metadata nesting depth")?, 2)?, + "Origin window presentation records", + )?; + records = checked_add( + records, + walk_fixed_block_list(cursor, checked_add(depth, 1, "metadata nesting depth")?, 1)?, + "Origin window presentation records", + )?; + for _ in 0..AXIS_PARAMETER_LISTS { + records = checked_add( + records, + walk_fixed_block_list(cursor, checked_add(depth, 1, "metadata nesting depth")?, 1)?, + "Origin window presentation records", + )?; + } + } + Ok(records) +} + +fn walk_fixed_block_list( + cursor: &mut MetadataCursor<'_>, + depth: usize, + blocks_per_item: usize, +) -> Result { + enforce_depth(depth, cursor.limits)?; + let mut items = 0_usize; + loop { + match cursor.read_block()? { + MetadataBlock::Null { .. } => break, + MetadataBlock::Data { .. } => cursor.charge_record()?, + } + items = checked_add(items, 1, "Origin nested records")?; + for _ in 1..blocks_per_item { + let _ = cursor.read_block()?; + } + } + Ok(items) +} + +fn parse_parameters( + cursor: &mut MetadataCursor<'_>, + usage: &mut OriginResourceUsage, + diagnostics: &mut Vec, +) -> Result, OriginError> { + // Origin7V552 parameters use an LF-terminated name, one little-endian f64 + // plus LF, and a NUL-name terminator as described by pinned MIT OpenOPJ: + // https://github.com/jgonera/openopj/blob/42ddcf1eb3a490744c54fca0a4ed6fe7a5e723ca/lib/OpenOPJ/ParametersSection.php + let mut parameters = Vec::new(); + loop { + let (name_offset, name_bytes) = cursor.read_line()?; + if name_bytes == [0] { + break; + } + cursor.charge_record()?; + if name_bytes.is_empty() { + return Err(OriginError::CorruptStructure { + offset: name_offset, + detail: "an Origin parameter name cannot be empty".to_owned(), + }); + } + + let value_offset = cursor.absolute_offset()?; + let value_bytes = cursor.read_parameter_value()?; + let value = f64::from_le_bytes(value_bytes); + + let name = match validate_ascii(name_bytes, name_offset, "Origin parameter name") { + Ok(name) => name, + Err(OriginError::UnsupportedEncoding { .. }) => { + push_diagnostic( + diagnostics, + OriginDiagnosticCode::MetadataSkipped, + "PlotX skipped a non-ASCII Origin parameter after validating its value boundary.", + Some(name_offset), + cursor.limits, + usage, + )?; + continue; + } + Err(error) => return Err(error), + }; + if !value.is_finite() { + push_diagnostic( + diagnostics, + OriginDiagnosticCode::MetadataSkipped, + "PlotX skipped a non-finite Origin parameter value.", + Some(value_offset), + cursor.limits, + usage, + )?; + continue; + } + + let key = copy_decoded_text(name, cursor.limits, usage)?; + let formatted = format_f64(value, value_offset)?; + let value = copy_parser_text(formatted, cursor.limits, usage)?; + try_reserve( + &mut parameters, + 1, + "Origin project parameters", + cursor.limits, + usage, + )?; + parameters.push(OriginMetadataEntry { key, value }); + } + + require_null_block( + cursor.read_block()?, + "the Origin parameter section must end with a null block", + )?; + Ok(parameters) +} + +fn parse_notes( + cursor: &mut MetadataCursor<'_>, + usage: &mut OriginResourceUsage, + diagnostics: &mut Vec, +) -> Result<(Vec, usize), OriginError> { + // Each note is exactly header/name/content framed blocks. PlotX retains + // only bounded ASCII name/content and reports the opaque header properties: + // https://github.com/jgonera/openopj/blob/42ddcf1eb3a490744c54fca0a4ed6fe7a5e723ca/lib/OpenOPJ/NoteSection.php + let mut notes = Vec::new(); + let mut note_properties = 0_usize; + loop { + let header_offset = match cursor.read_block()? { + MetadataBlock::Null { .. } => break, + MetadataBlock::Data { offset, .. } => offset, + }; + cursor.charge_record()?; + note_properties = checked_add(note_properties, 1, "Origin note properties")?; + let (name_offset, name_block) = require_data_block( + cursor.read_block()?, + "an Origin note name must be a data block", + )?; + let (content_offset, content_block) = require_data_block( + cursor.read_block()?, + "an Origin note content must be a data block", + )?; + + let name = validate_nul_terminated_ascii( + name_block, + name_offset, + cursor.limits, + "Origin note name", + ); + let content = validate_nul_terminated_ascii( + content_block, + content_offset, + cursor.limits, + "Origin note content", + ); + let (name, content) = match (name, content) { + (Ok(name), Ok(content)) => (name, content), + (Err(OriginError::UnsupportedEncoding { .. }), _) + | (_, Err(OriginError::UnsupportedEncoding { .. })) => { + push_diagnostic( + diagnostics, + OriginDiagnosticCode::MetadataSkipped, + "PlotX skipped a non-ASCII Origin note after validating its block boundaries.", + Some(header_offset), + cursor.limits, + usage, + )?; + continue; + } + (Err(error), _) | (_, Err(error)) => return Err(error), + }; + + let name = copy_decoded_text(name, cursor.limits, usage)?; + let content = copy_decoded_text(content, cursor.limits, usage)?; + try_reserve(&mut notes, 1, "Origin project notes", cursor.limits, usage)?; + notes.push(OriginNote { name, content }); + } + Ok((notes, note_properties)) +} + +fn decode_window_name( + header: &[u8], + block_offset: usize, + limits: &OriginLimits, + usage: &mut OriginResourceUsage, +) -> Result { + if header.len() < WINDOW_HEADER_MIN_LEN { + return Err(OriginError::CorruptStructure { + offset: block_offset, + detail: format!( + "an Origin7V552 window header must contain at least {WINDOW_HEADER_MIN_LEN} bytes" + ), + }); + } + let end = checked_add( + WINDOW_NAME_OFFSET, + WINDOW_NAME_WIDTH, + "Origin window name range", + )?; + let field = header + .get(WINDOW_NAME_OFFSET..end) + .ok_or(OriginError::Truncated { + offset: block_offset, + needed: WINDOW_HEADER_MIN_LEN, + have: header.len(), + })?; + let length = field + .iter() + .position(|byte| *byte == 0) + .unwrap_or(field.len()); + let text = field.get(..length).ok_or(OriginError::ArithmeticOverflow { + resource: "Origin window name", + })?; + let text_offset = checked_add( + block_offset, + checked_add( + BLOCK_PREFIX_LEN, + WINDOW_NAME_OFFSET, + "Origin window name offset", + )?, + "Origin window name offset", + )?; + let text = validate_ascii(text, text_offset, "Origin window name")?; + copy_decoded_text(text, limits, usage) +} + +fn validate_nul_terminated_ascii<'a>( + bytes: &'a [u8], + offset: usize, + limits: &OriginLimits, + field: &'static str, +) -> Result<&'a str, OriginError> { + let Some(text) = bytes.strip_suffix(&[0]) else { + return Err(OriginError::CorruptStructure { + offset, + detail: format!("{field} must end with a NUL byte inside its framed block"), + }); + }; + if let Some(relative) = text.iter().position(|byte| *byte == 0) { + return Err(OriginError::CorruptStructure { + offset: checked_add(offset, relative, "embedded metadata NUL offset")?, + detail: format!("{field} contains an embedded NUL byte"), + }); + } + enforce_limit("string bytes", text.len(), limits.max_string_bytes)?; + validate_ascii(text, offset, field) +} + +fn validate_ascii<'a>( + bytes: &'a [u8], + offset: usize, + field: &'static str, +) -> Result<&'a str, OriginError> { + if let Some(relative) = bytes.iter().position(|byte| !byte.is_ascii()) { + return Err(OriginError::UnsupportedEncoding { + offset: checked_add(offset, relative, "metadata ASCII offset")?, + encoding: format!("non-ASCII byte in {field}"), + }); + } + std::str::from_utf8(bytes).map_err(|_| OriginError::UnsupportedEncoding { + offset, + encoding: format!("non-ASCII byte in {field}"), + }) +} + +fn format_f64(value: f64, offset: usize) -> Result { + let mut output = FixedText::default(); + write!(&mut output, "{value}").map_err(|_| OriginError::CorruptStructure { + offset, + detail: "an Origin parameter value could not be represented as bounded text".to_owned(), + })?; + Ok(output) +} + +#[derive(Default)] +struct FixedText { + bytes: [u8; FORMATTED_F64_CAPACITY], + len: usize, +} + +impl FixedText { + fn as_str(&self) -> Result<&str, OriginError> { + std::str::from_utf8( + self.bytes + .get(..self.len) + .ok_or(OriginError::ArithmeticOverflow { + resource: "formatted Origin parameter", + })?, + ) + .map_err(|_| OriginError::CorruptStructure { + offset: 0, + detail: "formatted Origin parameter text is not ASCII".to_owned(), + }) + } +} + +impl fmt::Write for FixedText { + fn write_str(&mut self, text: &str) -> fmt::Result { + let end = self.len.checked_add(text.len()).ok_or(fmt::Error)?; + let destination = self.bytes.get_mut(self.len..end).ok_or(fmt::Error)?; + destination.copy_from_slice(text.as_bytes()); + self.len = end; + Ok(()) + } +} + +fn copy_decoded_text( + text: &str, + limits: &OriginLimits, + usage: &mut OriginResourceUsage, +) -> Result { + enforce_limit("string bytes", text.len(), limits.max_string_bytes)?; + charge_text(text.len(), limits, usage)?; + copy_after_charge(text, "decoded Origin metadata") +} + +fn copy_parser_text( + text: FixedText, + limits: &OriginLimits, + usage: &mut OriginResourceUsage, +) -> Result { + let text = text.as_str()?; + enforce_limit("string bytes", text.len(), limits.max_string_bytes)?; + charge_parser(text.len(), limits, usage)?; + copy_after_charge(text, "formatted Origin parameter") +} + +fn copy_static_text( + text: &'static str, + limits: &OriginLimits, + usage: &mut OriginResourceUsage, +) -> Result { + charge_parser(text.len(), limits, usage)?; + copy_after_charge(text, "Origin diagnostic text") +} + +pub(super) fn copy_generated_text( + text: &'static str, + limits: &OriginLimits, + usage: &mut OriginResourceUsage, +) -> Result { + copy_static_text(text, limits, usage) +} + +fn copy_after_charge(text: &str, resource: &'static str) -> Result { + let mut output = String::new(); + output + .try_reserve_exact(text.len()) + .map_err(|_| OriginError::AllocationFailed { + resource, + requested: text.len(), + })?; + output.push_str(text); + Ok(output) +} + +pub(super) fn try_reserve( + values: &mut Vec, + additional: usize, + resource: &'static str, + limits: &OriginLimits, + usage: &mut OriginResourceUsage, +) -> Result<(), OriginError> { + let requested_len = checked_add(values.len(), additional, resource)?; + let old_capacity = values.capacity(); + if requested_len <= old_capacity || size_of::() == 0 { + return Ok(()); + } + + let minimum_delta = requested_len + .checked_sub(old_capacity) + .ok_or(OriginError::ArithmeticOverflow { resource })?; + let minimum_bytes = checked_mul(minimum_delta, size_of::(), resource)?; + let mut preflight = usage.clone(); + charge_parser(minimum_bytes, limits, &mut preflight)?; + + let available_bytes = limits + .max_parser_bytes + .saturating_sub(usage.parser_bytes) + .min( + limits + .max_total_owned_bytes + .saturating_sub(usage.total_owned_bytes), + ); + let geometric_capacity = if old_capacity == 0 { + requested_len + } else { + old_capacity.checked_mul(2).unwrap_or(requested_len) + }; + let desired_capacity = requested_len.max(geometric_capacity); + let desired_delta = desired_capacity + .checked_sub(old_capacity) + .ok_or(OriginError::ArithmeticOverflow { resource })?; + let affordable_delta = (available_bytes / size_of::()).min(desired_delta); + let target_capacity = checked_add(old_capacity, affordable_delta, resource)?; + let reserve_additional = target_capacity + .checked_sub(values.len()) + .ok_or(OriginError::ArithmeticOverflow { resource })?; + let planned_delta = target_capacity + .checked_sub(old_capacity) + .ok_or(OriginError::ArithmeticOverflow { resource })?; + let planned_bytes = checked_mul(planned_delta, size_of::(), resource)?; + values + .try_reserve_exact(reserve_additional) + .map_err(|_| OriginError::AllocationFailed { + resource, + requested: planned_bytes, + })?; + + let actual_delta = values + .capacity() + .checked_sub(old_capacity) + .ok_or(OriginError::ArithmeticOverflow { resource })?; + let actual_bytes = checked_mul(actual_delta, size_of::(), resource)?; + charge_parser(actual_bytes, limits, usage) +} + +pub(super) fn push_diagnostic( + diagnostics: &mut Vec, + code: OriginDiagnosticCode, + message: &'static str, + byte_offset: Option, + limits: &OriginLimits, + usage: &mut OriginResourceUsage, +) -> Result<(), OriginError> { + let message = copy_static_text(message, limits, usage)?; + try_reserve(diagnostics, 1, "Origin diagnostics", limits, usage)?; + diagnostics.push(OriginDiagnostic { + code, + severity: OriginDiagnosticSeverity::Warning, + location: byte_offset.map(|byte_offset| OriginObjectLocation { + workbook: None, + worksheet: None, + column: None, + byte_offset: Some(byte_offset), + }), + message, + }); + Ok(()) +} + +pub(super) fn push_summary( + summaries: &mut Vec, + kind: &'static str, + count: usize, + limits: &OriginLimits, + usage: &mut OriginResourceUsage, +) -> Result<(), OriginError> { + let kind = copy_static_text(kind, limits, usage)?; + try_reserve( + summaries, + 1, + "Origin unsupported-object summaries", + limits, + usage, + )?; + summaries.push(OriginUnsupportedObjectSummary { kind, count }); + Ok(()) +} + +fn charge_text( + bytes: usize, + limits: &OriginLimits, + usage: &mut OriginResourceUsage, +) -> Result<(), OriginError> { + let decoded = checked_add(usage.decoded_text_bytes, bytes, "decoded text bytes")?; + enforce_limit("decoded text bytes", decoded, limits.max_decoded_text_bytes)?; + charge_parser(bytes, limits, usage)?; + usage.decoded_text_bytes = decoded; + Ok(()) +} + +fn charge_parser( + bytes: usize, + limits: &OriginLimits, + usage: &mut OriginResourceUsage, +) -> Result<(), OriginError> { + let parser = checked_add(usage.parser_bytes, bytes, "parser bytes")?; + enforce_limit("parser bytes", parser, limits.max_parser_bytes)?; + let total = checked_add(usage.total_owned_bytes, bytes, "total owned bytes")?; + enforce_limit("total owned bytes", total, limits.max_total_owned_bytes)?; + usage.parser_bytes = parser; + usage.total_owned_bytes = total; + Ok(()) +} + +fn enforce_depth(depth: usize, limits: &OriginLimits) -> Result<(), OriginError> { + enforce_limit("metadata nesting depth", depth, limits.max_metadata_depth) +} + +fn enforce_limit(resource: &'static str, actual: usize, limit: usize) -> Result<(), OriginError> { + if actual > limit { + return Err(OriginError::LimitExceeded { + resource, + limit, + actual, + }); + } + Ok(()) +} + +fn require_data_block<'a>( + block: MetadataBlock<'a>, + detail: &'static str, +) -> Result<(usize, &'a [u8]), OriginError> { + match block { + MetadataBlock::Data { offset, payload } => Ok((offset, payload)), + MetadataBlock::Null { offset } => Err(OriginError::CorruptStructure { + offset, + detail: detail.to_owned(), + }), + } +} + +fn require_null_block(block: MetadataBlock<'_>, detail: &'static str) -> Result<(), OriginError> { + match block { + MetadataBlock::Null { .. } => Ok(()), + MetadataBlock::Data { offset, .. } => Err(OriginError::CorruptStructure { + offset, + detail: detail.to_owned(), + }), + } +} + +#[cfg(test)] +#[path = "metadata_tests.rs"] +mod metadata_tests; + +#[cfg(test)] +#[path = "metadata_modern_tests.rs"] +mod metadata_modern_tests; diff --git a/crates/io/src/origin/opj/metadata/cursor.rs b/crates/io/src/origin/opj/metadata/cursor.rs new file mode 100644 index 0000000..edac826 --- /dev/null +++ b/crates/io/src/origin/opj/metadata/cursor.rs @@ -0,0 +1,226 @@ +use std::mem::size_of; + +use crate::origin::{OriginError, OriginLimits}; + +use super::super::super::reader::checked_add; + +const LF: u8 = b'\n'; +const PARAMETER_VALUE_LEN: usize = size_of::() + 1; + +pub(super) enum MetadataBlock<'a> { + Null { offset: usize }, + Data { offset: usize, payload: &'a [u8] }, +} + +pub(super) struct MetadataCursor<'a> { + bytes: &'a [u8], + offset: usize, + base_offset: usize, + metadata_records: usize, + pub(super) limits: &'a OriginLimits, +} + +impl<'a> MetadataCursor<'a> { + pub(super) fn new(bytes: &'a [u8], base_offset: usize, limits: &'a OriginLimits) -> Self { + Self { + bytes, + offset: 0, + base_offset, + metadata_records: 0, + limits, + } + } + + pub(super) fn charge_record(&mut self) -> Result<(), OriginError> { + let actual = checked_add(self.metadata_records, 1, "metadata records")?; + enforce_limit("metadata records", actual, self.limits.max_metadata_records)?; + self.metadata_records = actual; + Ok(()) + } + + pub(super) fn metadata_records(&self) -> usize { + self.metadata_records + } + + pub(super) fn read_block(&mut self) -> Result, OriginError> { + let start = self.offset; + let size_end = checked_add(start, size_of::(), "metadata block size")?; + let size_bytes = self.bytes.get(start..size_end).ok_or_else(|| { + self.truncated( + start, + size_of::(), + self.bytes.len().saturating_sub(start), + ) + })?; + let size_array: [u8; 4] = size_bytes + .try_into() + .map_err(|_| self.truncated(start, size_of::(), size_bytes.len()))?; + let size = usize::try_from(u32::from_le_bytes(size_array)).map_err(|_| { + OriginError::ArithmeticOverflow { + resource: "metadata block size", + } + })?; + enforce_limit("block bytes", size, self.limits.max_block_bytes)?; + + let size_lf = size_end; + self.require_lf(size_lf, "metadata block size delimiter")?; + let payload_start = checked_add(size_lf, 1, "metadata block payload")?; + if size == 0 { + self.offset = payload_start; + return Ok(MetadataBlock::Null { + offset: self.absolute(start)?, + }); + } + + let payload_end = checked_add(payload_start, size, "metadata block payload")?; + let payload = self.bytes.get(payload_start..payload_end).ok_or_else(|| { + self.truncated( + payload_start, + size, + self.bytes.len().saturating_sub(payload_start), + ) + })?; + self.require_lf(payload_end, "metadata block payload delimiter")?; + self.offset = checked_add(payload_end, 1, "metadata block end")?; + Ok(MetadataBlock::Data { + offset: self.absolute(start)?, + payload, + }) + } + + pub(super) fn read_line(&mut self) -> Result<(usize, &'a [u8]), OriginError> { + let start = self.offset; + let available = self + .bytes + .get(start..) + .ok_or_else(|| self.truncated(start, 1, self.bytes.len().saturating_sub(start)))?; + let oversize = checked_add(self.limits.max_string_bytes, 1, "metadata line bound")?; + let scan_len = available.len().min(oversize); + let scan = available + .get(..scan_len) + .ok_or(OriginError::ArithmeticOverflow { + resource: "metadata line scan", + })?; + let Some(relative_lf) = scan.iter().position(|byte| *byte == LF) else { + if available.len() > self.limits.max_string_bytes { + return Err(OriginError::LimitExceeded { + resource: "string bytes", + limit: self.limits.max_string_bytes, + actual: oversize, + }); + } + return Err(self.truncated( + checked_add(start, available.len(), "metadata line end")?, + 1, + 0, + )); + }; + enforce_limit("string bytes", relative_lf, self.limits.max_string_bytes)?; + let line = available + .get(..relative_lf) + .ok_or(OriginError::ArithmeticOverflow { + resource: "metadata line", + })?; + self.offset = checked_add( + start, + checked_add(relative_lf, 1, "metadata line length")?, + "metadata line end", + )?; + Ok((self.absolute(start)?, line)) + } + + pub(super) fn read_parameter_value(&mut self) -> Result<[u8; 8], OriginError> { + let start = self.offset; + let end = checked_add(start, PARAMETER_VALUE_LEN, "parameter value")?; + let bytes = self.bytes.get(start..end).ok_or_else(|| { + self.truncated( + start, + PARAMETER_VALUE_LEN, + self.bytes.len().saturating_sub(start), + ) + })?; + let value = bytes + .get(..size_of::()) + .ok_or_else(|| self.truncated(start, size_of::(), bytes.len()))?; + let delimiter = bytes + .get(size_of::()) + .copied() + .ok_or_else(|| self.truncated(start, PARAMETER_VALUE_LEN, bytes.len()))?; + if delimiter != LF { + return Err(OriginError::CorruptStructure { + offset: self.absolute(checked_add(start, size_of::(), "parameter LF")?)?, + detail: "an Origin parameter value must end with LF".to_owned(), + }); + } + let value: [u8; 8] = value + .try_into() + .map_err(|_| self.truncated(start, size_of::(), value.len()))?; + self.offset = end; + Ok(value) + } + + pub(super) fn read_exact(&mut self, length: usize) -> Result<&'a [u8], OriginError> { + let start = self.offset; + let end = checked_add(start, length, "terminal OPJ record")?; + let bytes = self + .bytes + .get(start..end) + .ok_or_else(|| self.truncated(start, length, self.bytes.len().saturating_sub(start)))?; + self.offset = end; + Ok(bytes) + } + + pub(super) fn skip_exact(&mut self, length: usize) -> Result<(), OriginError> { + let _ = self.read_exact(length)?; + Ok(()) + } + + pub(super) fn relative_offset(&self) -> usize { + self.offset + } + + pub(super) fn absolute_offset(&self) -> Result { + self.absolute(self.offset) + } + + pub(super) fn remaining(&self) -> usize { + self.bytes.len().saturating_sub(self.offset) + } + + fn require_lf(&self, offset: usize, field: &'static str) -> Result<(), OriginError> { + let byte = + self.bytes.get(offset).copied().ok_or_else(|| { + self.truncated(offset, 1, self.bytes.len().saturating_sub(offset)) + })?; + if byte != LF { + return Err(OriginError::CorruptStructure { + offset: self.absolute(offset)?, + detail: format!("{field} must be LF"), + }); + } + Ok(()) + } + + fn absolute(&self, relative: usize) -> Result { + checked_add(self.base_offset, relative, "metadata file offset") + } + + fn truncated(&self, offset: usize, needed: usize, have: usize) -> OriginError { + OriginError::Truncated { + offset: self.base_offset.saturating_add(offset), + needed, + have, + } + } +} + +fn enforce_limit(resource: &'static str, actual: usize, limit: usize) -> Result<(), OriginError> { + if actual > limit { + return Err(OriginError::LimitExceeded { + resource, + limit, + actual, + }); + } + Ok(()) +} diff --git a/crates/io/src/origin/opj/metadata/tail.rs b/crates/io/src/origin/opj/metadata/tail.rs new file mode 100644 index 0000000..0a80002 --- /dev/null +++ b/crates/io/src/origin/opj/metadata/tail.rs @@ -0,0 +1,212 @@ +use std::mem::size_of; + +use crate::origin::{ + OriginDiagnostic, OriginDiagnosticCode, OriginError, OriginResourceUsage, + OriginUnsupportedObjectSummary, +}; + +use super::cursor::{MetadataBlock, MetadataCursor}; +use super::{push_diagnostic, push_summary}; +use crate::origin::reader::checked_add; + +const TREE_SPAN_PAYLOAD_LEN: usize = size_of::(); + +// The unchanged public OpenOPJ MIT fixture has its first attachment header at +// absolute byte offset 276,350. Its bytes encode a 52-byte header, type +// 0x7fca0459, and an OLE compound signature immediately after that header. +// These Origin7V552 constants are derived directly from those fixture bytes; +// the pinned source and license are recorded in the fixture README. +const ATTACHMENT_HEADER_LEN: usize = 52; +const ATTACHMENT_TYPE_OLE: usize = 0x7fca_0459; +const OLE_COMPOUND_SIGNATURE: &[u8] = &[0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1]; + +pub(super) fn parse( + cursor: &mut MetadataCursor<'_>, + diagnostics: &mut Vec, + unsupported_objects: &mut Vec, + usage: &mut OriginResourceUsage, +) -> Result<(), OriginError> { + parse_project_tree(cursor)?; + push_summary(unsupported_objects, "project tree", 1, cursor.limits, usage)?; + push_diagnostic( + diagnostics, + OriginDiagnosticCode::UnsupportedObjectSkipped, + "PlotX preserved worksheet data but did not import the bounded Origin project tree.", + None, + cursor.limits, + usage, + )?; + + let attachment_count = parse_attachments(cursor)?; + if attachment_count > 0 { + push_summary( + unsupported_objects, + "embedded attachments", + attachment_count, + cursor.limits, + usage, + )?; + push_diagnostic( + diagnostics, + OriginDiagnosticCode::UnsupportedObjectSkipped, + "PlotX did not extract, open, or execute bounded embedded Origin attachments.", + None, + cursor.limits, + usage, + )?; + } + Ok(()) +} + +fn parse_project_tree(cursor: &mut MetadataCursor<'_>) -> Result<(), OriginError> { + let tree_start = cursor.relative_offset(); + let (block_offset, span_payload) = match cursor.read_block()? { + MetadataBlock::Data { offset, payload } => (offset, payload), + MetadataBlock::Null { offset } => { + return Err(OriginError::CorruptStructure { + offset, + detail: "the Origin project tree requires a bounded span block".to_owned(), + }); + } + }; + cursor.charge_record()?; + if span_payload.len() != TREE_SPAN_PAYLOAD_LEN { + return Err(OriginError::CorruptStructure { + offset: block_offset, + detail: "the Origin7V552 project-tree span must be a 4-byte value".to_owned(), + }); + } + let span_bytes: [u8; 4] = + span_payload + .try_into() + .map_err(|_| OriginError::CorruptStructure { + offset: block_offset, + detail: "the Origin7V552 project-tree span is incomplete".to_owned(), + })?; + let declared_span = usize::try_from(u32::from_le_bytes(span_bytes)).map_err(|_| { + OriginError::ArithmeticOverflow { + resource: "Origin project-tree span", + } + })?; + enforce_limit("block bytes", declared_span, cursor.limits.max_block_bytes)?; + let consumed = cursor.relative_offset().checked_sub(tree_start).ok_or( + OriginError::ArithmeticOverflow { + resource: "Origin project-tree span", + }, + )?; + let remaining_span = + declared_span + .checked_sub(consumed) + .ok_or_else(|| OriginError::CorruptStructure { + offset: block_offset, + detail: "the Origin project-tree span is shorter than its framing".to_owned(), + })?; + + // OpenOPJ documents the project tree as the section following notes. The + // public MIT fixture independently supplies this outer span, so PlotX can + // skip exactly to the attachment header without interpreting tree names, + // paths, or executable content. + // https://github.com/jgonera/openopj/blob/42ddcf1eb3a490744c54fca0a4ed6fe7a5e723ca/docs/opj_format.markdown + cursor.skip_exact(remaining_span) +} + +fn parse_attachments(cursor: &mut MetadataCursor<'_>) -> Result { + let mut count = 0_usize; + while cursor.remaining() > 0 { + cursor.charge_record()?; + let header_offset = cursor.absolute_offset()?; + let header = cursor.read_exact(ATTACHMENT_HEADER_LEN)?; + let header_len = read_u32(header, 0, header_offset, "attachment header size")?; + if header_len != ATTACHMENT_HEADER_LEN { + return Err(OriginError::UnsupportedFeature { + feature: "an Origin attachment header layout is not verified".to_owned(), + }); + } + let attachment_type = read_u32(header, 4, header_offset, "attachment type")?; + if attachment_type != ATTACHMENT_TYPE_OLE { + return Err(OriginError::UnsupportedFeature { + feature: "an embedded Origin attachment type is not supported".to_owned(), + }); + } + let payload_size = read_u32(header, 8, header_offset, "attachment payload size")?; + enforce_limit("block bytes", payload_size, cursor.limits.max_block_bytes)?; + let payload = cursor.read_exact(payload_size)?; + if !payload.starts_with(OLE_COMPOUND_SIGNATURE) { + return Err(OriginError::UnsupportedFeature { + feature: "an embedded Origin attachment payload is not a verified OLE object" + .to_owned(), + }); + } + count = checked_add(count, 1, "embedded Origin attachments")?; + } + Ok(count) +} + +fn read_u32( + bytes: &[u8], + offset: usize, + file_offset: usize, + resource: &'static str, +) -> Result { + let end = checked_add(offset, size_of::(), resource)?; + let value = bytes.get(offset..end).ok_or(OriginError::Truncated { + offset: checked_add(file_offset, offset, resource)?, + needed: size_of::(), + have: bytes.len().saturating_sub(offset), + })?; + let value_offset = checked_add(file_offset, offset, resource)?; + let value: [u8; 4] = value.try_into().map_err(|_| OriginError::Truncated { + offset: value_offset, + needed: size_of::(), + have: value.len(), + })?; + usize::try_from(u32::from_le_bytes(value)) + .map_err(|_| OriginError::ArithmeticOverflow { resource }) +} + +fn enforce_limit(resource: &'static str, actual: usize, limit: usize) -> Result<(), OriginError> { + if actual > limit { + return Err(OriginError::LimitExceeded { + resource, + limit, + actual, + }); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::{ATTACHMENT_HEADER_LEN, ATTACHMENT_TYPE_OLE, OLE_COMPOUND_SIGNATURE}; + + const OPENOPJ_FIXTURE: &[u8] = + include_bytes!("../../../../tests/fixtures/origin/test-origin-7.0552.opj"); + const ATTACHMENT_HEADER_OFFSET: usize = 276_350; + + #[test] + fn attachment_constants_match_the_pinned_openopj_fixture_bytes() { + let header_end = ATTACHMENT_HEADER_OFFSET + .checked_add(ATTACHMENT_HEADER_LEN) + .expect("fixture offsets are small"); + let header = OPENOPJ_FIXTURE + .get(ATTACHMENT_HEADER_OFFSET..header_end) + .expect("pinned fixture contains the documented attachment header"); + + let header_len_bytes = u32::try_from(ATTACHMENT_HEADER_LEN) + .expect("attachment header length fits u32") + .to_le_bytes(); + let attachment_type_bytes = u32::try_from(ATTACHMENT_TYPE_OLE) + .expect("attachment type fits u32") + .to_le_bytes(); + assert_eq!(header.get(0..4), Some(header_len_bytes.as_slice())); + assert_eq!(header.get(4..8), Some(attachment_type_bytes.as_slice())); + + let signature_end = header_end + .checked_add(OLE_COMPOUND_SIGNATURE.len()) + .expect("fixture offsets are small"); + assert_eq!( + OPENOPJ_FIXTURE.get(header_end..signature_end), + Some(OLE_COMPOUND_SIGNATURE) + ); + } +} diff --git a/crates/io/src/origin/opj/metadata_modern_tests.rs b/crates/io/src/origin/opj/metadata_modern_tests.rs new file mode 100644 index 0000000..7756828 --- /dev/null +++ b/crates/io/src/origin/opj/metadata_modern_tests.rs @@ -0,0 +1,191 @@ +use std::panic::catch_unwind; + +use crate::origin::{OriginCell, OriginError, OriginLimits, read_origin}; + +const SIGNATURE: &[u8] = b"CPYA 4.3268 195 W64 #\n"; +const GLOBAL_HEADER_LEN: usize = 115; +const DATA_HEADER_LEN: usize = 147; +const EMPTY_F64: f64 = -1.23456789E-300; + +fn push_block(bytes: &mut Vec, payload: Option<&[u8]>) { + let payload = payload.unwrap_or_default(); + bytes.extend_from_slice(&u32::try_from(payload.len()).unwrap().to_le_bytes()); + bytes.push(b'\n'); + if !payload.is_empty() { + bytes.extend_from_slice(payload); + bytes.push(b'\n'); + } +} + +fn modern_header(name: &str, row_count: u32) -> [u8; DATA_HEADER_LEN] { + let mut header = [0_u8; DATA_HEADER_LEN]; + header[0x16..0x18].copy_from_slice(&0x6121_u16.to_le_bytes()); + header[0x18] = 0x03; + header[0x19..0x1d].copy_from_slice(&row_count.to_le_bytes()); + header[0x1d..0x21].copy_from_slice(&0_u32.to_le_bytes()); + header[0x21..0x25].copy_from_slice(&row_count.to_le_bytes()); + header[0x3d] = 10; + header[0x3f] = 0x10; + header[0x58..0x58 + name.len()].copy_from_slice(name.as_bytes()); + header[0x71..0x73].copy_from_slice(&0x10ca_u16.to_le_bytes()); + header +} + +fn numeric_slots(values: &[f64]) -> Vec { + values + .iter() + .flat_map(|value| [0, 0].into_iter().chain(value.to_le_bytes())) + .collect() +} + +fn push_window(bytes: &mut Vec, name: &str) { + let mut header = [0_u8; 27]; + header[2..2 + name.len()].copy_from_slice(name.as_bytes()); + push_block(bytes, Some(&header)); + push_block(bytes, None); +} + +struct ModernProject { + bytes: Vec, + window_list_end: usize, + final_window_terminator_start: usize, +} + +fn modern_project(records: &[(&str, &[f64])], windows: &[&str]) -> ModernProject { + let mut bytes = SIGNATURE.to_vec(); + let mut global = [0_u8; GLOBAL_HEADER_LEN]; + global[0x1b..0x23].copy_from_slice(&9.510195_f64.to_le_bytes()); + push_block(&mut bytes, Some(&global)); + push_block(&mut bytes, None); + + for (name, values) in records { + push_block( + &mut bytes, + Some(&modern_header(name, u32::try_from(values.len()).unwrap())), + ); + push_block(&mut bytes, Some(&numeric_slots(values))); + push_block(&mut bytes, None); + } + push_block(&mut bytes, None); + + for name in windows { + push_window(&mut bytes, name); + } + let final_window_terminator_start = bytes.len(); + push_block(&mut bytes, None); + let window_list_end = bytes.len(); + + // This payload deliberately resembles a dataset name. A windows-only + // parser must report and ignore it rather than scan it for table markers. + bytes.extend_from_slice(b"opaque Book1_FAKE project tail\0\xff"); + ModernProject { + bytes, + window_list_end, + final_window_terminator_start, + } +} + +#[test] +fn modern_assembles_worksheets_and_reports_the_opaque_tail() { + let fixture = modern_project( + &[("Book1_A", &[0.05, 0.10]), ("Book1_B", &[1.5, 2.5])], + &["Book1", "Graph1"], + ); + let project = read_origin(&fixture.bytes, OriginLimits::default()).unwrap(); + + assert_eq!(project.workbooks.len(), 1); + let worksheet = &project.workbooks[0].worksheets[0]; + assert_eq!(project.workbooks[0].name, "Book1"); + assert_eq!(worksheet.row_count, 2); + assert_eq!(worksheet.columns.len(), 2); + assert_eq!(worksheet.columns[0].name, "A"); + assert_eq!(worksheet.columns[1].name, "B"); + assert_eq!( + worksheet.columns[0].cells, + [OriginCell::Float(0.05), OriginCell::Float(0.10)] + ); + assert!(project.diagnostics.iter().any(|diagnostic| { + diagnostic + .message + .contains("remaining Origin 9.51 project objects") + })); + assert!(!worksheet.columns.iter().any(|column| column.name == "FAKE")); +} + +#[test] +fn modern_accepts_an_exact_window_boundary_without_inventing_a_tail() { + let fixture = modern_project(&[("Book1_A", &[1.0])], &["Book1"]); + let project = read_origin( + &fixture.bytes[..fixture.window_list_end], + OriginLimits::default(), + ) + .unwrap(); + assert_eq!(project.workbooks.len(), 1); + assert!(!project.diagnostics.iter().any(|diagnostic| { + diagnostic + .message + .contains("remaining Origin 9.51 project objects") + })); +} + +#[test] +fn modern_rejects_every_prefix_before_the_window_list_terminator() { + let fixture = modern_project(&[("Book1_A", &[1.0])], &["Book1"]); + for end in 0..fixture.window_list_end { + let outcome = catch_unwind(|| read_origin(&fixture.bytes[..end], OriginLimits::default())); + assert!(outcome.is_ok(), "prefix {end} panicked"); + assert!(outcome.unwrap().is_err(), "prefix {end} succeeded"); + } +} + +#[test] +fn modern_rejects_a_missing_window_list_terminator() { + let fixture = modern_project(&[("Book1_A", &[1.0])], &["Book1"]); + let bytes = &fixture.bytes[..fixture.final_window_terminator_start]; + assert!(matches!( + read_origin(bytes, OriginLimits::default()), + Err(OriginError::Truncated { .. }) + )); +} + +#[test] +fn modern_enforces_the_window_record_limit_before_association() { + let fixture = modern_project(&[("Book1_A", &[1.0])], &["Book1", "Graph1"]); + let limits = OriginLimits { + max_window_records: 1, + ..OriginLimits::default() + }; + assert!(matches!( + read_origin(&fixture.bytes, limits), + Err(OriginError::LimitExceeded { + resource: "window records", + limit: 1, + actual: 2, + }) + )); +} + +#[test] +fn modern_rejects_ambiguous_or_absent_worksheet_associations() { + for fixture in [ + modern_project(&[("Book1_A", &[1.0])], &["Book1", "Book1"]), + modern_project(&[], &["Graph1"]), + ] { + assert_eq!( + read_origin(&fixture.bytes, OriginLimits::default()).unwrap_err(), + OriginError::NoSupportedWorksheet + ); + } +} + +#[test] +fn modern_empty_columns_remain_null_after_worksheet_padding() { + let fixture = modern_project( + &[("Book1_A", &[1.0, 2.0]), ("Book1_B", &[EMPTY_F64])], + &["Book1"], + ); + let project = read_origin(&fixture.bytes, OriginLimits::default()).unwrap(); + let worksheet = &project.workbooks[0].worksheets[0]; + assert_eq!(worksheet.row_count, 2); + assert_eq!(worksheet.columns[1].cells, [OriginCell::Null]); +} diff --git a/crates/io/src/origin/opj/metadata_tests.rs b/crates/io/src/origin/opj/metadata_tests.rs new file mode 100644 index 0000000..a04b487 --- /dev/null +++ b/crates/io/src/origin/opj/metadata_tests.rs @@ -0,0 +1,632 @@ +use crate::origin::{ + OriginCell, OriginDiagnosticCode, OriginError, OriginLimits, OriginMetadataEntry, OriginNote, + OriginResourceUsage, read_origin, +}; + +use std::mem::size_of; + +const OPENOPJ_FIXTURE: &[u8] = + include_bytes!("../../../tests/fixtures/origin/test-origin-7.0552.opj"); + +#[test] +fn imports_real_fixture_parameters_and_notes() { + let project = read_origin(OPENOPJ_FIXTURE, OriginLimits::default()) + .expect("the licensed OpenOPJ fixture should import"); + + assert!( + project + .parameters + .iter() + .any(|entry| entry.key == "ERR" && entry.value == "1") + ); + assert!(project.notes.iter().any(|note| { + note.name == "Results" && note.content == "Data1 Temperature:\t25.10242\r\n\r\n" + })); +} + +const SIGNATURE: &[u8] = b"CPYA 4.2673 552#\n"; +const ORIGIN_HEADER_LEN: usize = 39; +const DATA_HEADER_LEN: usize = 123; + +fn push_block(bytes: &mut Vec, payload: Option<&[u8]>) { + let payload = payload.unwrap_or_default(); + bytes.extend_from_slice(&u32::try_from(payload.len()).unwrap().to_le_bytes()); + bytes.push(b'\n'); + if !payload.is_empty() { + bytes.extend_from_slice(payload); + bytes.push(b'\n'); + } +} + +fn data_header(name: &str, supported: bool, row_count: u32) -> [u8; DATA_HEADER_LEN] { + let mut header = [0_u8; DATA_HEADER_LEN]; + header[0x16..0x18].copy_from_slice(&0x6001_u16.to_le_bytes()); + header[0x18] = 1; + header[0x19..0x1d].copy_from_slice(&row_count.to_le_bytes()); + header[0x1d..0x21].copy_from_slice(&0_u32.to_le_bytes()); + header[0x21..0x25].copy_from_slice(&row_count.to_le_bytes()); + header[0x3d] = 8; + header[0x3f] = u8::from(!supported); + let name = name.as_bytes(); + header[0x58..0x58 + name.len()].copy_from_slice(name); + header[0x71..0x73].copy_from_slice(&0x10ca_u16.to_le_bytes()); + header +} + +fn push_window(bytes: &mut Vec, name: &[u8]) { + let mut header = [0_u8; 27]; + header[2..2 + name.len()].copy_from_slice(name); + push_block(bytes, Some(&header)); + push_block(bytes, None); +} + +fn synthetic_project_with_parameters( + records: &[(&str, bool, u32)], + windows: &[&[u8]], + parameters: &[(&[u8], f64)], +) -> Vec { + let mut bytes = SIGNATURE.to_vec(); + let mut origin_header = [0_u8; ORIGIN_HEADER_LEN]; + origin_header[0x1b..0x23].copy_from_slice(&7.0552_f64.to_le_bytes()); + push_block(&mut bytes, Some(&origin_header)); + push_block(&mut bytes, None); + + for (name, supported, row_count) in records { + push_block(&mut bytes, Some(&data_header(name, *supported, *row_count))); + let mut content = Vec::new(); + for _ in 0..*row_count { + content.extend_from_slice(&1.5_f64.to_le_bytes()); + } + push_block(&mut bytes, Some(&content)); + push_block(&mut bytes, None); + } + push_block(&mut bytes, None); + + for name in windows { + push_window(&mut bytes, name); + } + push_block(&mut bytes, None); + + for (name, value) in parameters { + bytes.extend_from_slice(name); + bytes.push(b'\n'); + bytes.extend_from_slice(&value.to_le_bytes()); + bytes.push(b'\n'); + } + bytes.extend_from_slice(b"\0\n"); + push_block(&mut bytes, None); + + push_block(&mut bytes, Some(&[0_u8; 4])); + push_block(&mut bytes, Some(b"Results\0")); + push_block(&mut bytes, Some(b"ok\0")); + push_block(&mut bytes, None); + push_block(&mut bytes, Some(&10_u32.to_le_bytes())); + bytes +} + +fn synthetic_project(records: &[(&str, bool)], windows: &[&[u8]]) -> Vec { + let records = records + .iter() + .map(|(name, supported)| (*name, *supported, 1)) + .collect::>(); + synthetic_project_with_parameters(&records, windows, &[(b"ERR".as_slice(), 1.0)]) +} + +fn insert_layer_with_two_nested_lists(bytes: &mut Vec) { + let marker = [27_u32.to_le_bytes().as_slice(), b"\n\0\0Book"].concat(); + let window = bytes + .windows(marker.len()) + .position(|candidate| candidate == marker) + .expect("synthetic window framing"); + let layer_list = window + 5 + 27 + 1; + let mut layer = Vec::new(); + push_block(&mut layer, Some(&[0_u8])); + push_block(&mut layer, Some(&[1_u8])); + for _ in 1..4 { + push_block(&mut layer, None); + } + push_block(&mut layer, None); + push_block(&mut layer, Some(&[1_u8])); + push_block(&mut layer, None); + for _ in 0..5 { + push_block(&mut layer, None); + } + bytes.splice(layer_list..layer_list, layer); +} + +fn only_column(project: &crate::origin::OriginProject) -> &crate::origin::OriginColumn { + &project.workbooks[0].worksheets[0].columns[0] +} + +#[test] +fn chooses_the_longest_validated_window_prefix() { + let bytes = synthetic_project(&[("BookLong_Value", true)], &[b"Book", b"BookLong"]); + let project = read_origin(&bytes, OriginLimits::default()).unwrap(); + + assert_eq!(project.workbooks.len(), 1); + assert_eq!(project.workbooks[0].name, "BookLong"); + assert_eq!(only_column(&project).name, "Value"); + assert_eq!(only_column(&project).cells, [OriginCell::Float(1.5)]); +} + +#[test] +fn requires_an_underscore_between_window_and_column_names() { + let bytes = synthetic_project(&[("BookValue", true)], &[b"Book"]); + assert!(matches!( + read_origin(&bytes, OriginLimits::default()), + Err(OriginError::NoSupportedWorksheet) + )); +} + +#[test] +fn preserves_non_identifier_column_suffixes_after_an_exact_window_prefix() { + let bytes = synthetic_project(&[("Book_1", true), ("Book_A-B", true)], &[b"Book"]); + let project = read_origin(&bytes, OriginLimits::default()).unwrap(); + + assert_eq!(project.workbooks.len(), 1); + assert_eq!(project.workbooks[0].name, "Book"); + assert_eq!( + project.workbooks[0].worksheets[0] + .columns + .iter() + .map(|column| column.name.as_str()) + .collect::>(), + ["1", "A-B"] + ); +} + +#[test] +fn ambiguous_or_missing_window_associations_are_not_imported() { + for windows in [ + &[b"Other".as_slice()][..], + &[b"Book".as_slice(), b"Book".as_slice()][..], + ] { + let dataset = if windows.len() == 1 { + "Orphan_Value" + } else { + "Book_Value" + }; + let bytes = synthetic_project(&[(dataset, true)], windows); + assert!(matches!( + read_origin(&bytes, OriginLimits::default()), + Err(OriginError::NoSupportedWorksheet) + )); + } +} + +#[test] +fn unmatched_columns_are_skipped_without_inventing_cross_dataset_alignment() { + let bytes = synthetic_project( + &[ + ("Book_Good", true), + ("Orphan_Value", true), + ("Other_Value", true), + ], + &[b"Book"], + ); + let project = read_origin(&bytes, OriginLimits::default()).unwrap(); + + assert_eq!(project.workbooks.len(), 1); + assert_eq!(project.workbooks[0].name, "Book"); + assert_eq!(project.workbooks[0].worksheets[0].columns.len(), 1); + assert_eq!(only_column(&project).name, "Good"); + assert_eq!( + project + .unsupported_objects + .iter() + .find(|summary| summary.kind == "worksheet columns") + .map(|summary| summary.count), + Some(2) + ); +} + +#[test] +fn skips_an_independently_framed_unsupported_column() { + let bytes = synthetic_project(&[("Book_Good", true), ("Book_Unknown", false)], &[b"Book"]); + let project = read_origin(&bytes, OriginLimits::default()).unwrap(); + + assert_eq!(project.workbooks[0].worksheets[0].columns.len(), 1); + assert_eq!(only_column(&project).name, "Good"); + assert!( + project.diagnostics.iter().any(|diagnostic| { + diagnostic.code == OriginDiagnosticCode::UnsupportedColumnSkipped + }) + ); + assert!( + project + .unsupported_objects + .iter() + .any(|summary| { summary.kind == "worksheet columns" && summary.count == 1 }) + ); +} + +#[test] +fn does_not_treat_an_exact_window_name_as_a_worksheet_column() { + let bytes = synthetic_project(&[("Matrix", true)], &[b"Matrix"]); + assert!(matches!( + read_origin(&bytes, OriginLimits::default()), + Err(OriginError::NoSupportedWorksheet) + )); +} + +#[test] +fn rejects_project_with_only_zero_row_supported_columns() { + let bytes = synthetic_project_with_parameters( + &[("Book_Empty", true, 0)], + &[b"Book"], + &[(b"ERR".as_slice(), 1.0)], + ); + + assert!(matches!( + read_origin(&bytes, OriginLimits::default()), + Err(OriginError::NoSupportedWorksheet) + )); +} + +#[test] +fn keeps_workbook_with_a_nonempty_supported_column() { + let bytes = synthetic_project_with_parameters( + &[("Book_Empty", true, 0), ("Book_Value", true, 1)], + &[b"Book"], + &[(b"ERR".as_slice(), 1.0)], + ); + + let project = read_origin(&bytes, OriginLimits::default()).unwrap(); + + assert_eq!(project.workbooks.len(), 1); + assert_eq!(project.workbooks[0].worksheets[0].row_count, 1); + assert_eq!(project.workbooks[0].worksheets[0].columns.len(), 2); + assert_eq!(project.resource_usage.workbooks, 1); + assert_eq!(project.resource_usage.worksheets, 1); + assert_eq!(project.resource_usage.columns, 2); + assert_eq!(project.resource_usage.cells, 1); +} + +#[test] +fn skips_bounded_non_ascii_note_metadata_with_a_warning() { + let mut bytes = synthetic_project(&[("Book_A", true)], &[b"Book"]); + let note_content = bytes + .windows(3) + .rposition(|window| window == b"ok\0") + .expect("synthetic note content"); + bytes[note_content] = 0x80; + + let project = read_origin(&bytes, OriginLimits::default()).unwrap(); + assert!(project.notes.is_empty()); + assert!( + project + .diagnostics + .iter() + .any(|diagnostic| { diagnostic.code == OriginDiagnosticCode::MetadataSkipped }) + ); +} + +#[test] +fn enforces_workbook_and_metadata_limits() { + let bytes = synthetic_project(&[("One_A", true), ("Two_B", true)], &[b"One", b"Two"]); + let workbook_limits = OriginLimits { + max_workbooks: 1, + ..OriginLimits::default() + }; + assert!(matches!( + read_origin(&bytes, workbook_limits), + Err(OriginError::LimitExceeded { + resource: "workbooks", + limit: 1, + actual: 2, + }) + )); + + let string_limits = OriginLimits { + max_string_bytes: 2, + ..OriginLimits::default() + }; + assert!(matches!( + read_origin(&bytes, string_limits), + Err(OriginError::LimitExceeded { + resource: "string bytes", + limit: 2, + .. + }) + )); +} + +#[test] +fn repeated_single_item_reservations_grow_logarithmically_and_charge_capacity() { + let limits = OriginLimits::default(); + let mut usage = OriginResourceUsage::default(); + let mut values = Vec::::new(); + let mut capacity_changes = 0_usize; + + for value in 0..4096_u64 { + let previous_capacity = values.capacity(); + super::try_reserve(&mut values, 1, "test metadata", &limits, &mut usage).unwrap(); + if values.capacity() != previous_capacity { + capacity_changes += 1; + } + values.push(value); + } + + assert!( + capacity_changes <= 16, + "single-item appends reallocated {capacity_changes} times" + ); + assert_eq!(usage.parser_bytes, values.capacity() * size_of::()); + assert_eq!(usage.total_owned_bytes, usage.parser_bytes); +} + +#[test] +fn spare_vector_capacity_is_not_charged_as_a_new_allocation() { + let limits = OriginLimits::default(); + let mut usage = OriginResourceUsage::default(); + let mut values = Vec::::with_capacity(8); + let original_capacity = values.capacity(); + + super::try_reserve(&mut values, 1, "test metadata", &limits, &mut usage).unwrap(); + + assert_eq!(values.capacity(), original_capacity); + assert_eq!(usage.parser_bytes, 0); + assert_eq!(usage.total_owned_bytes, 0); +} + +#[test] +fn existing_capacity_does_not_overflow_an_unbounded_custom_budget() { + let limits = OriginLimits { + max_parser_bytes: usize::MAX, + max_total_owned_bytes: usize::MAX, + ..OriginLimits::default() + }; + let mut usage = OriginResourceUsage::default(); + let mut values = vec![0_u8; 8]; + let original_capacity = values.capacity(); + + super::try_reserve(&mut values, 1, "test metadata", &limits, &mut usage).unwrap(); + + assert!(values.capacity() > original_capacity); + assert_eq!(usage.parser_bytes, values.capacity() - original_capacity); + assert_eq!(usage.total_owned_bytes, usage.parser_bytes); +} + +#[test] +fn rejects_excess_window_records_before_dataset_association() { + let window_names = (0..1025) + .map(|index| format!("W{index:04}")) + .collect::>(); + let window_name_bytes = window_names + .iter() + .map(|name| name.as_bytes()) + .collect::>(); + let bytes = synthetic_project(&[("W0000_A", true)], &window_name_bytes); + + assert!(matches!( + read_origin(&bytes, OriginLimits::default()), + Err(OriginError::LimitExceeded { + resource: "window records", + limit: 1024, + actual: 1025, + }) + )); +} + +#[test] +fn metadata_records_do_not_consume_the_data_column_limit() { + let bytes = synthetic_project_with_parameters( + &[("Book_A", true, 1)], + &[b"Book"], + &[ + (b"ERR".as_slice(), 1.0), + (b"ALPHA".as_slice(), 2.0), + (b"BETA".as_slice(), 3.0), + ], + ); + let limits = OriginLimits { + max_columns: 1, + ..OriginLimits::default() + }; + let project = read_origin(&bytes, limits).unwrap(); + + assert_eq!(project.workbooks[0].worksheets[0].columns.len(), 1); + assert_eq!(project.parameters.len(), 3); +} + +#[test] +fn metadata_record_limit_is_cumulative_across_nested_lists() { + let mut bytes = synthetic_project(&[("Book_A", true)], &[b"Book"]); + insert_layer_with_two_nested_lists(&mut bytes); + let limits = OriginLimits { + // Window, layer, and the first nested item consume the full budget; + // the first item in the second independent list is record four. + max_metadata_records: 3, + ..OriginLimits::default() + }; + + assert!(matches!( + read_origin(&bytes, limits), + Err(OriginError::LimitExceeded { + resource: "metadata records", + limit: 3, + actual: 4, + }) + )); +} + +#[test] +fn skipped_parameters_still_consume_the_metadata_record_budget() { + let cases: &[(&[u8], f64)] = &[(&[0x80], 2.0), (b"NAN", f64::NAN)]; + for &(name, value) in cases { + let bytes = synthetic_project_with_parameters( + &[("Book_A", true, 1)], + &[b"Book"], + &[(b"ERR", 1.0), (name, value)], + ); + let limits = OriginLimits { + max_metadata_records: 2, + ..OriginLimits::default() + }; + assert!(matches!( + read_origin(&bytes, limits), + Err(OriginError::LimitExceeded { + resource: "metadata records", + limit: 2, + actual: 3, + }) + )); + } +} + +#[test] +fn metadata_record_count_equal_to_the_limit_succeeds() { + let bytes = synthetic_project(&[("Book_A", true)], &[b"Book"]); + let limits = OriginLimits { + max_metadata_records: 4, + ..OriginLimits::default() + }; + let project = read_origin(&bytes, limits).unwrap(); + + assert_eq!(project.resource_usage.metadata_records, 4); +} + +#[test] +fn enforces_metadata_nesting_depth_before_walking_nested_lists() { + let mut bytes = synthetic_project(&[("Book_A", true)], &[b"Book"]); + let marker = [27_u32.to_le_bytes().as_slice(), b"\n\0\0Book"].concat(); + let window = bytes + .windows(marker.len()) + .position(|candidate| candidate == marker) + .expect("synthetic window framing"); + let layer_list = window + 5 + 27 + 1; + let mut nested_layer = Vec::new(); + push_block(&mut nested_layer, Some(&[0_u8])); + for _ in 0..6 { + push_block(&mut nested_layer, None); + } + bytes.splice(layer_list..layer_list, nested_layer); + + let limits = OriginLimits { + max_metadata_depth: 2, + ..OriginLimits::default() + }; + assert!(matches!( + read_origin(&bytes, limits), + Err(OriginError::LimitExceeded { + resource: "metadata nesting depth", + limit: 2, + actual: 3, + }) + )); +} + +#[test] +fn accepts_framed_null_components_but_rejects_a_missing_component() { + let mut complete = synthetic_project(&[("Book_A", true)], &[b"Book"]); + let marker = [27_u32.to_le_bytes().as_slice(), b"\n\0\0Book"].concat(); + let window = complete + .windows(marker.len()) + .position(|candidate| candidate == marker) + .expect("synthetic window framing"); + let layer_list = window + 5 + 27 + 1; + let mut layer = Vec::new(); + push_block(&mut layer, Some(&[0_u8])); + push_block(&mut layer, Some(&[1_u8])); + for _ in 0..9 { + push_block(&mut layer, None); + } + let missing_component = layer_list + 14; + complete.splice(layer_list..layer_list, layer); + + let project = read_origin(&complete, OriginLimits::default()).unwrap(); + assert!( + project + .unsupported_objects + .iter() + .any(|summary| { summary.kind == "window presentation records" && summary.count == 2 }) + ); + + let mut truncated_item = complete; + truncated_item.drain(missing_component..missing_component + 5); + assert!(read_origin(&truncated_item, OriginLimits::default()).is_err()); +} + +#[test] +fn retains_validated_parameter_and_note_values() { + let bytes = synthetic_project(&[("Book_A", true)], &[b"Book"]); + let project = read_origin(&bytes, OriginLimits::default()).unwrap(); + + assert_eq!( + project.parameters, + [OriginMetadataEntry { + key: "ERR".to_owned(), + value: "1".to_owned(), + }] + ); + assert_eq!( + project.notes, + [OriginNote { + name: "Results".to_owned(), + content: "ok".to_owned(), + }] + ); +} + +#[test] +fn truncated_metadata_never_panics_or_returns_partial_output() { + let complete = synthetic_project(&[("Book_A", true)], &[b"Book"]); + let metadata_start = complete + .windows(27) + .position(|window| window.get(2..6) == Some(b"Book")) + .expect("synthetic window header"); + + for prefix in metadata_start..complete.len() { + let result = + std::panic::catch_unwind(|| read_origin(&complete[..prefix], OriginLimits::default())); + assert!(result.is_ok(), "metadata prefix {prefix} panicked"); + assert!( + result.unwrap().is_err(), + "metadata prefix {prefix} succeeded" + ); + } +} + +#[test] +fn rejects_corrupt_real_fixture_project_tree_and_attachment_bounds() { + for end in [ + 0x43602, + 0x43607, + 0x43700, + 0x4377f, + 0x43790, + OPENOPJ_FIXTURE.len() - 1, + ] { + let result = std::panic::catch_unwind(|| { + read_origin(&OPENOPJ_FIXTURE[..end], OriginLimits::default()) + }); + assert!(result.is_ok(), "real fixture prefix {end:#x} panicked"); + assert!( + result.unwrap().is_err(), + "real fixture prefix {end:#x} succeeded" + ); + } + + let tree_only = read_origin(&OPENOPJ_FIXTURE[..0x4377e], OriginLimits::default()) + .expect("an exact project-tree EOF is a valid no-attachment project"); + assert!( + tree_only + .unsupported_objects + .iter() + .any(|summary| { summary.kind == "project tree" && summary.count == 1 }) + ); + assert!( + !tree_only + .unsupported_objects + .iter() + .any(|summary| { summary.kind == "embedded attachments" }) + ); + + let mut tree = OPENOPJ_FIXTURE.to_vec(); + tree[0x43607..0x4360b].copy_from_slice(&381_u32.to_le_bytes()); + assert!(read_origin(&tree, OriginLimits::default()).is_err()); + + let mut attachment = OPENOPJ_FIXTURE.to_vec(); + attachment[0x43786..0x4378a].copy_from_slice(&5633_u32.to_le_bytes()); + assert!(read_origin(&attachment, OriginLimits::default()).is_err()); +} diff --git a/crates/io/src/origin/opj/records.rs b/crates/io/src/origin/opj/records.rs new file mode 100644 index 0000000..7fdb9ec --- /dev/null +++ b/crates/io/src/origin/opj/records.rs @@ -0,0 +1,548 @@ +use std::mem::size_of; + +use crate::origin::{ + OriginCell, OriginColumnType, OriginError, OriginLimits, OriginProfile, OriginResourceUsage, +}; + +use super::super::reader::{checked_add, checked_mul}; + +const HEADER_LEN: usize = 123; +const TYPE_OFFSET: usize = 0x16; +const SECONDARY_TYPE_OFFSET: usize = 0x18; +const TOTAL_ROWS_OFFSET: usize = 0x19; +const FIRST_ROW_OFFSET: usize = 0x1d; +const LAST_ROW_OFFSET: usize = 0x21; +const WIDTH_OFFSET: usize = 0x3d; +const UNSIGNED_FLAG_OFFSET: usize = 0x3f; +const NAME_OFFSET: usize = 0x58; +const NAME_WIDTH: usize = 25; +const TERTIARY_TYPE_OFFSET: usize = 0x71; + +const TYPE_F64: u16 = 0x6001; +const TYPE_F32: u16 = 0x6003; +const TYPE_I32: u16 = 0x6801; +const TYPE_I16: u16 = 0x6803; +const TYPE_TEXT: u16 = 0x6021; +const TYPE_MIXED: u16 = 0x6121; +const FIXED_TEXT_WIDTH: u8 = 25; +const SECONDARY: u8 = 0x01; +const TERTIARY_NUMERIC: u16 = 0x10ca; +const TERTIARY_FLOAT_OR_TEXT: u16 = 0x10e8; +const EMPTY_F64_BITS: u64 = 0x81aa_74fe_1c13_2c0e; + +#[derive(Debug, PartialEq)] +pub(super) struct DecodedColumnRecord { + pub(super) dataset_name: String, + pub(super) column_type: OriginColumnType, + pub(super) cells: Vec, + pub(super) first_row: usize, + pub(super) last_row_exclusive: usize, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ValueKind { + F64, + F32, + I32, + I16, + FixedText, + Mixed, + ModernF64, +} + +pub(super) fn decode_column_record( + profile: OriginProfile, + header: &[u8], + content: Option<&[u8]>, + limits: &OriginLimits, + usage: &mut OriginResourceUsage, +) -> Result { + require_header_length(profile, header)?; + + // This layout and the exact type combinations below are reimplemented + // from the pinned MIT-licensed OpenOPJ Origin 7.0552 description and are + // cross-checked against its redistributable test fixture: + // https://github.com/jgonera/openopj/blob/42ddcf1eb3a490744c54fca0a4ed6fe7a5e723ca/docs/opj_format.markdown + // https://github.com/jgonera/openopj/blob/42ddcf1eb3a490744c54fca0a4ed6fe7a5e723ca/lib/OpenOPJ/DataSection.php + let data_type = read_u16_at(header, TYPE_OFFSET, "dataset type")?; + let secondary = read_u8_at(header, SECONDARY_TYPE_OFFSET, "secondary dataset type")?; + let total_rows = read_u32_at(header, TOTAL_ROWS_OFFSET, "total rows")?; + let first_row = read_u32_at(header, FIRST_ROW_OFFSET, "first row")?; + let last_row = read_u32_at(header, LAST_ROW_OFFSET, "last row")?; + let width = read_u8_at(header, WIDTH_OFFSET, "value width")?; + let unsigned_flag = read_u8_at(header, UNSIGNED_FLAG_OFFSET, "unsigned flag")?; + let tertiary = read_u16_at(header, TERTIARY_TYPE_OFFSET, "tertiary dataset type")?; + + let (total_rows, first_row, last_row_exclusive) = + validate_geometry(total_rows, first_row, last_row, limits)?; + let kind = classify_type( + profile, + data_type, + secondary, + width, + unsigned_flag, + tertiary, + last_row_exclusive == 0, + )?; + let width = usize::from(width); + let expected_bytes = checked_mul(total_rows, width, "OPJ column content bytes")?; + let content = require_content(content, expected_bytes)?; + let dataset_name = decode_ascii( + field(header, NAME_OFFSET, NAME_WIDTH, "dataset name")?, + NAME_OFFSET, + limits, + usage, + )?; + + charge_column_and_cells(last_row_exclusive, limits, usage)?; + let cell_bytes = checked_mul( + last_row_exclusive, + size_of::(), + "decoded OPJ cells", + )?; + charge_parser(cell_bytes, limits, usage)?; + let mut cells = Vec::new(); + cells + .try_reserve_exact(last_row_exclusive) + .map_err(|_| OriginError::AllocationFailed { + resource: "decoded OPJ cells", + requested: cell_bytes, + })?; + + // The public fixture proves that lastRow is exclusive: TestW_Float stores + // lastRow=2 for two values, while TestW_firstRow stores firstRow=1 and + // lastRow=3 for [missing, 5.23, -7]. Decode existing payload slots rather + // than prepending firstRow synthetic nulls, which would shift the data. + for row in 0..last_row_exclusive { + let start = checked_mul(row, width, "OPJ cell offset")?; + let end = checked_add(start, width, "OPJ cell range")?; + let slot = content + .get(start..end) + .ok_or_else(|| truncated_range(start, width, content.len().saturating_sub(start)))?; + let cell = decode_cell(kind, slot, start, limits, usage)?; + if row < first_row && cell != OriginCell::Null { + return Err(OriginError::CorruptStructure { + offset: start, + detail: "a payload slot before firstRow is not the verified missing-value sentinel" + .to_owned(), + }); + } + cells.push(cell); + } + + Ok(DecodedColumnRecord { + dataset_name, + column_type: column_type(kind), + cells, + first_row, + last_row_exclusive, + }) +} + +fn require_header_length(profile: OriginProfile, header: &[u8]) -> Result<(), OriginError> { + let expected = match profile { + OriginProfile::Origin7V552 => HEADER_LEN, + OriginProfile::Origin9V951 => 147, + }; + if header.len() < expected { + return Err(OriginError::Truncated { + offset: header.len(), + needed: expected - header.len(), + have: 0, + }); + } + if header.len() > expected { + return Err(OriginError::CorruptStructure { + offset: expected, + detail: format!( + "{} data header must be exactly {expected} bytes, not {}", + profile_name(profile), + header.len() + ), + }); + } + Ok(()) +} + +fn classify_type( + profile: OriginProfile, + data_type: u16, + secondary: u8, + width: u8, + unsigned_flag: u8, + tertiary: u16, + is_empty: bool, +) -> Result { + if profile == OriginProfile::Origin9V951 { + return classify_modern_type( + data_type, + secondary, + width, + unsigned_flag, + tertiary, + is_empty, + ); + } + if unsigned_flag != 0 { + return unsupported("unsigned Origin integers are not verified for Origin7V552"); + } + if secondary != SECONDARY { + return unsupported("the secondary Origin dataset type is not verified for Origin7V552"); + } + + match (data_type, width, tertiary) { + (TYPE_F64, 8, TERTIARY_NUMERIC) => Ok(ValueKind::F64), + (TYPE_F32, 4, TERTIARY_FLOAT_OR_TEXT) => Ok(ValueKind::F32), + (TYPE_I32, 4, TERTIARY_NUMERIC) => Ok(ValueKind::I32), + (TYPE_I16, 2, TERTIARY_NUMERIC) => Ok(ValueKind::I16), + (TYPE_TEXT, FIXED_TEXT_WIDTH, TERTIARY_FLOAT_OR_TEXT) => Ok(ValueKind::FixedText), + (TYPE_MIXED, 10, TERTIARY_NUMERIC) => Ok(ValueKind::Mixed), + _ => unsupported("the Origin dataset type and value width combination is not verified"), + } +} + +fn classify_modern_type( + data_type: u16, + secondary: u8, + width: u8, + storage_flag: u8, + tertiary: u16, + is_empty: bool, +) -> Result { + // Match complete tuples from two Origin 9.51 projects. Checking fields + // independently would silently accept an unobserved Cartesian product. + let verified = matches!( + ( + data_type, + secondary, + width, + storage_flag, + tertiary, + is_empty + ), + (0x5121, 0x03, 10, 0x20, 0x10c8, false) + | (0x5121, 0x03, 10, 0x30, 0x10c8, false) + | (0x5121, 0x03, 10, 0x30, 0x10ca, false) + | (0x6121, 0x03, 10, 0x00, 0x11ca, false) + | (0x6121, 0x03, 10, 0x10, 0x10c8, false) + | (0x6121, 0x03, 10, 0x10, 0x10c9, false) + | (0x6121, 0x03, 10, 0x10, 0x10ca, false) + | (0x6121, 0x01, 10, 0x00, 0x10ca, true) + | (0x6121, 0x03, 10, 0x00, 0x10ca, true) + ); + if !verified { + return unsupported("the complete Origin9V951 dataset type tuple is not verified"); + } + Ok(ValueKind::ModernF64) +} + +fn profile_name(profile: OriginProfile) -> &'static str { + match profile { + OriginProfile::Origin7V552 => "Origin7V552", + OriginProfile::Origin9V951 => "Origin9V951", + } +} + +fn validate_geometry( + total_rows: u32, + first_row: u32, + last_row: u32, + limits: &OriginLimits, +) -> Result<(usize, usize, usize), OriginError> { + if [total_rows, first_row, last_row] + .into_iter() + .any(|value| value > i32::MAX as u32) + { + return Err(OriginError::CorruptStructure { + offset: TOTAL_ROWS_OFFSET, + detail: "Origin7V552 row geometry contains a negative or unverified high-bit value" + .to_owned(), + }); + } + if first_row > last_row || last_row > total_rows { + return Err(OriginError::CorruptStructure { + offset: FIRST_ROW_OFFSET, + detail: "Origin7V552 rows must satisfy firstRow <= lastRow <= totalRows".to_owned(), + }); + } + + let total_rows = usize::try_from(total_rows).map_err(|_| OriginError::ArithmeticOverflow { + resource: "OPJ total rows", + })?; + let first_row = usize::try_from(first_row).map_err(|_| OriginError::ArithmeticOverflow { + resource: "OPJ first row", + })?; + let last_row = usize::try_from(last_row).map_err(|_| OriginError::ArithmeticOverflow { + resource: "OPJ last row", + })?; + enforce_limit("rows per column", total_rows, limits.max_rows_per_column)?; + Ok((total_rows, first_row, last_row)) +} + +fn require_content(content: Option<&[u8]>, expected: usize) -> Result<&[u8], OriginError> { + let content = content.unwrap_or_default(); + if content.len() < expected { + return Err(OriginError::Truncated { + offset: 0, + needed: expected, + have: content.len(), + }); + } + if content.len() > expected { + return Err(OriginError::CorruptStructure { + offset: expected, + detail: format!( + "Origin column content has {} bytes but its geometry requires {expected}", + content.len() + ), + }); + } + Ok(content) +} + +fn decode_cell( + kind: ValueKind, + slot: &[u8], + offset: usize, + limits: &OriginLimits, + usage: &mut OriginResourceUsage, +) -> Result { + match kind { + ValueKind::F64 => decode_f64(slot, offset), + ValueKind::F32 => Ok(OriginCell::Float(f64::from(f32::from_le_bytes( + read_array(slot, offset)?, + )))), + ValueKind::I32 => Ok(OriginCell::Integer(i64::from(i32::from_le_bytes( + read_array(slot, offset)?, + )))), + ValueKind::I16 => Ok(OriginCell::Integer(i64::from(i16::from_le_bytes( + read_array(slot, offset)?, + )))), + ValueKind::FixedText => Ok(OriginCell::Text(decode_ascii(slot, offset, limits, usage)?)), + ValueKind::Mixed => decode_mixed(slot, offset, limits, usage), + ValueKind::ModernF64 => decode_modern_f64(slot, offset), + } +} + +fn decode_modern_f64(slot: &[u8], offset: usize) -> Result { + let discriminator = *slot.first().ok_or_else(|| truncated_range(offset, 1, 0))?; + let reserved = *slot.get(1).ok_or_else(|| truncated_range(offset, 2, 1))?; + if reserved != 0 { + return Err(OriginError::CorruptStructure { + offset: checked_add(offset, 1, "Origin9V951 reserved prefix offset")?, + detail: "the reserved Origin9V951 numeric prefix byte must be zero".to_owned(), + }); + } + if discriminator != 0 { + return unsupported("Origin9V951 text and unknown cell discriminators are not supported"); + } + let payload = slot + .get(2..) + .ok_or_else(|| truncated_range(offset, 2, slot.len()))?; + decode_f64( + payload, + checked_add(offset, 2, "Origin9V951 numeric value offset")?, + ) +} + +fn decode_f64(slot: &[u8], offset: usize) -> Result { + let value = f64::from_le_bytes(read_array(slot, offset)?); + if value.to_bits() == EMPTY_F64_BITS { + Ok(OriginCell::Null) + } else { + Ok(OriginCell::Float(value)) + } +} + +fn decode_mixed( + slot: &[u8], + offset: usize, + limits: &OriginLimits, + usage: &mut OriginResourceUsage, +) -> Result { + let prefix = *slot.first().ok_or_else(|| truncated_range(offset, 1, 0))?; + let reserved = *slot.get(1).ok_or_else(|| truncated_range(offset, 2, 1))?; + if reserved != 0 { + return Err(OriginError::CorruptStructure { + offset: checked_add(offset, 1, "mixed prefix offset")?, + detail: "the reserved mixed-cell prefix byte must be zero".to_owned(), + }); + } + let payload = slot + .get(2..) + .ok_or_else(|| truncated_range(offset, 2, slot.len()))?; + match prefix { + 0 => decode_f64(payload, checked_add(offset, 2, "mixed value offset")?), + 1 => Ok(OriginCell::Text(decode_ascii( + payload, + checked_add(offset, 2, "mixed text offset")?, + limits, + usage, + )?)), + _ => Err(OriginError::CorruptStructure { + offset, + detail: "mixed Origin cells require a numeric prefix 0 or text prefix 1".to_owned(), + }), + } +} + +fn decode_ascii( + field: &[u8], + offset: usize, + limits: &OriginLimits, + usage: &mut OriginResourceUsage, +) -> Result { + let length = field + .iter() + .position(|byte| *byte == 0) + .unwrap_or(field.len()); + let text = field.get(..length).ok_or(OriginError::ArithmeticOverflow { + resource: "bounded ASCII field", + })?; + if let Some(relative) = text.iter().position(|byte| !byte.is_ascii()) { + return Err(OriginError::UnsupportedEncoding { + offset: checked_add(offset, relative, "ASCII byte offset")?, + encoding: "non-ASCII byte in Origin7V552 text".to_owned(), + }); + } + enforce_limit("string bytes", length, limits.max_string_bytes)?; + charge_text(length, limits, usage)?; + + let mut decoded = String::new(); + decoded + .try_reserve_exact(length) + .map_err(|_| OriginError::AllocationFailed { + resource: "decoded Origin text", + requested: length, + })?; + let text = std::str::from_utf8(text).map_err(|_| OriginError::UnsupportedEncoding { + offset, + encoding: "non-ASCII byte in Origin7V552 text".to_owned(), + })?; + decoded.push_str(text); + Ok(decoded) +} + +fn column_type(kind: ValueKind) -> OriginColumnType { + match kind { + ValueKind::F64 | ValueKind::F32 => OriginColumnType::Float, + ValueKind::I32 | ValueKind::I16 => OriginColumnType::Integer, + ValueKind::FixedText => OriginColumnType::Text, + ValueKind::Mixed => OriginColumnType::Mixed, + ValueKind::ModernF64 => OriginColumnType::Float, + } +} + +fn field<'a>( + bytes: &'a [u8], + offset: usize, + length: usize, + resource: &'static str, +) -> Result<&'a [u8], OriginError> { + let end = checked_add(offset, length, resource)?; + bytes + .get(offset..end) + .ok_or_else(|| truncated_range(offset, length, bytes.len().saturating_sub(offset))) +} + +fn read_u8_at(bytes: &[u8], offset: usize, resource: &'static str) -> Result { + field(bytes, offset, 1, resource)? + .first() + .copied() + .ok_or_else(|| truncated_range(offset, 1, 0)) +} + +fn read_u16_at(bytes: &[u8], offset: usize, resource: &'static str) -> Result { + Ok(u16::from_le_bytes(read_array( + field(bytes, offset, size_of::(), resource)?, + offset, + )?)) +} + +fn read_u32_at(bytes: &[u8], offset: usize, resource: &'static str) -> Result { + Ok(u32::from_le_bytes(read_array( + field(bytes, offset, size_of::(), resource)?, + offset, + )?)) +} + +fn read_array(bytes: &[u8], offset: usize) -> Result<[u8; N], OriginError> { + bytes.try_into().map_err(|_| OriginError::Truncated { + offset, + needed: N, + have: bytes.len(), + }) +} + +fn charge_column_and_cells( + cells: usize, + limits: &OriginLimits, + usage: &mut OriginResourceUsage, +) -> Result<(), OriginError> { + let columns = checked_add(usage.columns, 1, "decoded OPJ columns")?; + enforce_limit("columns", columns, limits.max_columns)?; + let total_cells = checked_add(usage.cells, cells, "decoded OPJ cells")?; + enforce_limit("cells", total_cells, limits.max_cells)?; + usage.columns = columns; + usage.cells = total_cells; + Ok(()) +} + +fn charge_text( + bytes: usize, + limits: &OriginLimits, + usage: &mut OriginResourceUsage, +) -> Result<(), OriginError> { + let decoded = checked_add(usage.decoded_text_bytes, bytes, "decoded text bytes")?; + enforce_limit("decoded text bytes", decoded, limits.max_decoded_text_bytes)?; + charge_parser(bytes, limits, usage)?; + usage.decoded_text_bytes = decoded; + Ok(()) +} + +fn charge_parser( + bytes: usize, + limits: &OriginLimits, + usage: &mut OriginResourceUsage, +) -> Result<(), OriginError> { + let parser = checked_add(usage.parser_bytes, bytes, "parser bytes")?; + enforce_limit("parser bytes", parser, limits.max_parser_bytes)?; + let total = checked_add(usage.total_owned_bytes, bytes, "total owned bytes")?; + enforce_limit("total owned bytes", total, limits.max_total_owned_bytes)?; + usage.parser_bytes = parser; + usage.total_owned_bytes = total; + Ok(()) +} + +fn enforce_limit(resource: &'static str, actual: usize, limit: usize) -> Result<(), OriginError> { + if actual > limit { + return Err(OriginError::LimitExceeded { + resource, + limit, + actual, + }); + } + Ok(()) +} + +fn unsupported(feature: &'static str) -> Result { + Err(OriginError::UnsupportedFeature { + feature: feature.to_owned(), + }) +} + +fn truncated_range(offset: usize, needed: usize, have: usize) -> OriginError { + OriginError::Truncated { + offset, + needed, + have, + } +} + +#[cfg(test)] +#[path = "records_tests.rs"] +mod records_tests; + +#[cfg(test)] +#[path = "records_modern_tests.rs"] +mod records_modern_tests; diff --git a/crates/io/src/origin/opj/records_modern_tests.rs b/crates/io/src/origin/opj/records_modern_tests.rs new file mode 100644 index 0000000..950c77d --- /dev/null +++ b/crates/io/src/origin/opj/records_modern_tests.rs @@ -0,0 +1,288 @@ +use std::panic::catch_unwind; + +use super::{DecodedColumnRecord, decode_column_record}; +use crate::origin::{ + OriginCell, OriginColumnType, OriginError, OriginLimits, OriginProfile, OriginResourceUsage, +}; + +const HEADER_LEN: usize = 147; +const TYPE_OFFSET: usize = 0x16; +const SECONDARY_OFFSET: usize = 0x18; +const TOTAL_ROWS_OFFSET: usize = 0x19; +const FIRST_ROW_OFFSET: usize = 0x1d; +const LAST_ROW_OFFSET: usize = 0x21; +const WIDTH_OFFSET: usize = 0x3d; +const STORAGE_FLAG_OFFSET: usize = 0x3f; +const NAME_OFFSET: usize = 0x58; +const TERTIARY_OFFSET: usize = 0x71; +const EMPTY_F64: f64 = -1.23456789E-300; + +#[derive(Clone, Copy)] +struct ModernHeader { + data_type: u16, + secondary: u8, + total_rows: u32, + first_row: u32, + last_row: u32, + width: u8, + storage_flag: u8, + tertiary: u16, +} + +impl Default for ModernHeader { + fn default() -> Self { + Self { + data_type: 0x6121, + secondary: 0x03, + total_rows: 2, + first_row: 0, + last_row: 2, + width: 10, + storage_flag: 0x10, + tertiary: 0x10ca, + } + } +} + +fn header(spec: ModernHeader) -> Vec { + let mut bytes = vec![0_u8; HEADER_LEN]; + bytes[TYPE_OFFSET..TYPE_OFFSET + 2].copy_from_slice(&spec.data_type.to_le_bytes()); + bytes[SECONDARY_OFFSET] = spec.secondary; + bytes[TOTAL_ROWS_OFFSET..TOTAL_ROWS_OFFSET + 4].copy_from_slice(&spec.total_rows.to_le_bytes()); + bytes[FIRST_ROW_OFFSET..FIRST_ROW_OFFSET + 4].copy_from_slice(&spec.first_row.to_le_bytes()); + bytes[LAST_ROW_OFFSET..LAST_ROW_OFFSET + 4].copy_from_slice(&spec.last_row.to_le_bytes()); + bytes[WIDTH_OFFSET] = spec.width; + bytes[STORAGE_FLAG_OFFSET] = spec.storage_flag; + bytes[NAME_OFFSET..NAME_OFFSET + 7].copy_from_slice(b"Book1_A"); + bytes[TERTIARY_OFFSET..TERTIARY_OFFSET + 2].copy_from_slice(&spec.tertiary.to_le_bytes()); + bytes +} + +fn slots(values: &[f64]) -> Vec { + values + .iter() + .flat_map(|value| [0, 0].into_iter().chain(value.to_le_bytes())) + .collect() +} + +fn decode(header: &[u8], content: &[u8]) -> Result { + let mut usage = OriginResourceUsage::default(); + decode_column_record( + OriginProfile::Origin9V951, + header, + Some(content), + &OriginLimits::default(), + &mut usage, + ) +} + +#[test] +fn modern_decodes_only_the_observed_numeric_field_combinations() { + let observed = [ + (0x5121, 0x20, 0x10c8), + (0x5121, 0x30, 0x10c8), + (0x5121, 0x30, 0x10ca), + (0x6121, 0x00, 0x11ca), + (0x6121, 0x10, 0x10c8), + (0x6121, 0x10, 0x10c9), + (0x6121, 0x10, 0x10ca), + ]; + for (data_type, storage_flag, tertiary) in observed { + let decoded = decode( + &header(ModernHeader { + data_type, + storage_flag, + tertiary, + ..ModernHeader::default() + }), + &slots(&[0.05, 20.0]), + ) + .unwrap(); + assert_eq!(decoded.dataset_name, "Book1_A"); + assert_eq!(decoded.column_type, OriginColumnType::Float); + assert_eq!( + decoded.cells, + vec![OriginCell::Float(0.05), OriginCell::Float(20.0)] + ); + } +} + +#[test] +fn modern_rejects_unobserved_cross_product_of_verified_fields() { + let result = decode( + &header(ModernHeader { + data_type: 0x5121, + storage_flag: 0x00, + tertiary: 0x11ca, + ..ModernHeader::default() + }), + &slots(&[0.05, 20.0]), + ); + + assert!(matches!( + result, + Err(OriginError::UnsupportedFeature { .. }) + )); +} + +#[test] +fn modern_maps_the_verified_missing_sentinel_to_null() { + let decoded = decode(&header(ModernHeader::default()), &slots(&[EMPTY_F64, 1.0])).unwrap(); + assert_eq!( + decoded.cells, + vec![OriginCell::Null, OriginCell::Float(1.0)] + ); +} + +#[test] +fn modern_retains_an_empty_column_with_validated_storage() { + for secondary in [0x01, 0x03] { + let spec = ModernHeader { + secondary, + total_rows: 3, + last_row: 0, + storage_flag: 0x00, + tertiary: 0x10ca, + ..ModernHeader::default() + }; + let decoded = decode(&header(spec), &slots(&[EMPTY_F64; 3])).unwrap(); + assert!(decoded.cells.is_empty()); + assert_eq!(decoded.first_row, 0); + assert_eq!(decoded.last_row_exclusive, 0); + } +} + +#[test] +fn modern_rejects_text_discriminator_without_guessing() { + let mut content = slots(&[1.0, 2.0]); + content[0] = 1; + assert!(matches!( + decode(&header(ModernHeader::default()), &content), + Err(OriginError::UnsupportedFeature { .. }) + )); +} + +#[test] +fn modern_rejects_nonzero_reserved_prefix_as_corrupt() { + let mut content = slots(&[1.0, 2.0]); + content[1] = 1; + assert!(matches!( + decode(&header(ModernHeader::default()), &content), + Err(OriginError::CorruptStructure { .. }) + )); +} + +#[test] +fn modern_rejects_unobserved_header_fields() { + let cases = [ + ModernHeader { + data_type: 0x6001, + ..ModernHeader::default() + }, + ModernHeader { + secondary: 0x01, + ..ModernHeader::default() + }, + ModernHeader { + width: 8, + ..ModernHeader::default() + }, + ModernHeader { + storage_flag: 0x40, + ..ModernHeader::default() + }, + ModernHeader { + tertiary: 0x10e8, + ..ModernHeader::default() + }, + ]; + for spec in cases { + assert!(matches!( + decode(&header(spec), &slots(&[1.0, 2.0])), + Err(OriginError::UnsupportedFeature { .. }) + )); + } +} + +#[test] +fn modern_rejects_wrong_header_and_content_lengths() { + let complete_header = header(ModernHeader::default()); + let complete_content = slots(&[1.0, 2.0]); + for length in [146, 148] { + let mut wrong = complete_header.clone(); + wrong.resize(length, 0); + assert!(decode(&wrong, &complete_content).is_err()); + } + assert!(decode(&complete_header, &complete_content[..19]).is_err()); + let mut extra = complete_content.clone(); + extra.push(0); + assert!(decode(&complete_header, &extra).is_err()); +} + +#[test] +fn modern_rejects_invalid_geometry_and_configured_limits() { + let invalid = header(ModernHeader { + first_row: 2, + last_row: 1, + ..ModernHeader::default() + }); + assert!(matches!( + decode(&invalid, &slots(&[1.0, 2.0])), + Err(OriginError::CorruptStructure { .. }) + )); + + let mut usage = OriginResourceUsage::default(); + let limits = OriginLimits { + max_rows_per_column: 1, + ..OriginLimits::default() + }; + let content = slots(&[1.0, 2.0]); + assert!(matches!( + decode_column_record( + OriginProfile::Origin9V951, + &header(ModernHeader::default()), + Some(&content), + &limits, + &mut usage, + ), + Err(OriginError::LimitExceeded { + resource: "rows per column", + .. + }) + )); + + let mut usage = OriginResourceUsage::default(); + let limits = OriginLimits { + max_cells: 1, + ..OriginLimits::default() + }; + assert!(matches!( + decode_column_record( + OriginProfile::Origin9V951, + &header(ModernHeader::default()), + Some(&content), + &limits, + &mut usage, + ), + Err(OriginError::LimitExceeded { + resource: "cells", + .. + }) + )); +} + +#[test] +fn modern_every_truncated_prefix_returns_an_error_without_panicking() { + let complete_header = header(ModernHeader::default()); + let complete_content = slots(&[1.0, 2.0]); + for end in 0..complete_header.len() { + let outcome = catch_unwind(|| decode(&complete_header[..end], &complete_content)); + assert!(outcome.is_ok(), "header prefix {end} panicked"); + assert!(outcome.unwrap().is_err(), "header prefix {end} succeeded"); + } + for end in 0..complete_content.len() { + let outcome = catch_unwind(|| decode(&complete_header, &complete_content[..end])); + assert!(outcome.is_ok(), "content prefix {end} panicked"); + assert!(outcome.unwrap().is_err(), "content prefix {end} succeeded"); + } +} diff --git a/crates/io/src/origin/opj/records_tests.rs b/crates/io/src/origin/opj/records_tests.rs new file mode 100644 index 0000000..b0cac62 --- /dev/null +++ b/crates/io/src/origin/opj/records_tests.rs @@ -0,0 +1,668 @@ +use std::mem::size_of; +use std::panic::catch_unwind; + +use super::{DecodedColumnRecord, decode_column_record}; +use crate::origin::{ + OriginCell, OriginColumnType, OriginError, OriginLimits, OriginProfile, OriginResourceUsage, +}; + +const HEADER_LEN: usize = 123; +const TYPE_OFFSET: usize = 0x16; +const SECONDARY_TYPE_OFFSET: usize = 0x18; +const TOTAL_ROWS_OFFSET: usize = 0x19; +const FIRST_ROW_OFFSET: usize = 0x1d; +const LAST_ROW_OFFSET: usize = 0x21; +const WIDTH_OFFSET: usize = 0x3d; +const UNSIGNED_FLAG_OFFSET: usize = 0x3f; +const NAME_OFFSET: usize = 0x58; +const NAME_WIDTH: usize = 25; +const TERTIARY_TYPE_OFFSET: usize = 0x71; + +const TYPE_F64: u16 = 0x6001; +const TYPE_F32: u16 = 0x6003; +const TYPE_I32: u16 = 0x6801; +const TYPE_I16: u16 = 0x6803; +const TYPE_TEXT: u16 = 0x6021; +const TYPE_MIXED: u16 = 0x6121; + +const SECONDARY: u8 = 0x01; +const TERTIARY_NUMERIC: u16 = 0x10ca; +const TERTIARY_FLOAT_OR_TEXT: u16 = 0x10e8; +const EMPTY_F64: f64 = -1.23456789E-300; +const FIXTURE_F32: f32 = f32::from_bits(0x43ac_cccd); +const FIXTURE_MIXED_F64: f64 = f64::from_bits(0x4009_1eb8_51eb_851f); + +#[derive(Clone)] +struct RecordBytes { + header: Vec, + content: Vec, +} + +#[derive(Clone, Copy)] +struct HeaderSpec { + data_type: u16, + secondary: u8, + total_rows: u32, + first_row: u32, + last_row: u32, + width: u8, + unsigned_flag: u8, + tertiary: u16, +} + +impl HeaderSpec { + fn fixture_type( + data_type: u16, + total_rows: u32, + first_row: u32, + last_row: u32, + width: u8, + tertiary: u16, + ) -> Self { + Self { + data_type, + secondary: SECONDARY, + total_rows, + first_row, + last_row, + width, + unsigned_flag: 0, + tertiary, + } + } +} + +fn header(spec: HeaderSpec, name: &str) -> Vec { + let mut bytes = vec![0_u8; HEADER_LEN]; + bytes[TYPE_OFFSET..TYPE_OFFSET + 2].copy_from_slice(&spec.data_type.to_le_bytes()); + bytes[SECONDARY_TYPE_OFFSET] = spec.secondary; + bytes[TOTAL_ROWS_OFFSET..TOTAL_ROWS_OFFSET + 4].copy_from_slice(&spec.total_rows.to_le_bytes()); + bytes[FIRST_ROW_OFFSET..FIRST_ROW_OFFSET + 4].copy_from_slice(&spec.first_row.to_le_bytes()); + bytes[LAST_ROW_OFFSET..LAST_ROW_OFFSET + 4].copy_from_slice(&spec.last_row.to_le_bytes()); + bytes[WIDTH_OFFSET] = spec.width; + bytes[UNSIGNED_FLAG_OFFSET] = spec.unsigned_flag; + bytes[TERTIARY_TYPE_OFFSET..TERTIARY_TYPE_OFFSET + 2] + .copy_from_slice(&spec.tertiary.to_le_bytes()); + let name = name.as_bytes(); + assert!(name.len() < NAME_WIDTH); + bytes[NAME_OFFSET..NAME_OFFSET + name.len()].copy_from_slice(name); + bytes +} + +fn f64_record(values: &[f64]) -> RecordBytes { + let row_count = u32::try_from(values.len()).unwrap(); + RecordBytes { + header: header( + HeaderSpec::fixture_type(TYPE_F64, row_count, 0, row_count, 8, TERTIARY_NUMERIC), + "Data1_INJV", + ), + content: values + .iter() + .flat_map(|value| value.to_le_bytes()) + .collect(), + } +} + +fn f32_record(values: &[f32]) -> RecordBytes { + let row_count = u32::try_from(values.len()).unwrap(); + RecordBytes { + header: header( + HeaderSpec::fixture_type(TYPE_F32, row_count, 0, row_count, 4, TERTIARY_FLOAT_OR_TEXT), + "TestW_Float", + ), + content: values + .iter() + .flat_map(|value| value.to_le_bytes()) + .collect(), + } +} + +fn i32_record(values: &[i32]) -> RecordBytes { + let row_count = u32::try_from(values.len()).unwrap(); + RecordBytes { + header: header( + HeaderSpec::fixture_type(TYPE_I32, row_count, 0, row_count, 4, TERTIARY_NUMERIC), + "TestW_Long", + ), + content: values + .iter() + .flat_map(|value| value.to_le_bytes()) + .collect(), + } +} + +fn i16_record(values: &[i16]) -> RecordBytes { + let row_count = u32::try_from(values.len()).unwrap(); + RecordBytes { + header: header( + HeaderSpec::fixture_type(TYPE_I16, row_count, 0, row_count, 2, TERTIARY_NUMERIC), + "TestW_Integer", + ), + content: values + .iter() + .flat_map(|value| value.to_le_bytes()) + .collect(), + } +} + +fn text_record(value: &str) -> RecordBytes { + const WIDTH: usize = 25; + let mut content = vec![0_u8; WIDTH]; + content[..value.len()].copy_from_slice(value.as_bytes()); + content[value.len() + 1..].fill(0xff); + RecordBytes { + header: header( + HeaderSpec::fixture_type(TYPE_TEXT, 1, 0, 1, WIDTH as u8, TERTIARY_FLOAT_OR_TEXT), + "TestW_Text", + ), + content, + } +} + +fn mixed_record() -> RecordBytes { + let mut content = Vec::new(); + content.extend_from_slice(&[1, 0]); + content.extend_from_slice(b"text\0\xff\xfe\xfd"); + content.extend_from_slice(&[0, 0]); + content.extend_from_slice(&FIXTURE_MIXED_F64.to_le_bytes()); + RecordBytes { + header: header( + HeaderSpec::fixture_type(TYPE_MIXED, 2, 0, 2, 10, TERTIARY_NUMERIC), + "TestW_TextNumeric", + ), + content, + } +} + +fn decode(record: &RecordBytes) -> Result { + let mut usage = OriginResourceUsage::default(); + decode_with(record, OriginLimits::default(), &mut usage) +} + +fn decode_with( + record: &RecordBytes, + limits: OriginLimits, + usage: &mut OriginResourceUsage, +) -> Result { + decode_column_record( + OriginProfile::Origin7V552, + &record.header, + Some(&record.content), + &limits, + usage, + ) +} + +fn clear_dataset_name(record: &mut RecordBytes) { + record.header[NAME_OFFSET..NAME_OFFSET + NAME_WIDTH].fill(0); +} + +fn assert_limit(error: OriginError, resource: &'static str, limit: usize, actual: usize) { + assert_eq!( + error, + OriginError::LimitExceeded { + resource, + limit, + actual, + } + ); +} + +#[test] +fn decodes_fixture_backed_f64_and_missing_sentinel() { + let decoded = decode(&f64_record(&[0.4, EMPTY_F64])).unwrap(); + assert_eq!(decoded.dataset_name, "Data1_INJV"); + assert_eq!(decoded.column_type, OriginColumnType::Float); + assert_eq!( + decoded.cells, + vec![OriginCell::Float(0.4), OriginCell::Null] + ); +} + +#[test] +fn decodes_fixture_backed_f32_losslessly_into_f64() { + let decoded = decode(&f32_record(&[FIXTURE_F32])).unwrap(); + assert_eq!( + decoded.cells, + vec![OriginCell::Float(f64::from(FIXTURE_F32))] + ); +} + +#[test] +fn decodes_fixture_backed_signed_i32() { + let decoded = decode(&i32_record(&[345, -100_000])).unwrap(); + assert_eq!(decoded.column_type, OriginColumnType::Integer); + assert_eq!( + decoded.cells, + vec![OriginCell::Integer(345), OriginCell::Integer(-100_000)] + ); +} + +#[test] +fn decodes_fixture_backed_signed_i16() { + let decoded = decode(&i16_record(&[34, -1000])).unwrap(); + assert_eq!( + decoded.cells, + vec![OriginCell::Integer(34), OriginCell::Integer(-1000)] + ); +} + +#[test] +fn decodes_fixture_backed_fixed_ascii_text() { + let decoded = decode(&text_record("test string 123")).unwrap(); + assert_eq!(decoded.column_type, OriginColumnType::Text); + assert_eq!( + decoded.cells, + vec![OriginCell::Text("test string 123".to_owned())] + ); +} + +#[test] +fn decodes_fixture_backed_mixed_text_and_number() { + let decoded = decode(&mixed_record()).unwrap(); + assert_eq!(decoded.column_type, OriginColumnType::Mixed); + assert_eq!( + decoded.cells, + vec![ + OriginCell::Text("text".to_owned()), + OriginCell::Float(FIXTURE_MIXED_F64) + ] + ); +} + +#[test] +fn fixture_exclusive_last_row_preserves_verified_leading_null_slot() { + let mut content = Vec::new(); + content.extend_from_slice(&EMPTY_F64.to_le_bytes()); + content.extend_from_slice(&5.23_f64.to_le_bytes()); + content.extend_from_slice(&(-7.0_f64).to_le_bytes()); + let record = RecordBytes { + header: header( + HeaderSpec::fixture_type(TYPE_F64, 3, 1, 3, 8, TERTIARY_NUMERIC), + "TestW_firstRow", + ), + content, + }; + + let decoded = decode(&record).unwrap(); + assert_eq!(decoded.first_row, 1); + assert_eq!(decoded.last_row_exclusive, 3); + assert_eq!( + decoded.cells, + vec![ + OriginCell::Null, + OriginCell::Float(5.23), + OriginCell::Float(-7.0) + ] + ); +} + +#[test] +fn rejects_nonnull_payload_slots_before_first_row() { + let mut content = Vec::new(); + content.extend_from_slice(&999.0_f64.to_le_bytes()); + content.extend_from_slice(&5.23_f64.to_le_bytes()); + let record = RecordBytes { + header: header( + HeaderSpec::fixture_type(TYPE_F64, 2, 1, 2, 8, TERTIARY_NUMERIC), + "InvalidFirstRow", + ), + content, + }; + + assert!(matches!( + decode(&record), + Err(OriginError::CorruptStructure { .. }) + )); +} + +#[test] +fn rejects_width_and_fixture_type_mismatches() { + let cases = [ + (TYPE_F64, 4, TERTIARY_NUMERIC), + (TYPE_F32, 8, TERTIARY_FLOAT_OR_TEXT), + (TYPE_I32, 2, TERTIARY_NUMERIC), + (TYPE_I16, 4, TERTIARY_NUMERIC), + (TYPE_TEXT, 8, TERTIARY_FLOAT_OR_TEXT), + (TYPE_TEXT, 24, TERTIARY_FLOAT_OR_TEXT), + (TYPE_TEXT, 26, TERTIARY_FLOAT_OR_TEXT), + (TYPE_MIXED, 9, TERTIARY_NUMERIC), + ]; + for (data_type, width, tertiary) in cases { + let record = RecordBytes { + header: header( + HeaderSpec::fixture_type(data_type, 1, 0, 1, width, tertiary), + "Mismatch", + ), + content: vec![0; usize::from(width)], + }; + assert!(matches!( + decode(&record), + Err(OriginError::UnsupportedFeature { .. }) + )); + } +} + +#[test] +fn rejects_geometry_outside_total_rows_or_signed_range() { + let beyond_total = RecordBytes { + header: header( + HeaderSpec::fixture_type(TYPE_F64, 1, 0, 2, 8, TERTIARY_NUMERIC), + "BeyondTotal", + ), + content: vec![0; 8], + }; + assert!(matches!( + decode(&beyond_total), + Err(OriginError::CorruptStructure { .. }) + )); + + let reversed = RecordBytes { + header: header( + HeaderSpec::fixture_type(TYPE_F64, 3, 2, 1, 8, TERTIARY_NUMERIC), + "Reversed", + ), + content: vec![0; 24], + }; + assert!(matches!( + decode(&reversed), + Err(OriginError::CorruptStructure { .. }) + )); + + let signed_negative = RecordBytes { + header: header( + HeaderSpec::fixture_type(TYPE_F64, u32::MAX, 0, 1, 8, TERTIARY_NUMERIC), + "NegativeGeometry", + ), + content: Vec::new(), + }; + assert!(matches!( + decode(&signed_negative), + Err(OriginError::CorruptStructure { .. }) + )); +} + +#[test] +fn rejects_incomplete_fixed_text_and_mixed_numeric_payloads() { + let mut fixed = text_record("test string 123"); + fixed.content.pop(); + assert!(matches!(decode(&fixed), Err(OriginError::Truncated { .. }))); + + let mut mixed = mixed_record(); + mixed.header = header( + HeaderSpec::fixture_type(TYPE_MIXED, 1, 0, 1, 10, TERTIARY_NUMERIC), + "MixedNumeric", + ); + mixed.content = vec![0, 0, 1, 2, 3, 4, 5, 6, 7]; + assert!(matches!(decode(&mixed), Err(OriginError::Truncated { .. }))); +} + +#[test] +fn rejects_content_one_byte_larger_than_declared_geometry() { + let mut oversized = f64_record(&[0.4]); + oversized.content.push(0); + + assert!(matches!( + decode(&oversized), + Err(OriginError::CorruptStructure { .. }) + )); +} + +#[test] +fn enforces_row_column_and_cell_limits_with_exact_counts() { + let record = f64_record(&[0.4, EMPTY_F64]); + let mut usage = OriginResourceUsage::default(); + let row_limits = OriginLimits { + max_rows_per_column: 1, + ..OriginLimits::default() + }; + assert_limit( + decode_with(&record, row_limits, &mut usage).unwrap_err(), + "rows per column", + 1, + 2, + ); + + let mut usage = OriginResourceUsage { + columns: 1, + ..OriginResourceUsage::default() + }; + let column_limits = OriginLimits { + max_columns: 1, + ..OriginLimits::default() + }; + assert_limit( + decode_with(&record, column_limits, &mut usage).unwrap_err(), + "columns", + 1, + 2, + ); + + let mut usage = OriginResourceUsage { + cells: 1, + ..OriginResourceUsage::default() + }; + let cell_limits = OriginLimits { + max_cells: 2, + ..OriginLimits::default() + }; + assert_limit( + decode_with(&record, cell_limits, &mut usage).unwrap_err(), + "cells", + 2, + 3, + ); +} + +#[test] +fn enforces_string_and_cumulative_decoded_text_limits() { + let named_record = f64_record(&[0.4]); + let mut usage = OriginResourceUsage::default(); + let dataset_name_limits = OriginLimits { + max_string_bytes: 9, + ..OriginLimits::default() + }; + assert_limit( + decode_with(&named_record, dataset_name_limits, &mut usage).unwrap_err(), + "string bytes", + 9, + 10, + ); + + let mut record = text_record("test string 123"); + clear_dataset_name(&mut record); + + let mut usage = OriginResourceUsage::default(); + let string_limits = OriginLimits { + max_string_bytes: 14, + ..OriginLimits::default() + }; + assert_limit( + decode_with(&record, string_limits, &mut usage).unwrap_err(), + "string bytes", + 14, + 15, + ); + + let mut usage = OriginResourceUsage { + decoded_text_bytes: 1, + ..OriginResourceUsage::default() + }; + let text_limits = OriginLimits { + max_decoded_text_bytes: 15, + ..OriginLimits::default() + }; + assert_limit( + decode_with(&record, text_limits, &mut usage).unwrap_err(), + "decoded text bytes", + 15, + 16, + ); +} + +#[test] +fn enforces_parser_and_total_owned_limits_before_cell_vector_allocation() { + let mut record = f64_record(&[0.4, EMPTY_F64]); + clear_dataset_name(&mut record); + let cell_bytes = 2 * size_of::(); + + let mut usage = OriginResourceUsage { + parser_bytes: 1, + ..OriginResourceUsage::default() + }; + let parser_limits = OriginLimits { + max_parser_bytes: cell_bytes, + ..OriginLimits::default() + }; + assert_limit( + decode_with(&record, parser_limits, &mut usage).unwrap_err(), + "parser bytes", + cell_bytes, + cell_bytes + 1, + ); + + let mut usage = OriginResourceUsage { + total_owned_bytes: 1, + ..OriginResourceUsage::default() + }; + let total_limits = OriginLimits { + max_total_owned_bytes: cell_bytes, + ..OriginLimits::default() + }; + assert_limit( + decode_with(&record, total_limits, &mut usage).unwrap_err(), + "total owned bytes", + cell_bytes, + cell_bytes + 1, + ); +} + +#[test] +fn rejects_invalid_mixed_prefix_and_nonzero_reserved_prefix() { + let mut invalid = mixed_record(); + invalid.content[0] = 2; + assert!(matches!( + decode(&invalid), + Err(OriginError::CorruptStructure { .. }) + )); + + let mut reserved = mixed_record(); + reserved.content[1] = 1; + assert!(matches!( + decode(&reserved), + Err(OriginError::CorruptStructure { .. }) + )); +} + +#[test] +fn rejects_non_ascii_fixed_and_mixed_text() { + let mut fixed = text_record("ascii"); + fixed.content[0] = 0xff; + assert!(matches!( + decode(&fixed), + Err(OriginError::UnsupportedEncoding { .. }) + )); + + let mut mixed = mixed_record(); + mixed.content[2] = 0xff; + assert!(matches!( + decode(&mixed), + Err(OriginError::UnsupportedEncoding { .. }) + )); +} + +#[test] +fn rejects_unsupported_eight_bit_integer_and_unsigned_flag() { + let eight_bit = RecordBytes { + header: header( + HeaderSpec::fixture_type(TYPE_I32, 1, 0, 1, 1, TERTIARY_NUMERIC), + "EightBit", + ), + content: vec![1], + }; + assert!(matches!( + decode(&eight_bit), + Err(OriginError::UnsupportedFeature { .. }) + )); + + let mut unsigned_spec = HeaderSpec::fixture_type(TYPE_I32, 1, 0, 1, 4, TERTIARY_NUMERIC); + unsigned_spec.unsigned_flag = 8; + let unsigned = RecordBytes { + header: header(unsigned_spec, "Unsigned"), + content: 345_i32.to_le_bytes().to_vec(), + }; + assert!(matches!( + decode(&unsigned), + Err(OriginError::UnsupportedFeature { .. }) + )); +} + +#[test] +fn rejects_unverified_secondary_and_tertiary_type_fields() { + let mut secondary = HeaderSpec::fixture_type(TYPE_F64, 1, 0, 1, 8, TERTIARY_NUMERIC); + secondary.secondary = 0x91; + let secondary = RecordBytes { + header: header(secondary, "Secondary"), + content: 0.4_f64.to_le_bytes().to_vec(), + }; + assert!(matches!( + decode(&secondary), + Err(OriginError::UnsupportedFeature { .. }) + )); + + let tertiary = RecordBytes { + header: header( + HeaderSpec::fixture_type(TYPE_F64, 1, 0, 1, 8, 0x50ca), + "Tertiary", + ), + content: 0.4_f64.to_le_bytes().to_vec(), + }; + assert!(matches!( + decode(&tertiary), + Err(OriginError::UnsupportedFeature { .. }) + )); +} + +#[test] +fn every_truncated_minimal_record_returns_an_error_without_panicking() { + let records = [ + f64_record(&[0.4, EMPTY_F64]), + f32_record(&[FIXTURE_F32]), + i32_record(&[345, -100_000]), + i16_record(&[34, -1000]), + text_record("test string 123"), + mixed_record(), + ]; + + for record in records { + for end in 0..record.header.len() { + let outcome = catch_unwind(|| { + let mut usage = OriginResourceUsage::default(); + decode_column_record( + OriginProfile::Origin7V552, + &record.header[..end], + Some(&record.content), + &OriginLimits::default(), + &mut usage, + ) + }); + assert!(outcome.is_ok(), "header prefix {end} panicked"); + assert!(outcome.unwrap().is_err(), "header prefix {end} succeeded"); + } + + for end in 0..record.content.len() { + let outcome = catch_unwind(|| { + let mut usage = OriginResourceUsage::default(); + decode_column_record( + OriginProfile::Origin7V552, + &record.header, + Some(&record.content[..end]), + &OriginLimits::default(), + &mut usage, + ) + }); + assert!(outcome.is_ok(), "content prefix {end} panicked"); + assert!(outcome.unwrap().is_err(), "content prefix {end} succeeded"); + } + } +} diff --git a/crates/io/src/origin/opju.rs b/crates/io/src/origin/opju.rs new file mode 100644 index 0000000..2814ed6 --- /dev/null +++ b/crates/io/src/origin/opju.rs @@ -0,0 +1,11 @@ +use super::{OriginError, OriginProbe, OriginProject}; + +/// OPJU is detection-only until a complete, bounded container profile has +/// public evidence. In particular, this path must not scan markers or attempt +/// to decode records after the validated first line. +pub(super) fn read(_probe: OriginProbe) -> Result { + Err(OriginError::UnsupportedOpjuVariant { + message: "This OPJU file uses a record layout that PlotX does not support yet. No data was imported." + .to_owned(), + }) +} diff --git a/crates/io/src/origin/reader.rs b/crates/io/src/origin/reader.rs new file mode 100644 index 0000000..562bbc9 --- /dev/null +++ b/crates/io/src/origin/reader.rs @@ -0,0 +1,358 @@ +use std::mem::size_of; + +use super::{OriginError, OriginLimits, OriginResourceUsage}; + +const LF: u8 = b'\n'; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum FramedBlock<'a> { + Null { offset: usize }, + Data { offset: usize, payload: &'a [u8] }, +} + +impl FramedBlock<'_> { + pub(super) fn offset(&self) -> usize { + match self { + Self::Null { offset } | Self::Data { offset, .. } => *offset, + } + } +} + +pub(super) struct Reader<'bytes, 'limits> { + bytes: &'bytes [u8], + offset: usize, + limits: &'limits OriginLimits, + usage: OriginResourceUsage, +} + +impl<'bytes, 'limits> Reader<'bytes, 'limits> { + #[cfg_attr(not(test), allow(dead_code))] + pub(super) fn new( + bytes: &'bytes [u8], + limits: &'limits OriginLimits, + ) -> Result { + Self::new_with_parser_bytes(bytes, limits, 0) + } + + pub(super) fn new_with_parser_bytes( + bytes: &'bytes [u8], + limits: &'limits OriginLimits, + initial_parser_bytes: usize, + ) -> Result { + limits.validate()?; + enforce_limit("input bytes", bytes.len(), limits.max_input_bytes)?; + enforce_limit( + "parser bytes", + initial_parser_bytes, + limits.max_parser_bytes, + )?; + let total_owned_bytes = + checked_add(bytes.len(), initial_parser_bytes, "total owned bytes")?; + enforce_limit( + "total owned bytes", + total_owned_bytes, + limits.max_total_owned_bytes, + )?; + + Ok(Self { + bytes, + offset: 0, + limits, + usage: OriginResourceUsage { + input_bytes: bytes.len(), + parser_bytes: initial_parser_bytes, + total_owned_bytes, + ..OriginResourceUsage::default() + }, + }) + } + + pub(super) fn offset(&self) -> usize { + self.offset + } + + pub(super) fn into_usage(self) -> OriginResourceUsage { + self.usage + } + + pub(super) fn read_slice(&mut self, length: usize) -> Result<&'bytes [u8], OriginError> { + let start = self.offset; + let end = checked_add(start, length, "reader offset")?; + let available = + self.bytes + .len() + .checked_sub(start) + .ok_or(OriginError::ArithmeticOverflow { + resource: "remaining input bytes", + })?; + let slice = self.bytes.get(start..end).ok_or(OriginError::Truncated { + offset: start, + needed: length, + have: available, + })?; + self.offset = end; + Ok(slice) + } + + pub(super) fn read_u8(&mut self) -> Result { + let start = self.offset; + let byte = self + .bytes + .get(start) + .copied() + .ok_or(OriginError::Truncated { + offset: start, + needed: 1, + have: 0, + })?; + self.offset = checked_add(start, 1, "reader offset")?; + Ok(byte) + } + + // Task 3 establishes these checked primitives before record decoding uses + // every width in production; their focused tests keep the staged API live. + #[cfg_attr(not(test), allow(dead_code))] + pub(super) fn read_u16_le(&mut self) -> Result { + Ok(u16::from_le_bytes(self.read_array()?)) + } + + pub(super) fn read_u32_le(&mut self) -> Result { + Ok(u32::from_le_bytes(self.read_array()?)) + } + + #[cfg_attr(not(test), allow(dead_code))] + pub(super) fn read_i16_le(&mut self) -> Result { + Ok(i16::from_le_bytes(self.read_array()?)) + } + + #[cfg_attr(not(test), allow(dead_code))] + pub(super) fn read_i32_le(&mut self) -> Result { + Ok(i32::from_le_bytes(self.read_array()?)) + } + + #[cfg_attr(not(test), allow(dead_code))] + pub(super) fn read_f32_le(&mut self) -> Result { + Ok(f32::from_le_bytes(self.read_array()?)) + } + + #[cfg_attr(not(test), allow(dead_code))] + pub(super) fn read_f64_le(&mut self) -> Result { + Ok(f64::from_le_bytes(self.read_array()?)) + } + + pub(super) fn read_block(&mut self) -> Result, OriginError> { + // OpenOPJ's MIT-licensed reader defines a block as little-endian u32 + // size plus LF, followed (only for nonzero size) by payload plus LF. + // https://github.com/jgonera/openopj/blob/42ddcf1eb3a490744c54fca0a4ed6fe7a5e723ca/lib/OpenOPJ/common.php + let block_offset = self.offset; + let payload_len = self.read_u32_le()?; + self.expect_lf("block-size delimiter")?; + if payload_len == 0 { + return Ok(FramedBlock::Null { + offset: block_offset, + }); + } + + let payload_len = + usize::try_from(payload_len).map_err(|_| OriginError::ArithmeticOverflow { + resource: "block bytes", + })?; + enforce_limit("block bytes", payload_len, self.limits.max_block_bytes)?; + let payload = self.read_slice(payload_len)?; + self.expect_lf("block-payload delimiter")?; + Ok(FramedBlock::Data { + offset: block_offset, + payload, + }) + } + + #[cfg_attr(not(test), allow(dead_code))] + pub(super) fn read_fixed_ascii(&mut self, width: usize) -> Result { + let field_offset = self.offset; + let field = self.read_slice(width)?; + let text_len = field + .iter() + .position(|byte| *byte == 0) + .unwrap_or(field.len()); + let text = field + .get(..text_len) + .ok_or(OriginError::ArithmeticOverflow { + resource: "fixed ASCII field", + })?; + if let Some(relative) = text.iter().position(|byte| !byte.is_ascii()) { + return Err(OriginError::UnsupportedEncoding { + offset: checked_add(field_offset, relative, "text byte offset")?, + encoding: "non-ASCII byte in fixed-width text".to_owned(), + }); + } + enforce_limit("string bytes", text_len, self.limits.max_string_bytes)?; + self.charge_text(text_len)?; + + let mut decoded = String::new(); + decoded + .try_reserve_exact(text_len) + .map_err(|_| OriginError::AllocationFailed { + resource: "decoded text", + requested: text_len, + })?; + let text = std::str::from_utf8(text).map_err(|_| OriginError::UnsupportedEncoding { + offset: field_offset, + encoding: "non-ASCII byte in fixed-width text".to_owned(), + })?; + decoded.push_str(text); + Ok(decoded) + } + + pub(super) fn try_reserve( + &mut self, + values: &mut Vec, + additional: usize, + resource: &'static str, + ) -> Result<(), OriginError> { + let requested_len = checked_add(values.len(), additional, resource)?; + let old_capacity = values.capacity(); + let element_size = size_of::(); + if requested_len <= old_capacity || element_size == 0 { + return Ok(()); + } + + let minimum_delta = requested_len + .checked_sub(old_capacity) + .ok_or(OriginError::ArithmeticOverflow { resource })?; + let minimum_bytes = checked_mul(minimum_delta, element_size, resource)?; + checked_parser_usage(&self.usage, self.limits, minimum_bytes)?; + + // Bound geometric growth by both remaining budgets. The measured + // capacity delta is charged after reserve so allocator rounding cannot + // bypass accounting; an over-budget rounded allocation fails closed. + let available_bytes = self + .limits + .max_parser_bytes + .saturating_sub(self.usage.parser_bytes) + .min( + self.limits + .max_total_owned_bytes + .saturating_sub(self.usage.total_owned_bytes), + ); + let geometric_capacity = if old_capacity == 0 { + requested_len + } else { + old_capacity.checked_mul(2).unwrap_or(requested_len) + }; + let desired_capacity = requested_len.max(geometric_capacity); + let desired_delta = desired_capacity + .checked_sub(old_capacity) + .ok_or(OriginError::ArithmeticOverflow { resource })?; + let affordable_delta = (available_bytes / element_size).min(desired_delta); + let target_capacity = checked_add(old_capacity, affordable_delta, resource)?; + let reserve_additional = target_capacity + .checked_sub(values.len()) + .ok_or(OriginError::ArithmeticOverflow { resource })?; + let planned_delta = target_capacity + .checked_sub(old_capacity) + .ok_or(OriginError::ArithmeticOverflow { resource })?; + let planned_bytes = checked_mul(planned_delta, element_size, resource)?; + values.try_reserve_exact(reserve_additional).map_err(|_| { + OriginError::AllocationFailed { + resource, + requested: planned_bytes, + } + })?; + + let actual_delta = values + .capacity() + .checked_sub(old_capacity) + .ok_or(OriginError::ArithmeticOverflow { resource })?; + let actual_bytes = checked_mul(actual_delta, element_size, resource)?; + self.charge_parser(actual_bytes) + } + + fn read_array(&mut self) -> Result<[u8; N], OriginError> { + let start = self.offset; + let bytes = self.read_slice(N)?; + bytes.try_into().map_err(|_| OriginError::Truncated { + offset: start, + needed: N, + have: bytes.len(), + }) + } + + fn expect_lf(&mut self, field: &'static str) -> Result<(), OriginError> { + let offset = self.offset; + let delimiter = self.read_u8()?; + if delimiter != LF { + return Err(OriginError::CorruptStructure { + offset, + detail: format!("{field} must be LF"), + }); + } + Ok(()) + } + + #[cfg_attr(not(test), allow(dead_code))] + fn charge_text(&mut self, bytes: usize) -> Result<(), OriginError> { + let decoded_text_bytes = + checked_add(self.usage.decoded_text_bytes, bytes, "decoded text bytes")?; + enforce_limit( + "decoded text bytes", + decoded_text_bytes, + self.limits.max_decoded_text_bytes, + )?; + self.charge_parser(bytes)?; + self.usage.decoded_text_bytes = decoded_text_bytes; + Ok(()) + } + + fn charge_parser(&mut self, bytes: usize) -> Result<(), OriginError> { + let (parser_bytes, total_owned_bytes) = + checked_parser_usage(&self.usage, self.limits, bytes)?; + self.usage.parser_bytes = parser_bytes; + self.usage.total_owned_bytes = total_owned_bytes; + Ok(()) + } +} + +fn checked_parser_usage( + usage: &OriginResourceUsage, + limits: &OriginLimits, + bytes: usize, +) -> Result<(usize, usize), OriginError> { + let parser_bytes = checked_add(usage.parser_bytes, bytes, "parser bytes")?; + enforce_limit("parser bytes", parser_bytes, limits.max_parser_bytes)?; + let total_owned_bytes = checked_add(usage.total_owned_bytes, bytes, "total owned bytes")?; + enforce_limit( + "total owned bytes", + total_owned_bytes, + limits.max_total_owned_bytes, + )?; + Ok((parser_bytes, total_owned_bytes)) +} + +pub(super) fn checked_add( + left: usize, + right: usize, + resource: &'static str, +) -> Result { + left.checked_add(right) + .ok_or(OriginError::ArithmeticOverflow { resource }) +} + +pub(super) fn checked_mul( + left: usize, + right: usize, + resource: &'static str, +) -> Result { + left.checked_mul(right) + .ok_or(OriginError::ArithmeticOverflow { resource }) +} + +fn enforce_limit(resource: &'static str, actual: usize, limit: usize) -> Result<(), OriginError> { + if actual > limit { + return Err(OriginError::LimitExceeded { + resource, + limit, + actual, + }); + } + Ok(()) +} diff --git a/crates/io/src/origin/reader_tests.rs b/crates/io/src/origin/reader_tests.rs new file mode 100644 index 0000000..181ee5f --- /dev/null +++ b/crates/io/src/origin/reader_tests.rs @@ -0,0 +1,303 @@ +use super::reader::{FramedBlock, Reader, checked_add, checked_mul}; +use super::{OriginError, OriginLimits}; +use std::mem::size_of; + +#[test] +fn reads_checked_little_endian_primitives() { + let mut bytes = Vec::new(); + bytes.push(0xa5); + bytes.extend_from_slice(&0x1234_u16.to_le_bytes()); + bytes.extend_from_slice(&0x89ab_cdef_u32.to_le_bytes()); + bytes.extend_from_slice(&(-12_345_i16).to_le_bytes()); + bytes.extend_from_slice(&(-123_456_789_i32).to_le_bytes()); + bytes.extend_from_slice(&12.5_f32.to_le_bytes()); + bytes.extend_from_slice(&(-98.25_f64).to_le_bytes()); + let limits = OriginLimits::default(); + let mut reader = Reader::new(&bytes, &limits).unwrap(); + + assert_eq!(reader.read_u8().unwrap(), 0xa5); + assert_eq!(reader.read_u16_le().unwrap(), 0x1234); + assert_eq!(reader.read_u32_le().unwrap(), 0x89ab_cdef); + assert_eq!(reader.read_i16_le().unwrap(), -12_345); + assert_eq!(reader.read_i32_le().unwrap(), -123_456_789); + assert_eq!(reader.read_f32_le().unwrap(), 12.5); + assert_eq!(reader.read_f64_le().unwrap(), -98.25); + assert_eq!(reader.offset(), bytes.len()); +} + +#[test] +fn checked_slice_accepts_exact_end_and_rejects_one_byte_past_end() { + let limits = OriginLimits::default(); + let mut reader = Reader::new(b"abc", &limits).unwrap(); + assert_eq!(reader.read_slice(3).unwrap(), b"abc"); + + assert!(matches!( + reader.read_slice(1), + Err(OriginError::Truncated { + offset: 3, + needed: 1, + have: 0, + }) + )); +} + +#[test] +fn checked_arithmetic_reports_overflow() { + assert!(matches!( + checked_add(usize::MAX, 1, "test offset"), + Err(OriginError::ArithmeticOverflow { + resource: "test offset" + }) + )); + assert!(matches!( + checked_mul(usize::MAX, 2, "test capacity"), + Err(OriginError::ArithmeticOverflow { + resource: "test capacity" + }) + )); +} + +#[test] +fn reads_data_and_null_block_framing() { + let bytes = [ + 3, 0, 0, 0, b'\n', b'a', b'b', b'c', b'\n', 0, 0, 0, 0, b'\n', + ]; + let limits = OriginLimits::default(); + let mut reader = Reader::new(&bytes, &limits).unwrap(); + + assert!(matches!( + reader.read_block().unwrap(), + FramedBlock::Data { offset: 0, payload } if payload == b"abc" + )); + assert!(matches!( + reader.read_block().unwrap(), + FramedBlock::Null { offset: 9 } + )); + assert_eq!(reader.offset(), bytes.len()); +} + +#[test] +fn rejects_bad_block_delimiters_at_their_offsets() { + let limits = OriginLimits::default(); + let mut bad_size = Reader::new(&[1, 0, 0, 0, b'!', b'a', b'\n'], &limits).unwrap(); + assert!(matches!( + bad_size.read_block(), + Err(OriginError::CorruptStructure { offset: 4, .. }) + )); + + let mut bad_payload = Reader::new(&[1, 0, 0, 0, b'\n', b'a', b'!'], &limits).unwrap(); + assert!(matches!( + bad_payload.read_block(), + Err(OriginError::CorruptStructure { offset: 6, .. }) + )); +} + +#[test] +fn rejects_oversized_declared_block_before_slicing() { + let limits = OriginLimits { + max_block_bytes: 2, + ..OriginLimits::default() + }; + let mut reader = Reader::new(&[3, 0, 0, 0, b'\n'], &limits).unwrap(); + + assert!(matches!( + reader.read_block(), + Err(OriginError::LimitExceeded { + resource: "block bytes", + limit: 2, + actual: 3, + }) + )); +} + +#[test] +fn parser_budget_is_checked_before_vec_reserve() { + let limits = OriginLimits { + max_parser_bytes: 3, + ..OriginLimits::default() + }; + let mut reader = Reader::new(&[], &limits).unwrap(); + let mut values = Vec::::new(); + + assert!(matches!( + reader.try_reserve(&mut values, 2, "test values"), + Err(OriginError::LimitExceeded { + resource: "parser bytes", + limit: 3, + actual: 4, + }) + )); + assert_eq!(values.capacity(), 0); +} + +#[test] +fn repeated_single_item_reservations_grow_logarithmically_and_charge_capacity() { + let limits = OriginLimits::default(); + let mut reader = Reader::new(&[], &limits).unwrap(); + let mut values = Vec::::new(); + let mut capacity_changes = 0_usize; + + for value in 0..4096_u64 { + let previous_capacity = values.capacity(); + reader + .try_reserve(&mut values, 1, "test reader values") + .unwrap(); + if values.capacity() != previous_capacity { + capacity_changes += 1; + } + values.push(value); + } + + let usage = reader.into_usage(); + assert!( + capacity_changes <= 16, + "single-item appends reallocated {capacity_changes} times" + ); + assert_eq!(usage.parser_bytes, values.capacity() * size_of::()); + assert_eq!(usage.total_owned_bytes, usage.parser_bytes); +} + +#[test] +fn spare_vector_capacity_is_not_charged_as_a_new_reader_allocation() { + let limits = OriginLimits::default(); + let mut reader = Reader::new(&[], &limits).unwrap(); + let mut values = Vec::::with_capacity(8); + let original_capacity = values.capacity(); + + reader + .try_reserve(&mut values, 1, "test reader values") + .unwrap(); + + let usage = reader.into_usage(); + assert_eq!(values.capacity(), original_capacity); + assert_eq!(usage.parser_bytes, 0); + assert_eq!(usage.total_owned_bytes, 0); +} + +#[test] +fn reader_charges_the_actual_capacity_delta_with_an_unbounded_budget() { + let limits = OriginLimits { + max_parser_bytes: usize::MAX, + max_total_owned_bytes: usize::MAX, + ..OriginLimits::default() + }; + let mut reader = Reader::new(&[], &limits).unwrap(); + let mut values = vec![0_u8; 8]; + let original_capacity = values.capacity(); + + reader + .try_reserve(&mut values, 1, "test reader values") + .unwrap(); + + let usage = reader.into_usage(); + assert!(values.capacity() > original_capacity); + assert_eq!(usage.parser_bytes, values.capacity() - original_capacity); + assert_eq!(usage.total_owned_bytes, usage.parser_bytes); +} + +#[test] +fn decoded_text_budget_is_checked_before_string_reserve() { + let limits = OriginLimits { + max_decoded_text_bytes: 3, + ..OriginLimits::default() + }; + let mut reader = Reader::new(b"text", &limits).unwrap(); + + assert!(matches!( + reader.read_fixed_ascii(4), + Err(OriginError::LimitExceeded { + resource: "decoded text bytes", + limit: 3, + actual: 4, + }) + )); +} + +#[test] +fn impossible_capacity_requests_return_errors_without_panicking() { + let limits = OriginLimits { + max_parser_bytes: usize::MAX, + max_total_owned_bytes: usize::MAX, + ..OriginLimits::default() + }; + let result = std::panic::catch_unwind(|| { + let mut reader = Reader::new(&[], &limits)?; + let mut wide = Vec::::new(); + reader.try_reserve(&mut wide, usize::MAX, "wide values")?; + Ok::<(), OriginError>(()) + }); + assert!(result.is_ok()); + assert!(matches!( + result.unwrap(), + Err(OriginError::ArithmeticOverflow { .. }) + | Err(OriginError::LimitExceeded { .. }) + | Err(OriginError::AllocationFailed { .. }) + )); + + let allocation = std::panic::catch_unwind(|| { + let mut reader = Reader::new(&[], &limits)?; + let mut bytes = Vec::::new(); + reader.try_reserve(&mut bytes, isize::MAX as usize, "huge byte buffer")?; + Ok::<(), OriginError>(()) + }); + assert!(allocation.is_ok()); + assert!(matches!( + allocation.unwrap(), + Err(OriginError::AllocationFailed { .. }) | Err(OriginError::ArithmeticOverflow { .. }) + )); +} + +#[test] +fn every_truncated_block_prefix_returns_a_structured_error() { + let complete = [1, 0, 0, 0, b'\n', b'x', b'\n']; + let limits = OriginLimits::default(); + + for prefix_len in 0..complete.len() { + let result = std::panic::catch_unwind(|| { + let mut reader = Reader::new(&complete[..prefix_len], &limits)?; + reader.read_block() + }); + assert!(result.is_ok(), "prefix {prefix_len} panicked"); + assert!(matches!( + result.unwrap(), + Err(OriginError::Truncated { .. }) + )); + } +} + +#[test] +fn fixed_ascii_trims_only_in_field_nul_padding() { + let limits = OriginLimits::default(); + let mut reader = Reader::new(b"abc\0\0next", &limits).unwrap(); + + assert_eq!(reader.read_fixed_ascii(5).unwrap(), "abc"); + assert_eq!(reader.offset(), 5); +} + +#[test] +fn fixed_ascii_rejects_non_ascii_before_nul_but_ignores_bytes_after_nul() { + let limits = OriginLimits::default(); + let mut invalid = Reader::new(&[b'a', 0xff, 0, b'x'], &limits).unwrap(); + assert!(matches!( + invalid.read_fixed_ascii(4), + Err(OriginError::UnsupportedEncoding { offset: 1, .. }) + )); + + let mut padded = Reader::new(&[b'o', b'k', 0, 0xff], &limits).unwrap(); + assert_eq!(padded.read_fixed_ascii(4).unwrap(), "ok"); +} + +#[test] +fn reader_rejects_zero_custom_limits() { + let limits = OriginLimits { + max_block_bytes: 0, + ..OriginLimits::default() + }; + assert!(matches!( + Reader::new(&[], &limits), + Err(OriginError::InvalidLimit { + name: "max_block_bytes", + .. + }) + )); +} diff --git a/crates/io/src/origin/tests.rs b/crates/io/src/origin/tests.rs new file mode 100644 index 0000000..af68b45 --- /dev/null +++ b/crates/io/src/origin/tests.rs @@ -0,0 +1,716 @@ +use super::*; + +const OPENOPJ_FIXTURE: &[u8] = include_bytes!("../../tests/fixtures/origin/test-origin-7.0552.opj"); + +fn long_leading_zero_opju_header() -> Vec { + let mut bytes = b"CPYUA ".to_vec(); + bytes.extend(std::iter::repeat_n(b'0', 160)); + bytes.extend_from_slice(b"4."); + bytes.extend(std::iter::repeat_n(b'0', 160)); + bytes.extend_from_slice(b"3668 "); + bytes.extend(std::iter::repeat_n(b'0', 160)); + bytes.extend_from_slice(b"178\n"); + bytes +} + +fn supported_opj_probe_bytes() -> Vec { + const ORIGIN_HEADER_LEN: usize = 39; + let mut bytes = b"CPYA 4.2673 552#\n".to_vec(); + bytes.extend_from_slice(&(ORIGIN_HEADER_LEN as u32).to_le_bytes()); + bytes.push(b'\n'); + let mut header = [0_u8; ORIGIN_HEADER_LEN]; + header[0x1b..0x23].copy_from_slice(&7.0552_f64.to_le_bytes()); + bytes.extend_from_slice(&header); + bytes.push(b'\n'); + bytes.extend_from_slice(&[0, 0, 0, 0, b'\n']); + bytes +} + +#[test] +fn probes_opj_and_opju_by_content() { + let opj_bytes = supported_opj_probe_bytes(); + let opj = probe_origin(&opj_bytes).unwrap(); + assert_eq!(opj.format, OriginFormat::Opj); + assert_eq!(opj.profile, Some(OriginProfile::Origin7V552)); + assert_eq!(opj.support, OriginSupport::Supported); + + let opju = probe_origin(b"CPYUA 4.3668 178\n").unwrap(); + assert_eq!(opju.format, OriginFormat::Opju); + assert_eq!(opju.profile, None); + assert_eq!(opju.support, OriginSupport::RecognizedUnsupported); +} + +#[test] +fn parses_header_components_without_floating_point() { + let opj_bytes = supported_opj_probe_bytes(); + let opj = probe_origin(&opj_bytes).unwrap(); + assert_eq!(opj.raw_version, "4.2673 552"); + assert_eq!(opj.version.major, 4); + assert_eq!(opj.version.minor, 2673); + assert_eq!(opj.version.build, 552); + assert_eq!(opj.byte_order, OriginByteOrder::LittleEndian); + + let opju = probe_origin(b"CPYUA 4.3668 178\n").unwrap(); + assert_eq!(opju.raw_version, "4.3668 178"); + assert_eq!(opju.version.major, 4); + assert_eq!(opju.version.minor, 3668); + assert_eq!(opju.version.build, 178); + assert_eq!(opju.byte_order, OriginByteOrder::LittleEndian); +} + +#[test] +fn classic_version_line_without_initial_framing_is_truncated() { + assert!(matches!( + probe_origin(b"CPYA 4.2673 552#\n"), + Err(OriginError::Truncated { .. }) + )); +} + +#[test] +fn opju_is_recognized_but_not_partially_imported() { + let error = read_origin(b"CPYUA 4.3668 178\nrest", OriginLimits::default()).unwrap_err(); + assert!(matches!(error, OriginError::UnsupportedOpjuVariant { .. })); +} + +#[test] +fn rejects_unknown_or_truncated_headers() { + assert!(matches!( + probe_origin(b"CP"), + Err(OriginError::Truncated { .. }) + )); + assert!(matches!( + probe_origin(b"not an origin file"), + Err(OriginError::UnrecognizedFormat) + )); +} + +#[test] +fn rejects_malformed_classic_version_lines() { + for bytes in [ + b"CPYA 4.2673#\n".as_slice(), + b"CPYA 4.x 552#\n".as_slice(), + b"CPYA 4.2673 build#\n".as_slice(), + b"CPYA 4.2673 552\n".as_slice(), + b"CPYA 4.2673 552##\n".as_slice(), + b"CPYA 4.2673 552#\r\n".as_slice(), + ] { + assert!(matches!( + probe_origin(bytes), + Err(OriginError::MalformedHeader { .. }) + )); + } +} + +#[test] +fn rejects_headers_over_the_default_limit() { + let mut bytes = b"CPYA ".to_vec(); + bytes.resize(129, b'1'); + bytes.push(b'\n'); + + assert!(matches!( + probe_origin(&bytes), + Err(OriginError::HeaderTooLong { limit: 128 }) + )); +} + +#[test] +fn rejects_input_one_byte_over_a_custom_limit() { + let bytes = b"CPYUA 4.3668 178\n"; + let limit = bytes.len() - 1; + let limits = OriginLimits { + max_input_bytes: limit, + ..OriginLimits::default() + }; + + assert!(matches!( + read_origin(bytes, limits), + Err(OriginError::LimitExceeded { + resource: "input bytes", + limit: found_limit, + actual, + }) if found_limit == limit && actual == bytes.len() + )); +} + +#[test] +fn long_opju_version_is_bounded_by_the_string_limit_before_detection_returns() { + let bytes = long_leading_zero_opju_header(); + let limits = OriginLimits { + max_header_bytes: bytes.len(), + max_string_bytes: 64, + ..OriginLimits::default() + }; + + assert!(matches!( + read_origin(&bytes, limits), + Err(OriginError::LimitExceeded { + resource: "string bytes", + limit: 64, + actual, + }) if actual > 64 + )); +} + +#[test] +fn long_opju_version_is_bounded_by_the_parser_limit_before_detection_returns() { + let bytes = long_leading_zero_opju_header(); + let raw_version_len = bytes.len() - b"CPYUA \n".len(); + let limits = OriginLimits { + max_header_bytes: bytes.len(), + max_string_bytes: raw_version_len, + max_parser_bytes: raw_version_len - 1, + ..OriginLimits::default() + }; + + assert!(matches!( + read_origin(&bytes, limits), + Err(OriginError::LimitExceeded { + resource: "parser bytes", + limit, + actual, + }) if limit == raw_version_len - 1 && actual >= raw_version_len + )); +} + +#[test] +fn long_opju_version_is_bounded_by_source_plus_probe_total_bytes() { + let bytes = long_leading_zero_opju_header(); + let raw_version_len = bytes.len() - b"CPYUA \n".len(); + let total_limit = bytes.len() + raw_version_len - 1; + let limits = OriginLimits { + max_header_bytes: bytes.len(), + max_string_bytes: raw_version_len, + max_parser_bytes: raw_version_len, + max_total_owned_bytes: total_limit, + ..OriginLimits::default() + }; + + assert!(matches!( + read_origin(&bytes, limits), + Err(OriginError::LimitExceeded { + resource: "total owned bytes", + limit, + actual, + }) if limit == total_limit && actual > total_limit + )); +} + +#[test] +fn supported_opj_usage_covers_the_retained_probe_allocation() { + let limits = OriginLimits::default(); + let accounted = probe_origin_with_limits(OPENOPJ_FIXTURE, &limits, OPENOPJ_FIXTURE.len()) + .expect("the supported probe must be accounted under the read limits"); + let retained_probe_bytes = accounted.retained_parser_bytes; + assert_eq!(retained_probe_bytes, accounted.probe.raw_version.capacity()); + + let unseeded = opj::read(OPENOPJ_FIXTURE, &limits, accounted.probe, 0).unwrap(); + let project = read_origin(OPENOPJ_FIXTURE, limits).unwrap(); + + assert_eq!(retained_probe_bytes, project.probe.raw_version.capacity()); + assert_eq!(project.resource_usage.input_bytes, OPENOPJ_FIXTURE.len()); + assert_eq!( + project.resource_usage.parser_bytes, + unseeded.resource_usage.parser_bytes + retained_probe_bytes + ); + assert_eq!( + project.resource_usage.total_owned_bytes, + unseeded.resource_usage.total_owned_bytes + retained_probe_bytes + ); +} + +#[test] +fn opju_requires_the_exact_verified_header_grammar() { + for bytes in [ + b"CPYUA 178\n".as_slice(), + b"CPYUA 4.3668\n".as_slice(), + b"CPYUA 4.3668 178#\n".as_slice(), + b"CPYUA four.3668 178\n".as_slice(), + b"CPYUA 4.minor 178\n".as_slice(), + b"CPYUA 4.3668 build\n".as_slice(), + ] { + assert!(matches!( + probe_origin(bytes), + Err(OriginError::MalformedHeader { .. }) + )); + } +} + +#[test] +fn unsupported_classic_versions_are_not_claimed_as_supported() { + assert!(matches!( + probe_origin(b"CPYA 4.2673 551#\n"), + Err(OriginError::UnsupportedVersion { .. }) + )); +} + +#[test] +fn classic_w64_header_is_recognized_as_an_unsupported_version() { + assert_eq!( + probe_origin(b"CPYA 4.3224 220 W64 #\n").unwrap_err(), + OriginError::UnsupportedVersion { + raw_version: "4.3224 220 W64".to_owned(), + } + ); +} + +#[test] +fn default_limits_match_the_public_contract() { + let limits = OriginLimits::default(); + assert_eq!(limits.max_input_bytes, 128 * 1024 * 1024); + assert_eq!(limits.max_header_bytes, 128); + assert_eq!(limits.max_block_bytes, 32 * 1024 * 1024); + assert_eq!(limits.max_string_bytes, 1024 * 1024); + assert_eq!(limits.max_decoded_text_bytes, 32 * 1024 * 1024); + assert_eq!(limits.max_parser_bytes, 128 * 1024 * 1024); + assert_eq!(limits.max_total_owned_bytes, 384 * 1024 * 1024); + assert_eq!(limits.max_workbooks, 256); + assert_eq!(limits.max_window_records, 1024); + assert_eq!(limits.max_worksheets_per_workbook, 128); + assert_eq!(limits.max_columns, 4096); + assert_eq!(limits.max_metadata_records, 65_536); + assert_eq!(limits.max_rows_per_column, 1_000_000); + assert_eq!(limits.max_cells, 2_000_000); + assert_eq!(limits.max_metadata_depth, 32); +} + +#[test] +fn rejects_a_zero_metadata_record_limit() { + let limits = OriginLimits { + max_metadata_records: 0, + ..OriginLimits::default() + }; + assert!(matches!( + read_origin(b"CPYUA 4.3668 178\n", limits), + Err(OriginError::InvalidLimit { + name: "max_metadata_records", + value: 0, + .. + }) + )); +} + +#[test] +fn invalid_custom_limits_return_an_error_without_panicking() { + let limits = OriginLimits { + max_header_bytes: 0, + ..OriginLimits::default() + }; + let result = std::panic::catch_unwind(|| read_origin(b"CPYUA 4.3668 178\n", limits)); + + assert!(result.is_ok()); + assert!(matches!( + result.unwrap(), + Err(OriginError::InvalidLimit { + name: "max_header_bytes", + .. + }) + )); +} + +#[test] +fn rejects_max_string_limit_that_cannot_fit_the_metadata_sentinel() { + let limits = OriginLimits { + max_string_bytes: usize::MAX, + ..OriginLimits::default() + }; + + assert!(matches!( + read_origin(OPENOPJ_FIXTURE, limits), + Err(OriginError::InvalidLimit { + name: "max_string_bytes", + value: usize::MAX, + reason, + }) if reason.contains("sentinel") + )); +} + +#[test] +fn project_notes_keep_distinct_names_and_content() { + let project = OriginProject { + probe: probe_origin(&supported_opj_probe_bytes()).unwrap(), + parameters: Vec::new(), + notes: vec![ + OriginNote { + name: "Methods".to_owned(), + content: "Prepared under nitrogen.".to_owned(), + }, + OriginNote { + name: "Observations".to_owned(), + content: "The solution remained clear.".to_owned(), + }, + ], + workbooks: Vec::new(), + diagnostics: Vec::new(), + unsupported_objects: Vec::new(), + resource_usage: OriginResourceUsage::default(), + }; + + assert_eq!(project.notes.len(), 2); + assert_eq!(project.notes[0].name, "Methods"); + assert_eq!(project.notes[0].content, "Prepared under nitrogen."); + assert_eq!(project.notes[1].name, "Observations"); + assert_eq!(project.notes[1].content, "The solution remained clear."); +} + +#[test] +fn enforces_the_header_limit_at_the_lf_byte_boundary() { + let mut accepted = b"CPYUA ".to_vec(); + accepted.extend(std::iter::repeat_n(b'0', 111)); + accepted.extend_from_slice(b"4.3668 178\n"); + assert_eq!(accepted.len(), 128); + + let probe = probe_origin(&accepted).unwrap(); + assert_eq!(probe.format, OriginFormat::Opju); + assert_eq!(probe.version.major, 4); + assert_eq!(probe.version.minor, 3668); + assert_eq!(probe.version.build, 178); + + let mut too_long = b"CPYUA ".to_vec(); + too_long.extend(std::iter::repeat_n(b'0', 112)); + too_long.extend_from_slice(b"4.3668 178\n"); + assert_eq!(too_long.len(), 129); + assert!(matches!( + probe_origin(&too_long), + Err(OriginError::HeaderTooLong { limit: 128 }) + )); +} + +#[test] +fn rejects_complete_magic_with_an_invalid_following_byte() { + for bytes in [ + b"CPYAB 4.2673 552#\n".as_slice(), + b"CPYUAX 4.3668 178\n".as_slice(), + ] { + assert!(matches!( + probe_origin(bytes), + Err(OriginError::MalformedHeader { .. }) + )); + } +} + +#[test] +fn parses_numeric_maxima_and_rejects_integer_overflow() { + let opju = probe_origin(b"CPYUA 65535.65535 4294967295\n").unwrap(); + assert_eq!(opju.version.major, u16::MAX); + assert_eq!(opju.version.minor, u16::MAX); + assert_eq!(opju.version.build, u32::MAX); + + assert!(matches!( + probe_origin(b"CPYA 65535.65535 4294967295#\n"), + Err(OriginError::UnsupportedVersion { .. }) + )); + + for bytes in [ + b"CPYUA 65536.1 1\n".as_slice(), + b"CPYUA 1.65536 1\n".as_slice(), + b"CPYUA 1.1 4294967296\n".as_slice(), + ] { + let result = std::panic::catch_unwind(|| probe_origin(bytes)); + assert!(result.is_ok()); + assert!(matches!( + result.unwrap(), + Err(OriginError::MalformedHeader { .. }) + )); + } +} + +mod origin9_profile { + use super::*; + + const SIGNATURE: &[u8] = b"CPYA 4.3268 195 W64 #\n"; + const ORIGIN_HEADER_LEN: usize = 115; + + fn initial_structure(header_len: usize, embedded_version: f64) -> Vec { + let mut bytes = SIGNATURE.to_vec(); + bytes.extend_from_slice(&u32::try_from(header_len).unwrap().to_le_bytes()); + bytes.push(b'\n'); + let mut header = vec![0; header_len]; + header[0x1b..0x23].copy_from_slice(&embedded_version.to_le_bytes()); + bytes.extend_from_slice(&header); + bytes.push(b'\n'); + bytes.extend_from_slice(&[0, 0, 0, 0, b'\n']); + bytes + } + + fn assert_structured_probe_error(bytes: &[u8]) { + let result = std::panic::catch_unwind(|| probe_origin(bytes)); + let probe = result.expect("probing an invalid Origin 9 prefix must not panic"); + if let Ok(probe) = probe { + assert_ne!(probe.profile, Some(OriginProfile::Origin7V552)); + panic!("an invalid Origin 9 prefix must return a structured error"); + } + } + + #[test] + fn recognizes_exact_origin9_profile() { + let bytes = initial_structure(ORIGIN_HEADER_LEN, 9.510195); + + let probe = probe_origin(&bytes).unwrap(); + + assert_eq!(probe.profile, Some(OriginProfile::Origin9V951)); + assert_eq!(probe.support, OriginSupport::Supported); + assert_eq!(probe.raw_version, "4.3268 195 W64"); + } + + #[test] + fn rejects_114_byte_global_header() { + assert_structured_probe_error(&initial_structure(114, 9.510195)); + } + + #[test] + fn rejects_116_byte_global_header() { + assert_structured_probe_error(&initial_structure(116, 9.510195)); + } + + #[test] + fn rejects_changed_embedded_version() { + assert_structured_probe_error(&initial_structure(ORIGIN_HEADER_LEN, 9.510194)); + } + + #[test] + fn rejects_every_truncated_initial_structure_prefix() { + let bytes = initial_structure(ORIGIN_HEADER_LEN, 9.510195); + for end in 0..bytes.len() { + assert_structured_probe_error(&bytes[..end]); + } + } +} + +mod origin7_profile { + use super::*; + + const SIGNATURE: &[u8] = b"CPYA 4.2673 552#\n"; + const ORIGIN_HEADER_LEN: usize = 39; + const DATA_HEADER_LEN: usize = 123; + const SIZE_PREFIX_LEN: usize = 5; + const NULL_BLOCK_LEN: usize = 5; + + fn push_block(bytes: &mut Vec, payload: Option<&[u8]>) { + let payload = payload.unwrap_or_default(); + let length = u32::try_from(payload.len()).unwrap(); + bytes.extend_from_slice(&length.to_le_bytes()); + bytes.push(b'\n'); + if !payload.is_empty() { + bytes.extend_from_slice(payload); + bytes.push(b'\n'); + } + } + + fn origin_header() -> [u8; ORIGIN_HEADER_LEN] { + let mut header = [0; ORIGIN_HEADER_LEN]; + header[0x1b..0x23].copy_from_slice(&7.0552_f64.to_le_bytes()); + header + } + + fn data_header() -> [u8; DATA_HEADER_LEN] { + [0; DATA_HEADER_LEN] + } + + fn synthetic_project(contents: &[Option<&[u8]>]) -> Vec { + let mut bytes = SIGNATURE.to_vec(); + push_block(&mut bytes, Some(&origin_header())); + push_block(&mut bytes, None); + for content in contents { + push_block(&mut bytes, Some(&data_header())); + push_block(&mut bytes, *content); + push_block(&mut bytes, None); + } + push_block(&mut bytes, None); + bytes + } + + fn first_data_header_offset() -> usize { + SIGNATURE.len() + SIZE_PREFIX_LEN + ORIGIN_HEADER_LEN + 1 + NULL_BLOCK_LEN + } + + #[test] + fn accepts_exact_header_and_empty_data_list_framing() { + let bytes = synthetic_project(&[]); + let probe = probe_origin(&bytes).unwrap(); + assert_eq!(probe.profile, Some(OriginProfile::Origin7V552)); + assert!(matches!( + read_origin(&bytes, OriginLimits::default()), + Err(OriginError::NoSupportedWorksheet) + )); + } + + #[test] + fn framed_data_without_required_metadata_is_truncated() { + let bytes = synthetic_project(&[Some(b"values"), None]); + assert!(matches!( + read_origin(&bytes, OriginLimits::default()), + Err(OriginError::Truncated { .. }) + )); + } + + #[test] + fn rejects_other_producer_version() { + let mut bytes = synthetic_project(&[]); + bytes[14] = b'1'; + assert!(matches!( + read_origin(&bytes, OriginLimits::default()), + Err(OriginError::UnsupportedVersion { .. }) + )); + } + + #[test] + fn rejects_wrong_origin_header_length() { + let mut bytes = synthetic_project(&[]); + bytes[SIGNATURE.len()..SIGNATURE.len() + 4] + .copy_from_slice(&(ORIGIN_HEADER_LEN as u32 - 1).to_le_bytes()); + let final_header_byte = SIGNATURE.len() + SIZE_PREFIX_LEN + ORIGIN_HEADER_LEN - 1; + bytes.remove(final_header_byte); + assert!(matches!( + read_origin(&bytes, OriginLimits::default()), + Err(OriginError::CorruptStructure { + offset, + .. + }) if offset == SIGNATURE.len() + )); + } + + #[test] + fn rejects_wrong_embedded_origin_version() { + let mut bytes = synthetic_project(&[]); + let version_offset = SIGNATURE.len() + SIZE_PREFIX_LEN + 0x1b; + bytes[version_offset..version_offset + 8].copy_from_slice(&7.0551_f64.to_le_bytes()); + assert!(matches!( + read_origin(&bytes, OriginLimits::default()), + Err(OriginError::UnsupportedVersion { .. }) + )); + } + + #[test] + fn rejects_bad_origin_header_size_delimiter_with_offset() { + let mut bytes = synthetic_project(&[]); + let delimiter = SIGNATURE.len() + 4; + bytes[delimiter] = b'!'; + assert!(matches!( + read_origin(&bytes, OriginLimits::default()), + Err(OriginError::CorruptStructure { offset, .. }) if offset == delimiter + )); + } + + #[test] + fn rejects_bad_origin_header_payload_delimiter_with_offset() { + let mut bytes = synthetic_project(&[]); + let delimiter = SIGNATURE.len() + SIZE_PREFIX_LEN + ORIGIN_HEADER_LEN; + bytes[delimiter] = b'!'; + assert!(matches!( + read_origin(&bytes, OriginLimits::default()), + Err(OriginError::CorruptStructure { offset, .. }) if offset == delimiter + )); + } + + #[test] + fn rejects_declared_oversized_block_before_payload_access() { + let mut bytes = synthetic_project(&[]); + let limits = OriginLimits { + max_block_bytes: ORIGIN_HEADER_LEN, + ..OriginLimits::default() + }; + bytes[SIGNATURE.len()..SIGNATURE.len() + 4] + .copy_from_slice(&(ORIGIN_HEADER_LEN as u32 + 1).to_le_bytes()); + assert!(matches!( + read_origin(&bytes, limits), + Err(OriginError::LimitExceeded { + resource: "block bytes", + limit: ORIGIN_HEADER_LEN, + actual, + }) if actual == ORIGIN_HEADER_LEN + 1 + )); + } + + #[test] + fn rejects_missing_origin_header_null_block() { + let mut bytes = synthetic_project(&[Some(b"value")]); + let header_null = first_data_header_offset() - NULL_BLOCK_LEN; + bytes.drain(header_null..header_null + NULL_BLOCK_LEN); + assert!(matches!( + read_origin(&bytes, OriginLimits::default()), + Err(OriginError::CorruptStructure { offset, .. }) if offset == header_null + )); + } + + #[test] + fn rejects_wrong_data_header_length() { + let mut bytes = synthetic_project(&[Some(b"value")]); + let data_header = first_data_header_offset(); + bytes[data_header..data_header + 4] + .copy_from_slice(&(DATA_HEADER_LEN as u32 - 1).to_le_bytes()); + bytes.remove(data_header + SIZE_PREFIX_LEN + DATA_HEADER_LEN - 1); + assert!(matches!( + read_origin(&bytes, OriginLimits::default()), + Err(OriginError::CorruptStructure { offset, .. }) if offset == data_header + )); + } + + #[test] + fn rejects_missing_data_content_block_as_truncated() { + let mut bytes = synthetic_project(&[Some(b"value")]); + let content = first_data_header_offset() + SIZE_PREFIX_LEN + DATA_HEADER_LEN + 1; + let content_len = SIZE_PREFIX_LEN + b"value".len() + 1; + bytes.drain(content..content + content_len); + assert!(matches!( + read_origin(&bytes, OriginLimits::default()), + Err(OriginError::Truncated { .. }) + )); + } + + #[test] + fn rejects_missing_per_section_null_as_truncated() { + let mut bytes = synthetic_project(&[Some(b"value")]); + let section_null = first_data_header_offset() + + SIZE_PREFIX_LEN + + DATA_HEADER_LEN + + 1 + + SIZE_PREFIX_LEN + + b"value".len() + + 1; + bytes.drain(section_null..section_null + NULL_BLOCK_LEN); + assert!(matches!( + read_origin(&bytes, OriginLimits::default()), + Err(OriginError::Truncated { .. }) + )); + } + + #[test] + fn rejects_missing_data_list_terminator_as_truncated() { + let mut bytes = synthetic_project(&[]); + bytes.truncate(bytes.len() - NULL_BLOCK_LEN); + assert!(matches!( + read_origin(&bytes, OriginLimits::default()), + Err(OriginError::Truncated { .. }) + )); + } + + #[test] + fn rejects_the_4097th_raw_data_section() { + let contents = vec![None::<&[u8]>; 4097]; + let bytes = synthetic_project(&contents); + + assert!(matches!( + read_origin(&bytes, OriginLimits::default()), + Err(OriginError::LimitExceeded { + resource: "data sections", + limit: 4096, + actual: 4097, + }) + )); + } + + #[test] + fn every_truncated_project_prefix_returns_a_structured_error() { + let complete = synthetic_project(&[]); + for prefix_len in 0..complete.len() { + let result = std::panic::catch_unwind(|| { + read_origin(&complete[..prefix_len], OriginLimits::default()) + }); + assert!(result.is_ok(), "prefix {prefix_len} panicked"); + assert!(matches!( + result.unwrap(), + Err(OriginError::Truncated { .. }) + )); + } + } +} diff --git a/crates/io/tests/fixtures/origin/OPENOPJ-LICENSE.txt b/crates/io/tests/fixtures/origin/OPENOPJ-LICENSE.txt new file mode 100644 index 0000000..b49c2ba --- /dev/null +++ b/crates/io/tests/fixtures/origin/OPENOPJ-LICENSE.txt @@ -0,0 +1,20 @@ +Copyright (c) 2012 Juliusz Gonera, Minor Laboratory, University of Virginia + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/crates/io/tests/fixtures/origin/README.md b/crates/io/tests/fixtures/origin/README.md new file mode 100644 index 0000000..22a666a --- /dev/null +++ b/crates/io/tests/fixtures/origin/README.md @@ -0,0 +1,38 @@ +# Public Origin project fixtures + +This directory contains only publicly redistributed Origin project files used as +test fixtures. Neither fixture contains PlotX user data. + +## `test-origin-7.0552.opj` + +- Source URL: + +- Original filename: `test.opj` +- Byte length: 282,034 bytes +- SHA-256: `ac7f71c367562e85e9d4bb4ae418cbcaaa1b5dff80436180e8d3331c7e1d6308` +- License: MIT; the complete license is in + [`OPENOPJ-LICENSE.txt`](OPENOPJ-LICENSE.txt). +- Attribution: Copyright (c) 2012 Juliusz Gonera, Minor Laboratory, University + of Virginia. The fixture comes from the OpenOPJ project. + +The downloaded file content is unchanged; only its repository filename differs +from the original. + +## `RawData_Locust_Revision1_TIS_Mechanism.opju` + +- Source URL: +- Original filename: `RawData_Locust_Revision1_TIS_Mechanism.opju` +- Byte length: 64,954 bytes +- SHA-256: `13c47a6a5daaf14493da59c8f1b284d9efb08129c8320b6ad9fd0b5191faa55f` +- Figshare record and DOI: + +- License: Creative Commons Attribution 4.0 International (CC BY 4.0), + . +- Attribution: Aleksandar Opancar, Petra Ondrackova, David Rose, Jan Trajlinek, + Vedran Derek, and Eric Glowacki, "The same biophysical mechanism is involved + in both temporal interference and direct kHz stimulation of peripheral + nerves," Figshare dataset (2025), DOI 10.6084/m9.figshare.28535426.v1. + +The fixture is redistributed unchanged. CC BY 4.0 permits sharing and +adaptation provided appropriate credit is given, a link to the license is +included, and changes are indicated. diff --git a/crates/io/tests/fixtures/origin/RawData_Locust_Revision1_TIS_Mechanism.opju b/crates/io/tests/fixtures/origin/RawData_Locust_Revision1_TIS_Mechanism.opju new file mode 100644 index 0000000000000000000000000000000000000000..9b47048a145df9268347b9709bf4a42f4e6bafbb GIT binary patch literal 64954 zcmeGE1zZ(t*Z+^t-gLJhAt(Y$w{&+1QqqWkNJ}FK5>iqMA_5X34U$TyN=kP~skCgm z5q`toSm*ZK=Xmb#{r^AD>-Fq&*u%`8z24SF{xML2>%15M(4ZghOS1t$3>pA5@c;n#Z2)-A2!6r| z0JrtP5C7)N4Lq;nfYbyY0R@3?-W=dTh&N=<8z~tN5`cX63KGpd z2zmf?Mbr=c@rPi>V5-X~*_k*yIvI(FLR;|@$jIRK06_U<;=zV9H6`EDbGA&Gw$5i(JvS%AQN3ty zML)^grW)?pR{xlWA_mjedpQ&#f!Rb`;8I3?`Y!~Y`MLXk7AX0%z{~J90|3?L%Y&d1 zl=GM7jM2aAE{S#Do(hj=`0#-a2*1JV&IBjqgy@a!3S$`!Z`hG7UyRE7{<+W zWL`U78iFAb=$*%UYAd9!S=G|v$JB!c$|fe`rAWlG%5KF%R(Ed2>2MD|(3@ z_9allo;`6|3@gP&i9?^ReU1cMyu$b1bb*c8LC!er4(hC@D34)_Z*MuuD9p76F|?X%QD?Lh@0)_MTjS6ck2KlSIxob{--I>vfw7VBSJEXV)HX z8>0Hjm;Y7czj(@z8YTZsBj@$AjOR^Er@xQ0MCXnES<82#t1lm!l|rasqdJMrj2)Q% zY&~)xBluv9GQs6T3M832FXcn((PydPP6HLA7+rPKphf>_oPSfJ)SqcQ4CbU}?1~fo zuCe6aovvb?cF}e;^`|82q_E4@q!?3fXwjRbPfVQHXPE*|f$PhLJNUP=&qht%LAbf!Ou>_5>c^GA(JCy^WV zP+ZpJU_;}(#>Yk2_PHT%L^1BB*W4b%gFSsq(^XP-=Dys*UU3@+EXqf);bE@`p|z|# zT`TeU(j5_>1Zk8W9g|ARyq^^Qo=0>2PxNSJoJ+rQ^BAi4I#EP`72D6Tptryu+M@>7{?B3YUUqNr;+^eA8)_qUcCOR<_-rZ4;Z zp}~+n9ND-|v8tuuXDVo*dAXJ)5k}OZzif5^;}FWv`T1{n%a0nR{!HVkjz_%>RI4Ap zYy5nr?2P|ni>SRA)pMx>G#H=JbY5~F$(I@DaosR(uLvyZ6^>Z4l zyMey2n|)&@{pbAmn;ND6Orxf2>o8^zO8Iw<7ng9r06rkPYVEa?YKIQPrij+#IwSNU zT~CFh1=nlD21SXel@!r&sA90FT`xjEF$_YxqG|l0pDE%uHOl-^<5!yPzq%A-i8o39 zU1Rw{;Kt}+z38>)tCbD)STJiBMd1C_?=Eeup~jFL+$FmBsknnj z5FN&Czt!1TiZ+yCFUDiS?-fYt(0n$Ku)yIa=+)pbsA`}eVKdcD; z#xVb=QBv$rG={_oK9Ygu2YnxA>7kVk1)WAw)dqe7=Pwwr92V711`dj>Ksm<77#vuT zyGJLMhUgmOl^f^p5qVLLta_=X;M#!<-1-dZw_W;2jZ%N6(S(?KV5KTWLBd2IJG?Pm_-*gB9DlB zOZn4W{LOxp{xgl(_crf2FHEt0*O*p)oTd9khkQ5WpORz1ur9Nz>zazVbuv9_<-zc3 zpmt^(o+eeg6fAyi8pLjxgJ@9#2kL9M`%wHR8fE^d@vBQ)%?6D{7N|&n*9el%X8Fi# zQAzUA)D>Aw7#b@YR@_;n-R_+YvvD%mkYs?Fb`CGoo<2_6Au;S@SLBmbFAPzmuJ9+3 zKf5&lk6h~m!S^{{K~vp>;QY9P##1(SbTV^LmyvXKa4|Cy4}>=Vu#9#(L5W8hZ%I|d zi)1lTz$AtPoIY7b2R#=_#~Vt&y_2w3?B+hMN3kS#={-b!iP02pez_yx0o4D**uk6fF55gL}s#g9H8?7kng~ z9c*b>q>OEh&CS4F@O3502B51>uTl5d z8TE>yU=K3)#m!J<^od%ESx-uc;%+_(%uY11N^A#1O zKN|ac&$nvSXQp0m7F{O@7%>p1f~C!xzC#J(8VS^@yb*!rCCo7@?cqS=zj#bG#~k&& z=VgE0^Q^!Hu=y*e|Jd``;L+orzidaNX6AUy&erkAj{k>kl{s#!?62Eux9havR}yLW zy{#h@cZ@kTn?*Zgctd!N31BFFj)rX#94hu}sd#OuUIRCCz@--fDZC1W_TOvkKb&N; zzn)~n)SyR0ta`cISW&&2##ko-ji zlKWZA<&zn2#ISf&9!-l37XvR$(tq+xY|pM04X!t*Qa5*om4Fg2D!Wa^h3>>r!AZ`*t=)bzdKU6CJYo%U-JWCoD@aFp1pa_+RZ)fz32<*drU%Z z$Wl{9nZ;Hok`yK(u{iW6O+Tegut$)-gEwz@4h&WX6GM=6R$5 zMEPE|Xtx#5+#Zt`EW&G_^BOBYN*}40k~E$d3lpYacH0FhuvD<0g)D~46uZp8fex0_ zeh7B|M4!^H^|3Ue&|$v$_~N_18PU{<8r&+;?AW~6cn>caUAkG**#ZK*8yK_KpJSZB zFf%U8@iYX)a0`kui#Ywjy7+&nPx)tk;Qa*nt0K}^aU{%|^F$xI>enS1=&>%7PYKeh z`$Z3qj$cT)J2Td3OH1I|@UNu&g}+|@S;|*`on(w*J_>vnc`~=a8wp+gTA$e}OGVu# z;|Gl`U0~asNav`r&H$_^7>}>Z+&9@OTcVn|nAa`H#%XL?QM`ld(XRb!M0+KUEeq(jPn-4op?-s4U+r@>9pL9A1jXo)k5^ zVrGWwBZzCQ9S}C0EBK?6!#fHS@OB4T=4WvgmN5?3A`Z(bw^NuKNKPtf_!xMIKyMUg+Q>>O&lz4IUz&LpvUaIfhnZoU}0`y ztL|jy00NhY5aM$YAvO$PlR?hH5j=idMCh!Zq?EXZxE>BB8ygFY`b{%4Ck`4;cJ@mw zmsq$sXgJsfxHxbyaWD-Gz;E&br`WCms>kL40u={yV_OS%GY4b{6OCF#L5ZWR@2{;8Q2u2Z`a&Hs$K}q1P6P5G9hCoCB5#aUF>mvXX1i|-1 zF;bFM=X?ZF41r#NWC7lUXy|*WdFVG$BET&GwA&m86(v~{#~zwKU>b70G%5O8K8 zr$E8!aC`?A853A1MS_4-4E6R71&?A?#>Ip~jvz(gf5^b=%aepBV$r#Q6=EoGh5&%l zTmn$*d>%lKEkK~qvZr7V;DU$w092Zzt3W*#_`3pt8sy^*!7>Nd$tb~n32@)r8~Irr zcrJ+B;1MwZ)!Q2%fFMJVu{psBL<0pVQ1Af?RPYB4A6SQ=gU$h^5dlz}eY{bUu^zzQZb^m;5dVWKqK=E&?W)50?q)nj5B~r$mI;+^Ed-EkYq;4 zloABB_Ddc$crMswlG1OPE$0=);WnFv7>JpMo_3V;XxCI@zx zcP``#HuyU?F+dA>MhvJ@fjj*95PWPrpbG>1T^2wMx#taz{v|*h8Vs=&1hydL;BjF9 zl>&l{1ng}-M?D0RQNcrk0P2vp_c}x!*j{czp#mTR5H(WpI7bkG>;$j_NY54kF+~V! z`9}y<<`qZ^DyR!eBMG4TAVE1%0qR+BjuSu_kMPid6>z#BqtyBMp&lK6K{^5_nG&#d z_DB#}*1^KG8B!gNgaeiY9L#K;Xe7<7tyv+cL1cw-NI3GgE@lo+G#WQ8OsrVNZ~=Z> zR!j;2ACnlcr;)I8Fg0_)0qDTHtHAloc31#p5GVwap>>ctNHSDOX%1M0KroRYSP*cu zKrcjxo|oDO0BqnI%6V0`^Ox}eNId{;2MDpy03zf*07(cqf;#NCc{p(N%d>wG=2iBntjzLNuf@ z^bt&4NGN(47)eGz8$RJsB2rL}5vUA|LBKR94)H?7Eqn+c-~@RZ@P-R%RCN6H7*VyT z9N-Q8`9f&N7gEu<{Opfc@`nC!C2tfFV(ffM-Vszje78sxKg5fbQ*qi_W-oImoZ8$5OKVgI+ zI3xVT2Jb;OXga|L-9G_?$e+Mq>tDbC4i@JiP+ExJ4_J^6J^^+U!U;0N4Uid(ftvsv z4Lm?J@Hj>T4-gGJ04+E-)B_suq>l%Pp#Z;t1wQ~e_&Y3EBVfT6+62PFH$>QjAgCY* zqyl^ZfeYu3apBxASYY!H*l-1b4VnM~1uOt$S)c%{;eN%1Q&0phC<0*q^Zg10QxGv! zkQWvayg&$0Wja8P5drLHFfslKA_)CA5Mk>lL_mYALjQ;e05b9e5ElL^Bb0z?=m90< z7!d#ps5l%50Cq-HkP4a3L#})!0qdVhzz2f%5CF^pPAmZB*8icvH|Y3(jRI)!Z1W2g z(D+5t{<|oEev-lvD1i0@3jE6I-$4P46N?BG_&tliMghzdiwG3>J&V6a0jv{?2o(4| zi@!zz>=TO!6!<-h|7%b{^v_TL`tMKx6TCC#=*05hEFGBqhy))o>;D1?DE>VX{2&;e z@9&|3-e01D=O5AF!T%aGi2I+31}^`H2H&a&e*+D$|06W`v+_ZZ0=#^{@-MVt@|706 zex`*e2-@@iqXl@0;=furz=0>EU(kZee?SYkCs__b3phW}!apq>;GI}R(8BLo{537$ zpIAiD!tYu9H7yXFSVYjm?^*mcEfAhqM9{+TS^Qr^3u1pp3*etw{rOD>MB~BngO}hPXP_C}9{+$T9ekVvacd7e41WcyUY5ndBtO1aha)d1r3P-{ zr=x@C0p|qxTL6FtkOv>baD%+9N7Ihl9%r94BE*(j!xxh~kWEhx5Qg68Bz~ASnXM&e zHC>i{ebJKXLJ(FeZf*|ieZOY|5L~faG?D~p&9oFsGRiWbNC!8)W}j+V z2v60>-$?{y@WMYzFPWNz^MpobT*M$)L=xADj3n}d#`vk7r(-}xVMR-;v!?`db91ke z1$PqF&DGmiWhPj+6$V^cCJaCf7|T6Z3^5c}GW+VAHPN zhLQ>Mg_FLD3gaJOS4d1hFO`djY$mU885>g{SK;BKr)0!sl#ifI_CP^Vk+1Q25f$J$ zTGMKZOA$vCxy%R2?&p@}HQO&QB8|N!G?euSx%GIzt>?8hm2fZbMQxP)g>t)27y{V*14qPTKOO`v*6~MTLv4ZI$=z%!QJ(@XL*|* z(8S-JHuK^Uu^-OX)F3>-plpKFwYFjgJlqk2gr1>n3K?l>q_o;GWx`-8Mosi@e4)pMbckw-ToXcdn&4HaV?QW zd?ug5iPU4%tGU``hJ=f&Oj(pvyUo5f03W#!Y%#-{@aVgJD=?a*nB3ccbA^r z*>mmPyf9CSH*mJoZbsQ(d@8F~=WP7dU$hImvt}?Hd-bL3S-skY$!c8T1p9-GF-udq zU7p94pY%L&Fb_EXq+J@4Y(2{g^Saqcgjc8&^5)i8*O^;8dM34FNTYXm-21A_MA?VU zTlq+Kxc@L}6Z2P1REGDo`?znit-bV5e@)wYO?Fa)EHY8xA^zIfrEh~qyGPVvu!4So z7Q!fccPqQ$o>%mIe8%FHm9|mkn{lQ6$YjKc0tqBVw^0tV5dO8o5v^msi2c|@`LxHE z*PXpFs6u6HZf{DBd{S5C*4|Mqo^JdNJ#JG5cVV?IP2GHTk>yZ?R3*G2&S0T$AK&V%pffMWerdqnw#x+AJY} z{E2p@@+qE-F_e4 zt_Ti4F~PapB34?ES~ED_hVI3%Xq`ul^JAC@RjuzOj9xBWBq4Uoe?;{W71W@{|9|WafR4o_!L`OV7MxpeS$1a%0{!7vnDGpM zHhq$r5xyyY@oP|>{Pl2LIPr`}bcp$WLHZ=9Vt$*NDJi)1a~77?xy$tPp@V)`)N~FU z660wvyvunl(Uhewe$kNRl)&W#Ooi^gA#~s~2GO^Gtsir-sC2gO0s$ro^C^@^(ib!e zw24(QpQ+Ygzan-qwMQIC?rnijkV#tiya_p@t!dH~3^lehH!AX;=BX$&Uoa>fq7p%E zvNuR?9#2F%vYb?ELs{TM2D~6xO}$gKQgDkvvy#Hw(VI>Wvf{O0Jh8$!k{w&TzGhln zqWgNF1KsQDRkf`8*c54hQQz7Nyq`uaRi~21%|!`%+Jz|b0d#-NQz1Q9t{ytxsYy%> zR~{bl9~xZ6N7Rx*Ho=&h~FFgIB6G!v!EV?k^Uz*gS3 zoOhDMfj0_3?Rcx794@&Qz9m17CG?`H{>TTs=F7O9Daq0oJiZf}#G2%~D$lM=imErS z8Q!^N&w=E9{}NjfX10z@xKRgJlmZ&W`!d7eOsbLluuW!~#D{g{6@tj948lu+caYdM z4)^ABET=TC=JOutI7axUh>V*o+}qFU3`DhXyn6Q;#)GwYUWa7G%J0&To*t|znm^Lp z5cl_C=0?jF*wYoPP|_jaq>_Ahs?t0|7=PPj9P18wcB^yjIS&uq1E$+fPSxOGz{Yw(8_A?{{ zIX5|MUNtwoV3EF9?vfKj+wdSd%TmTA7-LYmGA!eoWp#6osxdpX)~Ta>wcy$FcOxd3 zO|ICNrdn0;Gi=V@W7|Grf2WU@iW7h7ZDp=ux%K0-h|xTkeaA7MSc`2Yok}!qij>Cf z@x;KT$=X}GdrO0iU2BNnRbjgm=a@`{s)$=3D1WuP{zt!oewTv+rNXzN?{~ zpVIKMpZtPVjI%-jJ$B!;JWlxM6;zAS5IhIg;-w*$mJa1!e+iTjX~+XvS%!cB)OTsr z;RpIN^=UCH(#(z?xoKN|FWHuD4+krK_eU;RZw^##zvg$GeL2ZjSGPZNd4$i!yR!QQ z#jcGf(jiucS$Y*6j!W#&lcfD!+qQF9s~qKGE_(KB1o`U2>HlfdqmYN^Ck ziMxu<$FV2k!Be5=GX75#a|Dfs@5T-q`qT%FI4Tr`2LjBRk{&^akK)53aH-bA4<6z? za5!9%l8u?slEzTkcoiWnPLLRHU}~3b%YIjHe@t>@-L+)mOPWzq>`{2v^Optbc?G6E ztm>wAoj!NzB8K*&eEjLc(q_G1P2b1)+!Ae`7>uns(s-H->mpHY63&%c0y42Vm(scC z{M9)5Jj_dshErRpx-&TgrTQvlCLDz^wsSV&-8{`r9jBgzF4^N43M674Ng!Oe%3j-*a63f@&@H!NHkZ=htkX;j98F5qrtP}ytu+7>3s zDxh2?PV4EuokPlo@@22+@xYwtVPNy5FcV;M#mBdc`tGl=-+3v!dmLP5_WR7kwt|(D`s?lhQyF5d}UG z=-uTkD0rkulIW9|O{I)SUF+UXZNj7Vsg~ta_x!sS5zzKf5gsUN zYh&DMM6H9` z1^b}p{HC^7e9e`+(TbBJXrvuQ&fH5^J@sOoUbD}QR-_Uc1SxkHt`&AIJW5vgBzls-w{6NXf*pmev}>Kc`b+SfCABZlP>yI{zq#Ul^B5o zN_)$#xH`Qi!i`w+UJ2jDyI-6o>QY`BIlfHxoEjkcvXsy;xV#--5WoK*B)p(dg^ne| zbFdYILCyc`jQrSQ<5;_o7J_=D0oQt}lM@+$Uq~w24i^1-p+O(G;IYN`9M@4}iYIe% z%}P^Z@U+y?T79G%!zGSLD>8ST>)dLyowM&oq**-hI*z8_Geut>|IFi48csLI#gp!j zyw2LlL{EX!75dnX%um8aV6QdzF0yXHJU~OtsLhP&EhHeo!Ocx}m{qzIXi!>WyDf5O zCe7cy^k{1mt+2KOCvkUCx+;R3tfoCfGu!RfA-z~>ch6+_e(jZvxxts$!x^hkoejKR zD9puSbek)PbKqB4aR8spP7S&9jqwv)GGDU9Mm^3N;ACj#GMux^x(uHTp24#nk=#zh zQ8qe~udnvM&dM`1=wEI#FprrQ;V5EvKbYh6wy*v@hDajyEWx~&SeF(#*U|g`LFU65V^fp&D6ANV>K26yl!o! z?<(ZfsCJ~y$uVl5#oBItnz4CLwaeyH{pnoGsIITbxN z;c9-Bf2KdmNsdt~J^S?VGS26W!V!uQRQH}W^zawP$@!i=)5Mxf_Hib!np8DCg&53D zH94|a5(FL>+kh#bi78&J$+ndbe7+Y16X|ogOcqdcK}lfFdeW%!s@(Hzhg@Yrfh>_k znV60YvN=xW_vLnHgkZ^*DDKKFEz`4U)z?NSIuk$Azqq5>ws_?#wN-H zre7G!c_wmi#gwky6^OZ%D1GVGtAy!m=Mhuk9PP)nh)(Ps2I;%4oBIiC9K<7K!Q_$y ziCo!9X3p8gp9pzM+!xo2PQ~!sepp+ph%6FNPJj6DSw70gk!lHY&1e&wk#6}-`|h@8 zE9#pUc@mc1omL_8%4}<99V{+MTDsQ7LWK+nB)r#HE7EKlIEf3F9?g!o&jkCl4&k#? z%iw;+DOx(4AQb|X{DQ!UNLPc)ZQFP&1L+T^Cg4Z}rMd6w z15i&tO3SfQmGr|_=X_MjyI5YvoCej|5M4yaIiSS^eu6AX{n$;|L;daEJC zD4dPPg#rne1K>;bBt6CUkRGiuoy+N`F$T+-9?!pC-V>a-`f6)qL&^DNfH7bABB%7$ z7A4uUnfnf1TofvSk%O9)fh`(4A@`26wvS5Jm8-NG33aDctO(IFbw4`KnOysX_$Y$H zZMDdKkDE(6|7A*f)9WV~v<&dUyW_ji=RPPO+Wat&|b29fcnpAgp3>MeE6xgo6T z+{p06m)f7SHnY>OjxWx06|dNPHAayNzOH3sX?W3tLSFgeUFn-erD|qho%W-N)$!h$e!j+ zN}^L3BA4OGbO*Us%HZJozNLM)$w74|TKX=mkRUxx5*8-(i$t%)tV3}#k z(XVE0tAuV_GbLcsoU-F@$BTZzV7=-o92FgrL_s#Q!iUX4{&;ZiFdd2Pph?Td_Zhb`3TybpYXM=fkIvvzvxJ27Rf^IaS5Rswc! zyHDjMw)*~|c?l`Clf!TD+ON}wSVR>Vo((k@FD}5-9TDr+YK!B1aqxx0yO68eQ_5NM z#1Stnf8bv+Tu#Aj+R`6*9fvi-z;~H)yFtN-g)uY;F^w!zad_%0m+1{hwpNC+J{wZ; z5YpWWXngodm&j!A{#|CC8$FXICmzS?X1i@@Q)?F!=gO+{_}XlT%lfTaf++U-{UDC)vAp@fSc)tgNvf#hrjU*psq6lHdIat}Gfr#BR-PKQQXk%R)_WPiGH zc6ybJA!1ulG-np|vbHkkopU&~B*|e){ow*$Sd>!OHEk2TmXKcSEZQhnd{Lu2xjd+i z%8sY6hNx)I*+~Q*iJg7tv0C5IfDLd)5w7x*`;D+&9ULutefrQ}LQ7v_XykPL@`530 z6o1K{(+=q%BN>E}PGXc>r%ie7DU#IogOEYYc_$R*(Q|c*`wI6jxcLj; z;Y9c$UR9^pxzzYr2}yV+0d6h#;M4=os;D7XcWNpU=1bligeuc-cP1*W`sKV?26L&dZ~&-f~~uGRK~kSLPF8Rv-Wxd{oY zqrRME3pdEg%tc!b%ABrvYL;8y>S}A0m$|a}fm?5(*^+w9tIuxt)og0yB{y|1V-mj? zBB`7c{RJdWLtP_%X!+}m@_q*Nyfd?2H588arOzI>E|s}q9Zjn@K9qRwHFor6t+Mot zE4Y$oO)$K5DSj?;sJx=2FPOeCAh9OPWX|$xn(z%Y=Gp95B5YRnu^$K4(pUVP+KTec zC-Sc$-+H@(R1=wxy*RGZgfBO zhgICmwzYR@aUHEmuA3b8yPErS12&D0)2)5v-`itXO^@T!jiZ#h`D*JhWGSrib?rk$ zed7&#Prony&XbwelpC$cdg}BCQMv23hQn3!Q9Wyxjaz+s9xS)xz1qhwO@8@^#>nVk z(1g-+wRA4rNX{q?FSIvzvNVF$m19FWINgUXF<|~Sv}bbC32Th<0BNvW9_!89yNRSaLeTwY{GNpZz?Z7Dc*k33$)&`>l%8G8%0RBTwODp>5&>QJ%oKi_Z-3fknjNFNR;S3Uunpu{Ikc<@=Jk*TRNCZ~;)~uNZMDBH zh~^FSV(BKnx^IWYVfr9nlO4);eaFQ9%Ie{$U%o%A9afk36kX&tCj+>Q16GWttEaa` z*kB?rqu15-on;lRHbjn=D%HZz=)29i9l{Dwz7E>mWX^s0i%+!0`XkWn@`VcSDjf=c zIQmmbP_SzocY(@AQ}%g@Qou{pGBh`~?=Y(EH7Ymnsm;<{7!^T+e z>D0uxAUJO+PIO1f&cl}Nay z_SmJtNJpc=_I$couZYLII&ZoRlg`GJiZFeCI-9;fIgduB;Z`Atn<+Nsx9{Y5DYbJ8 z?rzyhwV(S-3kIk?ERX1{v4bl)p15)tatzDK@H{!sw9lStXplo#aYJGwR!tTA9!wV- zDE;_>828c*|sI9M21+L2;y)h4L438CZHI{5d${W)a@8!uv4Dg#{+x7fXF1&negJnDb;8_HmjTTFJhnDANkFU&}rdF z`_K2Ma_}sFDMgzQB*z25HSKNEk6@|-myi_s7$tq)cM+_4K%gTFI#}0|tp}djLa4>6 z?z(ehL;Smv+^&0xNfG1gx6Ib-;d5A32hkf1ja@uMDQvs}wvK*xX}^NaaY9+KmwX%8 zlilb-C~Y&MAxm)N$9qt)J%i%Fax7pPFrJ>SpA+oGTr7Qw?gMN+#! zm8z>7%geo`qx^csay9v02EVa`q`?w?u%(6hgs>DMI)3Wuuo^7-BsQfKpV!^iYaiPk zx2^x?hJ&F(-gWGTMlXfppwAyuVjm8Qbg5YijK**!YCSr7Y9kyb9VjU8`og@^JwAG@ zKY!+nY37gEHh5}dB1iiN&8OyOxML64^Lv}5xo9g1fIILt;I7-{j^^yO z)mQ#U2vpcXf-hT*C9R0&oU}+|`6^4QnV54edV8pr7niixHqu_rhDPSj0@sABWUR&= zsbWsQxZUZjf1|)+Wf=iBqMTh(4>hMh^7GT?w0dDK;fTlD$|>L05JFF#huMyJwks#U z2^dHh!N1=u!Wwt_cEZVfV$|f%t`(ZtcqWoGV^~cV_4z09S9?|RV}H5|O1=B}07FKF zRI{mmG&&5Y?q=NuB)sf?^F?_B9YWRnd3oYEIGh)xPnKR1^b}A)QN~e63QW*~g78WqcWRBrKcJu9-pzEH->1Z|u8- zEHut*3=PMEcM%ZMq|c{t#yY5CJ`8F#Z56**QF#$a&dc*gEDU4MZ;VoD^^OfiEfwO; zugy>JcP-*lR?JaU4O?)!EJ#T-yH3=xQ#LjIj?GV>vfj3zodCpEmp)djU(%D*+3+_t z&yTL*+PIt`cgykVPE8()7?^PEN;7SrUiME&zz8Ko$kX}v-74&*#jr0dcKNlb zPr8tAbU3a#CRUCn$9D!thunDcds2^-Be?#fv5}Uh^6l>J8=Ctl*z#01JNWCf0IimU zp@hLwC~r{3bj}l!U{e5T59Bo1t${}p^t4Fc*$Xt^rWP%4)Vt}|7tW$X&my-<;3G?* zKk}tPzlVRpUkd!E0kGe|Zw`6)Y`&~gY+uCD|8(yo-*v8Xeb$3P)AI(I4{>9MJ2{_W z+L@DfJ>ku%rf+JbYrVNgaFz+fR}8%#y=8Q_&_0+m&P%FU0l08CA#wMB%IX7U{A%R; z(*As`wL!bl)tOx^uZGcBrrP}KyYlJJUfq-qddM4ntRb{(iHV_gQSi%vb@6GHm=di7 zen$^gGo`vl9*d0dX6~1#Jj)lBiuaYna5gaqR3`FkTxWuYTdC5WXpWn6H@Yf;nj+@H z^Lg2Oxyat6^YU!1qglE)V+Hqi57Z?o;`3+u8yk$~X*qna=BHzC06>5-{O*=m%bsAq zq0pF;{ef2kY52#W_D>GwD-$U@7u}Nr2d@)x6~{N>?b?KuQBcT3i#`Sf?d{0XF~!Eof|BeU!i5{gnWT6sk~z0VTZ&OfAA*wV6+kA6H)g3uk_^co)t zJQo&ChlOE4fZKwi2XY5n%Y$deNvt~R$L}j&4qzK8oi)`q&BVk|i=gDzDSfAg1pl&; zvxrcu>s_*=J7sOOcfv790K*B<_z9CKkvvxZj8{3#>x=3Z>D9|#Lr&w*72$L-|M z;3HgqJg(2PJMRT*IvqXVrkS9X$b3$RP>#CN@Lex`Tt&x|<6dF-F3i&J0i6r7IQV_3 zPrjbr^MV7~lq%5yHXE>0tKkXdH5aWjcweU8&{P?;D!*b=&|0p%j;M0mAQc*7|k|@(1HzZvp*9uJ;KV9^8njKf~o9-R)5tG{T8sPrE z#k`l}7l7EJC5GJWWnD6j01146(5KTFs%;q|G;h~f3xPaGhOU!_I#^h#IY2c#jk`C3otIWvxeCMHCR;7+RE!?D zOeYpb1!njizNyT5kz9PO=2BWMDZq~H_x150MEl6-jyqgT=U9r5+Hqr`U@$xncpH;p z(|!w}v!b0NBjhr{+RQiD$`7e1-K=B*aga2?eTqR>4!hz3|%T7G*rB78`RH3sYX zJ$RtR630hBtBUDMgD;UxOc;#(O+st`*x}Sdf9P2flB0Yrmp)BItevHg;Wa4Rv+fP3 zo3)VNjSjof`-QowWW0klnZriQLpeN5@lp2%t%Mfgu|E?;madz8BsAmx67(P%P*s0o z&r%s_vczt;aF6IXvL1G5I@*m_3=H8y>T?{u1iSlC6$fp2deRZ!uJD|ey6u@U5og2u zao!<|o%K+GI+&o!eN)?i<#oS#JTEd-Y-&t#>9dY{=0O|j^!b3p1wfSGc#7WJp(`&f z#=5B#j{`AWseD}-GcsJPx;@%f3Pt7A;Z;0}$(eVeDh9TLOSz0zn8rKw%dG-}>A%mk zoa$%kj@ZTdA0A%I71mSsg?C!gXxRFd2quf!l&h3M<>mWLboCwU(Wg@*l zmz_-!5$U<6d0tWGSOkoEG=r4#3N^Jw0b`FAMGs-P3zs?Hs9trogAyAbcxsVmr+!!m zuSRsSQt0{XoC}`zvLDLodmsHWn0BP&EhQTyBn0uJ@Cpg+mqoh3<^~L}?7|9$=HHcK z8D%JKuFVtPH+L&@z7J184Vj(@WgQdb{E4X5DQjcKg}pI$%XhrS936+Mhmd+_S=dWT zYRHI+IfO!4?T(B)W2@0z@HXTC;0-&R$C91CB;d%h=^Ih38K^7s!_(&-7tZebH(8au z9{x$4V@K6TiPS0}*BnD}CRus+2ZvBFQ^ z*K~>P%FyGg$PZeH7ij3gxO+fvcg=Sf+cEla;f=)JunSi^5_z`9=FGSf5&b}ONr-AI zpKD~@WMTXX_*dn#wfoaOehm~gJiVX!-T0iZ&fPC7lcJDR7U^H;?(gz;ewOC%&r~zq z*9#v7+(6kEqtuTbF0?t19R_3H*4Ix7dq$sRnbv)V#?S6BN?&$s6V|F6&)o!k&8ovKQfPP?Fw8G&b>mnQ%i84Y?`TaF{MHu| zvKUgrVl{A*SbVKGI^wsp6^{o*wWA>u5^-`UuIKEmaljYKq(@PK{Th3(W^KR_eI@>S zp7CeX9zBf#g$&E08G-JUh*RJle*(BSJ;d6Q&REtqu@}-LFX@iYbQ@Lb@4eY_cg(n= zEx~!^)y$67 zzJKX!iBrKi*nf4rI|$(zW8UXkirrxPgV}?zcoPHdYr1t%W6K*Q&#JeX!(HG}@NDy- zKYTnN=#RubdMeiMsLkzp-9Yf#7vBwr(vTmyhH2oq5~c$&MvnvQoE)D)Ui~ z)UMZUhcxWPaUX{%@FvE*d_Kh|CTI*sHdAg}DD}9q4N3yadNq%xdoO3mY=<+7_e^W4 za@`pfi#E44ci&ezF;Ta<;88*9jt~#mXxI3tct)w>n>M-l5&hi~jG@q)h1~bss3lxC zbSx*zqw{>3AGbLxs3>m7FQNfIPJ`p}FFf)5|1&U8j5#co>NyaWIO&KPIG;wiq^*NR zuYXP_uMK*eT!ktv6=!jhnFPu2#pOv0#iMCU0fQ!M$-1_(owCPgqQSe1HLfu2CYpZF zc>y$98^j2d=0ClVzr3MW+M-!2?~qnpG{DSA@QUHYeCdo}{hJ)k>;A!Cn7`Dl zwUzo_)Zdmqg9}H8pW-sGvbMT;HLZPbtN2ve(95gl3?vUEYT|}RwLBIULY4)FCUJmzp5Bx1`d&FI06Z{z>eZkXN&v z)!VTlIl>22m!$fdWXW^yo3_AX;}?_t4PV-YkF+>i(j%6Vy05!!XRysZm~NZA3W(io zc=CkbHp*(R-KO!;Dc8(LQ)9+f$`OYHr%Q$de-)b*y`RlqVm0#8s<}HcILr1JHf~zT zwl#dMmicoa9R~xK#@i}qG+e`RibKtvLmys{?`fSlER!Y%f+`Ciu9wObp&|#e32!}V zoR93)U+UT|7Ino|9((m>xs)rgXz}zFil>*;PO+_Quyb+29{w1d6L6+6E>Z92qQmG! zXyB_Ma|R+Rp{Es0-0Z7sEa7RZ@jbY!fj!RqZ`5%8qp@zrUk+UF416c_nx(PiEEp0x z0%dn$cpsDSVs78-Y%Zw{ShJiL^~D2RHr_F1bX-s?T8q^Vo_L^M>}akDexl6Kq&TTXx9 zbn=!J%R-s<%Bp4~!e&BvAMgoTjvW>}k~Gd++*)*ru@-P&sx%WRPg{B%C*0h_H|;eV zgZs(SS*!bvs}+*<8Io5PQKS9J@2!=&A>R^|e#*!;i%Fx&QoiC_nShZmhy~9{Xx=l{ z(Zv-R9Q!=fEm-jhb+%tnT}6t5D~nu24|(wV(@V>mF(3euJ$o8*1?kMmA)YkJjyKr* z{2~__@D`tF54-P!;jn^!NDyZi`lIN@h#I1Q^gbyQq4|vCM>E&Mj5KoIXztx^m!i{7i)8K zSvcf*0z43-uQ|il(6r}cO6chrR5vM4DJN+U;%e8@d}$G+4UFZY536=$GC7c865(ai%wwq?5U8`*r*qzjf|*)%&e?$58fim2BCy-PG*KP`9%bi%bh3uF2VQk z9YZI}4!xAB4+59X>4gi~T!4V5S^-YNwVJ;bmLJ4>J=gEr&8uQuYkFXmPmd@~q_yWq z6;}*;@vQd*&l$e5=c;-h;!3JDQGQA^IV*PDLx%y|Dt5dkxQ>*efVWYkhhU|5H*_GgvH3FSYNB)wdDG&AT3uS5Dxv5t}Z`(f4tV;e( zOg_wFWMVCzXvzHW_@oxGW|ws2o@o)f?n71caEjm%tG4Kat}6JCpB*p1<=){wWgV+- zxjvD#O(0lHJXkFfty@*s`=+h!Ny^+<@qV3arNld?oy(N%nNg<|J<9?HPBo=muf;nm z3AeYEV}Oi}V25%NI( z?r;maOdh&S^GqJ|A2$|qd#R6{Jyfmun|Q1N z+}1GN7+4X$O8flIcHk;JSkKOBQ3)AUE{Lh!gQv#O`(C3i`^5g6|Bt=5fU0uq+J^V0 zB?VNF6qS#vH*i0be>gZa#9|Z2E9J2-I;)@IyBMwy@t4`Y+mu&7 zuc6AmkEA?ZUmthE#f44W=O?=2Dw@kEwFX?)=yLr9+Q6TK`Cmp?GG)!l@d^_h@iO1? z!mI+E+a@1(xfp4R%W!M=uF8OVWQgXtA@g*vwduyzCKS$~e(C7aSOx!M37)6A#E?eE zDc;gR29jcu*~EPWK)&mx{mPH-ZElT=%`HlI(GHM6+ON-SFFO48&m}=jlcueME*xh+ z;V+UHKQPqDkV*ypO`1fl>*a@=R{V(IFpcnX+0s6rIU0UQS_GQE8jS0XZo-3q25`Li zuCk)q@Sx!0DVpyOkfj{bd9zsZ$#XVE02@q9@U!jHL2TseK%Uz*+0xQ12SWZ~Tnu4C zK0>E7mmMAcXR=-0(>&DUSTdnqLCQ1veKB}Y@ z{8MPYXUk@gh(T}E-D(FvJ+h5#nFIys=Wxj9u-x6B*)1${t2G#WRbI;BMk@O6(^fUJ z%Y~Y+s1nws-C8&j{XTa!WK_)=uUcenzPXdKkY4qho0Dr7cTS1S#;PlAa^J=d*b<4% zrnA*1J%ag1I+I}FR{nNdeNWUra#uXq73X$q812D)P@BMm&-rC{@=9ME2Xf}v=&#Hj z**7%}N>~3Mbb=jgRvhZIJnJc{*l6_6`C(-EoacU%ME2nm@Bea6*Kk2{0WeL#TmEPJ-!*A+l6hIN1K|PjQj@ z8%!))3-invUIp33j{R>$a3Ri^T)A(C`K4RQ%WmuC#$u+yL)Oy*+vvQ(TO1tm(b?9J zis(~9T5*v*cli_Joh|Y1;`!^l8%wvn3o_WdeyJZn)NK|ZBR52Z%_R{S=ze{Wm!Gd4 zewRmfJiW^!z3)1*58T|`0G~55?wZ(;Pov(}D72!tDdg9?w(jj+A5o&eR(EPw(_2Q5 z9lR6fK#ZSDevU45R;_b+M^)L-gMQ2{k&`R{O&$d zN$Gl5%%s-~9VljV#(}g5kbCS!*}2^-cf+n;Wl&4$VOHLme$xj0D#b>+x!5S%k8Bb& zzrXkF8rQ*4cHY51E&6-*Mp?M-jJdICmf{!pb9XxzJ9`-*`BH8SB%;j~8dLXj-%P4v zNL;n;jLHIy5g7{f6s_~{In7GLWC;T3e!7t}~hXx5L4!zsZvs=>+d*8Xy3R_6@^;v=m_y0ZUdB{_MQWDtfGMBiMwoCe2 z9snkC_NBOq4){%n;$ zWuWYK`PM|y-E9k$cMicl`uC2v5HbLqKidIOHQ>&$Ddy{Foz;0)1|l(?tdKQyj5%ZE zKa=4`o1HSFN!$+6+zavhBLP1e=EZSX7u{DY(LoZO4Xagg_#DncQB!lbO-Mr>68R>B zfH{?q2P?JfkcNPVeEA25&8{N}ZI06HT~Nv~{uoY?>f!qJprE>#F>GN#GcMzJ-S?Xa z_|^H>;b(RAAlQL=v4?AL?w*J+<6ZZ#`TPYxP`*eZqBRB62*BCwTI%ZFk3w$+0<8k( zeBMpIdi+q@Mi&e7Ij~(%%m)Y~7KxCIG$H+-@ZP=YntLcA9HXmiJdQmHtBC*dr8l!d+gUvL%4FNQPv3ufgWSesmf)yVUkWp4_?L`TdQ+upx`DV81n<|59 zrr3_V9Zq*^SD(rk7{yj77@$VEe_7il%;XxXp}RJp%qVo3PN+5C%YBH2E8)0#?;=$( z?ihcfM1*(qTSih*Vt64~Q}^YxRfBI^-0Wz4mBJO@Ihc^W-cEgx(JDV^he}1+ zl~s$en#$yK*~V0P8uVg}-gT_cC;HNOE7Rc9X|;>*wb@_}-*)=^&3R$_^-94;Y=)nv zmdmSGf+Mut$7Z>>gIC?Sgk+S443h?$&yi3jr;ToKx}Mcf{B6gncvNg>#crwEYTm6{7&%7`$p;9 z^R{Q27PA*RXD2Z_8L&7xU3c1l?(E#VDu6}cz8l2U`3Rp|nc+EIHpilg&I)hsio zM7+AJ+5~6OYFb{nw-`_D7^l;AD%-6(O&$g)P+u7;*GBI4rHNdx1m-5FTTqTW>+~jK z`W(9MT(kUv3{Rvs8xrkX!M2idHrN3%eq+=&v_r&W!|~rofjAA9eU{=!=CsuBNkOLrq`lovmOV&OxoQv# zkB6Qu+f2Nke+TOk%^(I(t;t%vbXDG?zBX9bzDIDPn?Qr+P)Q71ryvx-!$eP5;T;Rk zyP>UOF%kLL3?{$UKHaFB*)pAE4#V(jcv28`FbgV4yNsDFi!DZ{5#K)Cd=}ub9(&#R z5xtlDDrmcD2}?gej%UNN;%vxag;i`t4UcnG|FvE$iMyxqYP}2LYH-ZH(n(oWtzL`4 zWKSZG9P9oX?BvK&i|+JJ22b5~APbXb6IE&=o{g%(t>YYhxh{>Wl5{7@C+@qizPs6J z7CPa2;I`!!$*MBBKjw*lFr(T}wgBt?@o?Z)aL z&V<69-Nw50un1{pa;kDK_=@z(=_)0PgY;ZQp9j51wa;V1hU)1pxNV2HWxC<3?Duf3 zR$Vb)L_V8jX0AF5>fWK+j2!WXe#|nuWtTEh5O$5#R_<=5*PUlHe(MJt+E*qtb_ti_ z;FEn-Kr?3Nuw!pg$VLCz(*}F88o9w z0&%aO!F!_;N^IqM9M1dn#CXA<7+^}&K5khhrO_IKUJSeHzH`VX&&GGm z(7sHCoR)QZ>oK*sCmiui>kbp+SGWP9E@<6r2?MQnFuiWvXrf?I(@=B9a>ah56gWgJ zl64EO(n$)Mu6y#B#cggUhf|l!=f#E$#!H`Cqt?Lz*ps#Up7%WdWZ7+aKnT9YNI5I@ zQAP=}`6}>1Nk7H#yR~)|^KDnhRXKr{U38F$l$5W~4}cUkGc)7mg$mEg&zBGuev5H* zbOeZHGV%G_f+22{e}0gu;97M~nnd?zv!a+ZC|1O;LF@jg$l+mPZvZvtrIuy?0TRuk zsD4lCszW~aquPPm(FAq32ory?IMU9@}ZFCszk0>1vGMm5@S?@nJ zR5}zjenfW%ApuH9)rGx0!)Z4G!H3hz8XhK-gN6BdRYX-TmLt&x)76E}HrKnPtL~MS zi9Ukmk})p(sZRK$Jti*w5ERcjZcZ#8-66E=o!g$~U}?R1ebm%b&QjfbJzuA;>AK&3 z!+zO#nMo0p#IhkeB50mqEyZ#df&By=+*fro&TX(fxC~Z207XtVD?!*-b5sp+ZuRRY zn&vjEX*(>gL!=ZJu6kymhVs#PD+pGbD4lIzE`;6N>GF6`zl>xMS#vW|N#XWR0u4te zk+ahDmE}au1I?qPM7NVlECbERKf-pF{04#Y*M&~8ujYqKq&Pi^jQ;G?o6o%i4ww8sZ{txi$qh=UiyZex0~Vv(!D zlw;n1bnN-K9WX+p-gmuCo+`kk~rhzknj7 zT5@GS=xLzOgRFIsv#hVOYNsh5xHEftm11EcrLMGl?K*R~E$KD{I#}$;3NlG`9F|iVlFvUeM{%)cC6V z*n#|B-4-mY>`Heo+pMFK{y~)HESJrSb~JskTePzYk*+gmf4Rrz$~%X1@li?McEtD_!VmR~+w-ds>bmJb%`UF}=1 zjxO7UKbA(N@nh#w(p+@05p zsS7k57dxm*Xu98s73be%Xh5roWghx=sP5oQ)S(yBP}UWVe&Uc-bvr&o)byQM^hLFV zAeLHg=aIbT_2{|qL0PIGDk46&@y}#c`qK0^8~wC{sEBV%gH~%H{uNZY0rA{e#&5+t zbth37KC?+vwGIZg!+|V26sLtcOo2W#EE(;=)$XR_tk%=k?S7bmi^Pmd$c*>bZ9DvopqvZ0 z2-^ZNzU23i#qjzGKZm*nhsw%>`2^XA51WK8Nw~Q1!Df34uP;hVOYhygC!JDO#!N&6 z6df8Jg*F_?dwd(C!Ge$BkvqT6q3e@8&1&r8harxR`SAIDIL6!iP`Bv{7-xB|_10cI z%f8Bbts{xYwS2patTxpn5x-5Y7kNDk z?|8J7B4L9efAkPM1z7`RinMUR`}-dTl~>)B-~S(@r!q= zuD%gX64Fw3z7v9M5fMB^xE5pz8BqW)!fge#Ggh|spLbU>HFdVzlkM+MG_Gu9WV9>H zh%;=5pTnQ}T75K`|Mo((`3?9%B1vcv%{6h9nf+!rqRJxN<=@9?()DIKdUH3-u}`Y; zPEOH1>c3DT5kR*80vRkKAcOBV<%QS z{Pjsj?1x^`r;!Mu#?xP9%wOP>P9A>$e(-n39Gjv~*j3pXhv{Uh&P?=R?frVtLvC}$ z6!jC`t2hah+&=xGPYrEBc>BK`^BZMRu{dzo*yNztvWL#0jxzdB}0YK#u#94Agzz+uAQnMj^D9${AI7 z=gC?9Ildj*RGiUd*>9{#*x3;w%Kle{p)=V{j_)X zCugD(Va?~y;+j$vWr{!i`m}Pwqn4tG?8eO0X@)OP&*5*5c2k3s*zL!D4FU2UV*8z+ z*vVQ>8un>yG9-^!Jb!CTjcr?^B_>Yu!mrl)Bb^85>XVCX9{$??AialQY8|dzC(1sQlPg^J#Hmm z4S(=s5eVO5;XBbUmH;Wye`?;@Ac}-(mm}RoMJd&-RQuPXo{}CxR3%zwo_3T0GA=D5$7V2>b+7s|xrc3>kAa*uy{EUVz{D3|GH;@JDy&ciNz6 zUwy;{pXLCdcKdqDBgD0Dc>3WNQ3g^u;|#=b*}P7&k`1eD*naF z|3rw`ok{2)L(mL7E!`D1J==F7CY&-?`;CUKFHX8 zrKK9m?3C3Sl6< z{5x^74s#=jY0F7=F=u5O4hKd4H>WJ45vu7kwA2Zot8!Gex6qJ!q%M)#^P9META}V| zpS=}lK3%eh2Q|*O&A!+48zijJ`DIFBZL`|Mk@@0#Aj0}2Ehvbh2=X}MVA*{bc0a8)y5w-zhtIvmA$g;oy5s@wU-M2 ztM~*C4u8Tt`()(SL*uc@hD?%(!niPODIY=49h}`}+rK%(+f79* z*#sokaxkgvt*s}ywtgsD>`7!@|IdZQKS2DT1#1~;=uqlj|CjF(LzD}@9`<6H_)JfD zYj$lQ^r5Cks9h>4Prv1H+HcVMHWezhsZ}Q#kXZ*I1!#zLKmCo5+~zbFSjGK{Cd?WS z@_kS`|F5<89m&(q0eL~0i`S{G@vP3@e(lIiX$&qaDcp-0_CcFmGdWVCx;wA6?H*zM z0%*$gyV) zcK*t$9zJ@6ipRR!5kXNgF;Qx=Q+uLu$>MUdO`H;h&#_>PMXfeg=XTMT%vY#Z`7SqK zwY;ZnYrNP1iO4cRPC=n3noehLq2&~Qr8|zr{qi(C{K2J-FnOj#T$$~f%w@2=ynJ^w z-J+DoIi@>oNlHpeS63JKGaYJFG&D4%q@<4m=8ZY*H-$w+M8w6Lhru7!0wQGiN$uMY`*s;59ccO#xTI& z=y-d3UmS1Yayc{pbuz%&+1c9KT6=ptV1Iw#+S>X}(O7|cwXM38lvG=<{pMImXlQ73 zG=dhG$rwOSPY)Ox8VZk$48mix?oH(4{a`*_wKY{)tXghv25vKVc5V!14}!-hx@@yu zPDcbtaas1xTAK8)7J^Q`?HZJKg!u-{#L_cbXilO%(&ah-j)bDf?pAT!>(m)s6LNuY z?YTCX30ZG_J6|=oJN) zsYtG?&uywZdXK6rk*>G>FCFJyP^eUo@cUL9^-?WVgf5{sM)JW+lNA(%drRdc)?tj3 znPE_3NPVs{y@%En$6_&6VRdTP=kM=N&%l6-g9HB0{=q?#^LjSeSO88=PHb#!MM|cl z1?pfyakSdJ&d%&_@j2|%($b!jkeH8s(f*+y!H?LlWo2bmRM@STzJX8ZA%%i?G_7`v z|3ftH)6W@3U{Q}BKehl}9q%u90*4aE4(esapzVAK)oWG?kqA`l40e)cl*;t?7&7C_ z&{j*_4F)Tx#u?1_liY01LLT7F*yMc8F^VeBxH;Y<4||h;L8E!CKtTv@*TbXWl-O-3 zT$+xyQE*7&P3}uK#Ks=_U9kg{Y8;ESu`x*{n|oIzd+))p>s_07z)n_P6j0;VaV!>2 zxh$#W26-ZpzQ8%32|4T`ZK3IMsWU^G^S|tc1-CDpzKQlTTd=lgif+&(nm|FPfPg?uObiMNN<>6N02a;l)s9C}d_3}#pDwU9QJR~U zwsZozsV&$5{K_8b9|eG;#P)QxLsv9ih7r`a&Q9=}$f&8eM+;}`J)sX~-S_u`k4Z-& zPOh3H%s-?zS_`4RM&jl~Fj(jYUZEG-F%^JaJawD7{3egE+N z2t|#C-bmL9!+q$~M;_OpiRnK3&1eF`C^_4Yc-wTF$-BqH@V&+-2D4ShQY}}tVRaL` z5)99RB9D-4b||dj!P}ntlSr=(`dF8 zLq-cjSgqhk)I<>WI_s{z-`nCo-Na?2({x!&_9n#p?gw3)0Xp4&wn%!j*B+JVk#5!I z-V%460&P*VCkS>qBqSso8yn!}xuag2pN=ZQ3nkg%h4*-1N`VRpM?e^Nk)|)7FRI!2 z*X6Dl1}V-xez21QyAl$^re;#Oy`H+TXd12O8Yj%;?yV6twQgPQr*a*Kz-L@q4U=|~ zO2*BU!S-3Y5AQQY3~y68%*t05N}AYjU%S9BJ;XD%%g(E*vRYAZ$`3CCAR-~<=H@!= z%v`nNS^&nXwg)jcnay!c74XIW(7K&yU9uxf=Fygi6o>#2NeN~Y2puR;=lH9!SEy)c zXh=#*Qa)-L{==p}B6N;>O3v!P|E^o%aQUa^Q{U*}-Zs0T&JkXCjH zmpdM+1;&Jhg>7tXSe4qW_MrDKcSc#2`uO;qpF69M-o`TZ3QKLUV*|h;2ONVO54u?v zvA{P0sb0Lm;dHbF_f+4!c>@+}SsFp17=%riQdj5RVSgnJ9beh+gDv$tX?1gKf~%3`F|&?m-YY8>h<5SdLjHjW%Ysq!qCDT zGl2c(6COI46a4<}2O#WsHN-EUfN0-%VZcA`ed9&_#tZ#96o~MR7X|z>)OSB1Jop9V zZ@fS-@7H%Vq;I^i02wGSCjm?=UGjg+c-#W`hYs zAy`G)3?>YH4g`n*z^_65<99;vZvlwsKzKypDgb~01%rS9cOS4)*$FIw{s4uDfC|3K z52=V8~bq3DR3i@c3t*(7FKKNOWNU_*pLKe+&ZvgW)ta6-A_FQlJokVYh=Z;BVPL zQGp*3{viDdg2KcCFGmx+vK#QvW$@zezTUEdr+9WB$O^Cp#``oeKu_|_G)`mI@(2Uo zLjB`J@Q{h55^{{~e#1MvP^ zXdH-OcrV}`$Jpb)aEyihh0+u(S?#Cf=g*|+o=`oJGql#TF|;t%wEKC-vB;&Fe7zat#`tN8eT6tDkNJPGl?iifaQ{)Y+heoS!3KlaxNV*hah z%byecSU(-DiLmn%89jYHYdv$F|0Ev&hxj`(vcHPg`A6~2KgDA`lBWNaMhJdL6#M|Z z=Lz!*iSK_(e2jf7@fS)U{2^25Uu6E15`uop1S(14ab!QlN&kzu z|FCPO-|qT07(jx}UC8-&-vNjriGOhWok`yQ?H}L%2e|!)8sOEy+hn`+D_H*fCZ+zl$62gn^%}3- zl-3i#LX`Is`yR}-Tkk>QiOmf6&8ks89@MC}NV7~I!Mi6KOtN{fL7$10$^Eo9ptVkg z>%vPYwb+oIqWLLdw}=oH7t6*r zh2)Da9J&@QRNdAws~^|Ady6Fm1C3YwRb}W?$p7%b99p-M7ag>Jpul z$+yhbuV42tw5}ywpYk|CCzqzNm`qxaSM~3 z(QRf*V(wqrYx1pFHHCw{27e}@`O6WI7BC?dFOE3zm57Q>JG3`8kAjCB9ecvA)-xV9 zDcU*l-8kKpisNuKIuCYNR`I8i?~d=o?B&F~p|X6>BQd;7UHs8_TJ@tuw0uO=2E#1w zOS`pAC#=~Bolmdyw)1DKW)=hO=6c&jMUYhq0nne_dPfpa3zF{B1h9DLi*em-L~c>2 zHq#tGpW?bc;^sn3b0zepnK16jXlxVtanheCGC>dXCuw=%-T5BPbmq@fAI6??ukL5N zei*NlgAXTN#4NvbgZ`Jr9jSFmAz$SwI(dz@#>q^6sA zqjZ8Id-pK)rSd`_yRjQQ9pd8IN3e$z_i32$C% zT^8i1zDa2d_1)!+Q*$^OZlRQ{Ng4K9gMnK_{nVFY zqMAH$Cw@rCglD};)#8cBd;}!uVy|2f-p%dfyypcP`iW;G=e*N8mA0+0j;^F#(#9xy zaQ$*MBG)0(_4@JUz3f%g15Sm?SRFqHBt^=2pa=NrdDMlk$R5=193}d5iLnR*=BjNy zA0u7!>C&UfKZF4&lYd=MK*T&hqKXMHpxq(vXr*Q>9q{!~6e(G)+kW!m0*|ElBgaqy zk~p%_F3-B0ddKEHrTTXsHmj?Gqp&1CjKWgJ*2?Bx-Zvs1Q?IfOyM-lF_cxnnpgmaJ zovL4TEmOwk<$kwZrnLc{gLotcg!)__)Q$`Wa>175@_gg(lt3}swKzZUFzfah9!F%y z^hOqN)2x5PcbYVOf9p-(k>%&6o6O7;8Gm}MA|imAm~m|NRP?;`Rhla0YJ^_$7+|t zF71!o)0=Mwi;h#B*OB7n^S7=v3!?2$&b<^8(2+>uYPMJF|RMEj=VGACeDX*KMSR|jb-n5mjL&VMOswU$(H5 z|LU@BYY4r{efwiq7jt@s$GM!~l%Il-3;w(*W{!A_<_ts2^p_{4{a3~4Yz~vND-LEK zn88#U8AFFsNoKm{;+`IZy9LXQe5Jr7BAj8&sFZ8=iOuHnh^}D8a804o*Tp`sWyNjA%n3m0?UT%lV9JNXaDl3cj zrMt4veh~5U7{t1fxC*5aAC`E56Q`myHaMQs`uq*~rROmwY1q3m)3=h!Q_Jz6dr6XN zLVJxU;!CNYUo_q$mcC>*ts=DPqHx6`e6rj%-%q`L6c}5+;6qchwXcEBHeUE%C;HKW z17l_jcK7UbynR(${K*muoBs=zgqp^g2NgAN$s zqpSv*%=WQH6+wJ_PRtZ{5){C(>ygM|zhoX5Yit`YwzLe0-p;Vv{?(0yD_1ck`n7(<37yBxrZ|5n7 z?(s}jSom69o*M6bpAZ(pdiVK z;#DyKS+~0q0<*%j0I~BTmN=tvZZ2!sXn7NKO2Ozx?r?Q^wGf_LR|aFjsWE;DGgqyx zi`rb&f(_OXvx0G1`nQ?44Rm!USKgP@*9}`X^PrG+MmhQWfT=1OnrJijR%|bi4pv2r z$|3Q3D2ehFNqq{Nv}@#eAn3I{J}7UZY=o9PdB!9`xpmwbP!FPYvGKE2GBh5U3|N!N zsv7GKOY7&?*LzrImc6~V6OrIN#uJItthq%ipB?91?|@H3s0kGh*z@?{i+u6(Y2x16#G+J4xC_`+ayY3}l!v-|jq-Ty4e7*FWE(y#jSe$U{6? z){cBle7lO6d-%=|)+rn&@G>mp^Naw}A)_)-$ePJLd&T*aBaaDqCOf<}3hne2B}b_c z`J+?d0jYx*QbHqaroEX0r)Ai;0W&VnPaWmf{r(!ZK=g%G%sAL!R;^-M_ zEpr#2o$;l%GsGqFEU~hII>cp<*QHKkMwR4Se?WTobaih{Z6**5=_RP*c$9V)8nvS% zsjFAx9|tsxb3|fFnv5eq7v^OrG%z}p4^}6ro4%oI4W01h%wXbn@369Rb-H+kls$QK>?n9w+;wk5UwLw0KWqa!wA@ac<9)FqxH9Twmbhbxp5`a>-G-! zp;1JbN6Vgc7k6GLAL{(@%JW9>7A6L%{{E^ z?93x|WgP>9Rb4EhQ>=GS?nV=4)umwzj>i_wZ3WPK9*-;Q16x;pbBbs${XMn<9Ta+ykAJBU6o5>tnb8h!EraSob``vl)H-+{$f=fXp+ zq`kVR3>MHjj-+IJ`sBw-KCf+_t&p|^p^%^zGVdB-4^Fx`XNMQ9u*+lE#o=tiILl!J z?w}p6C%-tYTc+6Rb5(TmzLnk(1!Jn&;Sd!Uy>~0jn)UdDQ!$9wLFY$)`$l?(#E;r8 zBo!@%D?j{ZxiQCmmamY^9Qd|3L-je2E0e=Dudb<_s(izp3%v~Pb#usVvF$=|OjIqV zF}6}ST6R!ON?zT0v@NXS$TfO_+gRMzcynS37m`uH2f1-t8(& z$?${{w}8}GzZT-~oTrV1S7?X7Cj#5r9FUoUxqK0JYg`lC_Ap6n)LU!e8yv~V<$3j#aYsh0SCBviIBAKet>1JiVRHjIP3|MY?s^g= zU7*&CRL33T>WXfi7<%sZ64s2@ki@Yd$xE=hbgayDak7u~Cl0-O)l*5KxJ{II_7 zdzCx&CbiXUc#}7a2wvwkDsqN+-menOATC@*BeBF!p}Xn3wmb(KLPhy6r7ryqIxWH+gAmL|+Z+=(5~H`Uj) zGGI%4+WDh`gSqvCmtuDXOOtO(Q$$mjHxU}R_}pz{bb3sl-aU=zq*FfuvN@WkS>JHK z{%ZO);Mpbkk>P3@UeaXJ<)3EbU1;CIwaQKfQuu`+U79NGdL+N*>T%oIHb(-)w2Sa# zdhzYNW)Lz{nTG?VzOHo_lZ@Aey^%XwU=pD1kB0?4yM>TbOJb*d6mhrmq9L(S+_ogtnn7)<42bG%20U3*;ZvtcmqIysgk9TVXs@pIM%W7kfNWDpml&iU z`C75ZVZ$gtTbe%kTHd|V^s8@{pQGGt_S(=PZvPhopK)w`MP)Gu9kEGlS$jLNn^13t zuY(p8&d4=2q#3iW;%g>F@w9sjXPq2;UPuQ74u9ovLrmeDj(+HSYE3j83wJsW5KaCWYBn zRJmd6e%3J66-5+I{~VeYq}UeL+nzpQY``<1IF-nx+_skNGYFH?XkRUP*rS3K>i%NN zK-Tft8`D-Ls^t>D+lWr^z=0(*VViE1%ZV>2r9;9jp|f+C$OK~Ex^W>hxXal5vlrUY+d!@1onX9G=o`4re% zTyI9CUvHP|1Q7e)LG-ERLnkWY`Y(n%d3=;E-@DpnMU=ZdDANhARnRZd%mp;W?$Nqt zt^?}>4~{w{1G}My4g)@t^qEPoNow1#+JCR4mydAnatgDX=evv7r@8b!!>ZIwE>>mk zEA|X)%T=!F+tJx@$9*xf5=gt+`tiBtYQ`j5z(W(sB;#Z|hhAlt{dsR^b?2mX8sqcy zp0VKB!vOIwIDvz{H~3(Dnh!1+t*wriFV#QeRJLlZjdDA}r{ajL0+)amIRy!r6-P5M zr)-|8I|+X9fcbSu(jVDbra;_QPQ*Rj+%Oq5IZZAh9tKNQ%(}Nokyq%@jpfe8D9p7N z7}92qq=0n`Hyugb8}#KijH5h{&%A&rO)PHY+zzi8dPD`vv2$NI&$dw(4viPv>mFwF z_8o4ISKkNRIm6V;N_XHQJm_ZZc7H&ira!(TsJ3iXam7;`pEgPaWy9}X(Pd{uiTEU zns3I9uGXd-OK@HP$a}*~L1`yCU0*(G!-dkW;Wqi!Lin5ugIyXP##}nql><01R8MdtvtN!+<7bht~v!Z zK`QW~Yk$SyeNkg_#z@f3-cz0y$o`fqs-n$F;RAXapn$Z>QTm8W-urH>9iQ&YE5GXMPBa?wkcF;H0v_q za*_ogw#*$*tZWKvEW5fYp%_|oNu|z$WW|*& z8Q8k5G7K7OH_HgPEOLNncfb39_Km$5`(mi3!RZ$rP|Y8BZef z55Qj)XXpdog?`^KNItA@AX%Fi7&-@78FzDn=Nt=Y97NKfScUlkh@IzU_O>7rt=KqqU#MJ~^H_O^@+8yvkgg z?KL%rnczi_d({MZ`%U(NurqSt_u>rP2PgVnxwz{=C4yB-Q~=cC__!Mv8@xpW5Jf5yz8Jx}BKTTs=_7KMhRVMHw`P ziUK!fvbUP`gQ>nUynYv{M$Uv;#N4=hw^QwJP1GyH4SBqj^IB&&<#RHr(PU7FH_vCSd@dlJ#UP>OW@rz3)Y zrut(oHSE}0>g9tT39E~+fQ1^(A)Qra&qEdcpsnV|yqDxuE~L)N}J@n$cjv>=gO}}b10==n!dGmgs2C~?;G@? zt`13Pn`6sI4-DW)N_!LSYo;s!ky0|>Dz(=#oL^_+z#3Boc&U7|Fi;z0N0FHuVP`=bA7j3c1`;gY_7*W#V8>$XQHo%PpayypgxD+{dWzSk%IiQnF@K}1g7ms z@9UJ+3U?(B;qGu6S91u3=maHr=Q$|rbs>V%Q!1JmMk5T{;fz(r^AGW#%elA?N`r%i z#sj)OzI!1R!c$&&+|%T0let!`);bynG4PyMylMLSs))7m5>*ppWeS3a@rQ?gPm>C8 zo`MNEWWEzy6a2jEbL&5b_e4U$&u--w4&_H^_~iTZe$M%Mw~ehW4ApD02Y9IPK4{*& z4fB>--yo3_g5bASUGn}p*wl~-pNl+wfAkfOUS1G! zSBdpH#u~mB>YpY}FD)ymJ3bF&`FeoOnaPx5Wwjb6hkWpbDi^W{Pv)sp^16zuVTUbL zQiewWvJFkOs?j3zjrft%`n#f(FFP(lGr&K zM$f*BKt-)7{OMXRk|vh^YaflrCBRuJ*Gh*r7lN<2HY@R4ja`G%7_U8u;gT&s9SeC5 zu_tz{hx50RU9XQV8Q^rUtk>29r=VwBL8oVi@D{OgeXrD2A@_u^ZMpK9gnBiU%a?cT z9=7}dl&b5^GnYocsfCot0euPGFSFO`_t4&w0{~9|koWN4qhyEqFCEsdWBlOcJ!WM~ zDICQkHi1nZbc{M7cv&HDAq%wW-p^1FJL$`5N`CDbLPSnSMBo{c8khyJJf0)EnX7H4 z;gLQ&UMMfbx;CSN_^YKIlSr?IxN50WF%iliJ{w-T2HVXo`?xKLXIqXLZbuEz<-Crm zwAHM8d+5~Agdkz7R>GNWgz7t>I00;rbFyO3Qg$}gOdL4nAAfB}avwb2?u+TD#f=|C zignAN4@!6fwK{--D72-*`I?F*p!S&Ec5xN(@)qjX7eb)TP3$vUg*=K#r#_&9*j#o$ zdF;-FZdNyr&ybU%#*MK;Dk3;VrjW2i#UA~&1p=F4s?s|JN3YBuE%_^SO!YppmtWF- zAm`+NDe)>2Oy@v;xc>l~SW`4D*jj^mGy&oRhD~5A-MnSt)(WYFeh$ zgR>HO?&^cxz=(rGBb_4%P=oIU-vTr|6%iyFHNoFH?jOxW6-|&+P!RZS8HnERn*L91 zR~-=5_N@WIAiXLet)w)FblB1+_GjuaFyfZWE zRWJAb?tQ@@Gvo2Zjuqcpd!KKO*RCeryCy(M_`6$u1<2l`@y95|Mef&lTG!KcwdP>M zk=G>OrMWY6o%SPqT!8SD_z%xG`du4?f*!s9SGpUa$qe!y&#->EF&)FZN8aBb5X=X` zcR`HFco|edKX;7|&qb<8S*k5j)?=ZE;%DW>n4eO~h^^o?xdZ!B1&+q)T-$jm=yU^# z0>;ckRZAZw(XX7g@5Cy<^8#&U-^MCZ1-?sxQVnq-zBx}@jZ1G9Q7-bI>v3iAvM0~w za8o8SIfA#)DehF>bLC|8?XoG!37aB78E&*}9F+USIpIQw_pU)C|ag zUrZ-2lP#8dt6>EYqE)RyxpC&nz2@k9J=ym_hm3&D6*W_srFw?{exQbR>((L(#>0q8 z!NdBcrxc|F9-x{Q=}{xXQ>`Mw`t{PPcQ8uw=6Zx}hQ0w>A&6 zynI$&WS=v<7F+B|&{GE@ZAz$y$miH<6@hp~2V$g5-65g6L5_GrwU$hO07#*`^Vuv2 z)Hgt-sOo3APAiK$KbP`#orzPkkuBQ5ZPsRf`N~ROD9ws4KYaE}EWmRvD`NMF(I!uX z0_7MU=m^`@qAbNMC*ldk7q>j7k7O=vB!U_u&1LL1v0kDab&iD>QPyWGbHED-eX!_e z+7fhCAG}0~$eZtulzLR!ODRS7kNj*XJt*Z(i^>!Px3Ig-QZ^|vqxRaaGOcIWMj><= zHC7ZqP8QeHozS<4e;qM!mhot#3g5B3lSglVr;KzOJX4wDm@pR@JYLf*$G^4;kf{At zw=?UjJQ^`y9a0%R6m@;9l@V7O!|>S-k8 zhWKE{h_XSw$Lok<3)gAAr+%-}zBM(;TGa7IdeDrMO??KqCS)JiB>itzv^oM%6POSKR)+#pKnki&V#gM-`?jaL(ThOP07~y zBdKWz@$#T0sqMPX>L?}R=|?EXu?&J1T==O%K0S(WA++~2veAwlEFYZR)it$u4wlAE z@Mp~d_NgRZ zXq%sg2E?TS3Y0J73#?;ZZRjX=lFIA;8skV(<nh`5}>S!TV(n1cr%l==x^!)Jf0n}C%vCHs6+siSqK zJQNANtXw>pb|nJcBx*01xi+Uthhkh|67fv_1aGgJBITVY#%z^S-`wkbBs3PQWU(UZ z!QJ^q?1tkV5tQX5rx!=V_WsnYKr0KdcpfMF+f$;6k|VBBkmVc0^}UzBZ7RDWO4MZg z`2x}qaCxBA`YKVtix-$n@3L~XwqoJ4vu8Dv(N3BM?X!PukUR;QBaxu-u>5F3P20?x zscXB{TNWu)Aap(~5S*bfJlmDTKK!0}KVwqIt+7iyKH8$dK;eQ-%ndw9OWNp?IipnaH}CkalgIL?_7^fh-t&<_WV@m(Svb4F@iONxjw z?vBi$!`=uvFe>a-b6<+3StfRcRt&!6=A=D~mZ>Bqgrrt=6A+3fYLP`7Zj#QNQ=YxENCAeB35Md zzg*?IPS3!Ys^0Q7?mUa&#mpFD^Ndm8RZ%Mt3Qg)JaQNI{+9VV|t+_NB?smng741oP z_={Ub#PD{yn(Z{SGp7Ck)k}*(YEaDEQqOa0=n)u!|IM{wz z8X4S`OryZh)wvDlR<-UT(BwL~KB*A=kU)g*_y%`=CH7*;X2kV{Iw# zQkMB?IvEwT?it|||NkkOwg^@GoO73P&RD67nUSg>Dk*~bg&{Z)bQ4Qs$7|0_cc6b_ z{=sN5NC>wxA*^?xd~aZZe*~2T>V4MO5U;4r-Bw?cO^bx-1E~t4j^$7-BgRebDQ4_@ zjZz=C?i_df^lahuI~McykawkpMxoKIqNyO#K0UfB*{|+2-yJAQHP9&^n~hJ_vd52_ z3)P*S2NFb(RO{g9fnx4r#OQR*o40PZ*SCs~<1|WUbiSZH-$$^LPa&i)`PtVj5A)EI z9pz`*;`L^&=KH<|-OnqM+%1p7gp(9+`=Y zl9ZO_8P48VVn1gwkG0Jm!k>ow@+nJ`kd z(znz7Q;hp?@P{U0V#up4^Z~R2U&+l{yTnnEDbvzW-!!gMLu}J{#zp2OWV@)pDKf%u zJxU0}2Wc?1#T7I|Uw>9sRUff>{9UoI3sc`Dh`wO=R(St;R5TPBULKfO>4`a_b3a-t z7DK&@3W32TGhg~g22&%Ah941f4I;xlCGVB3CjzH}8)%tWcLLKFKGuCDTPW$@APAQz zMMt4)(v4QWgPR++=VI?T74yFW@Q9%h`Nt*&oqo!~K_CcaF7qZuA!|2PD-AMdP5mdO zeQlYAhTjIOw1Rm_#vD}N?Ookhk)4H(DS0D}P5(mx8k}CV);+i4NPth!ZWW)S!REO) zM+2clrksehs>;a`mrU^A8x@^rv?XUN!2Nk z#{4}{dlS_lP4sC*C?_^$prpk8M#QnWxY}!jl8WlK>?K?r|9anXv7~!80+g56zpr;A zE^?TN3%DSU9ro~x>^mO3o8&{DY^77fh3lowixv8o8dzn>JdC&j3`(ojl_K;RB z`fUHMB2@6&0*pR`c_D!LB8hx&o}03!#lpNr3SG0uUZcb2Vp??v(uFNEQo9a*?4Ap$ z;24?`>cBqwH2jec5*t;n9vOIVYQdW57#$S%b&*qTPa@*IIXUUk*7%a+#!W<`kj*Mh z!MK4J-UA_l)#{+ClK-e<@_kvQsz%V^j1}p}HJgt|AC@ClKs(!2g9;5ggQXTq;)^{n z&kPNr0jhu(c!a|};vr{~EKG@ncFypCYech6G>q6UN{Ht1bYWftv$r#lmbNecsH=MG^X5bhVv45$m7WtZvh)sm@4*iO?Adj9ffV+ zyLh;GN=aMXg>=lA1-Q$GGE0xkPzy}r{tgDxyxeBm-O7mEONPa@f6v$4V!5s$VzBEy zen-UpD8YudwCru#dyR|#Av%2+y$6R(h8@i5btK|qv=b|Oc_e?WuI1R2Q$Xjb&KNq2 zK;5UM$s7lJ8RP+zS5CRDs*y+_p=*8Jk$G#_6G$k%^C!Z5AE-s#@5b9bR1F2hhrwZ_4AjXlPXjt5Cl9$dE^ zJqxySyIlmk-&imi3p3VC11WYyCbYHeWdz2w&YghMG|aRYx~FI z1B?_$x8nqqQK0sj;6i;q0HGaFJ<_L;QFT~nEh_1B1>ND0cv7gr)|cbX3ZpDMH>Bk{ z^Gbe9*PT!JZk75O4@t4y$KLq_T4b&}2XN{K&h&JQ3zwud=opCKSNZZG(+LFc1vlG6 zXBr3c9z`}V8pkXY=k`}DR1Eu*RZ_!xDx?s6-11V~Z&%@{{>RSTet^g(E5ZQk(xB~- zm_6%ZP+l;n&%erosd$?mJ(hx`0_fYWJN0$i&fq->~-Q%w!cZlddI#>-I7)S{AmK zk0(qf2w0lz`U`RJSq{FxsPxy6`@nyAZ_XGDygz*ygOjitssAr6UP!x1{p!`oR;V@y zdYfKd&+voDD2J+G>z~m?@CL_%W|i*-KB#_!w4)W|*lYcDXR5Ijy)xx+DOKdZMeecf ze&b>sk@iqvFCogX%7!GaCCA_N)#Ugyo$Y}SZb+{cOUxc*Pd%UK&M5E&SFiMzEAt`{ z|Napvf!R{66F_U2xBgDPda%B8;sf`aGE4ld>hsnGLPPf@3i~ZDi%!`I0%bN)8%W z5r+b&W%(^d=1|0Dh^A)%e{hR)ZZXKKs3b2*$OS&}wQD=)9WOf5tFK?QHGuO)( zOHl8x%>lRj_AbjWp7ZC&f8X=@Q=0aUq9TzWQJQkIVZjM2E5G0Qw3@S4`!+m}Y*lJ+ zpEH|<^A0llTeTel>tylUB$%g(I?hWb5?alr%lF@;TIu9TVASkZa@dwO%gM>gtMZx* zTogVHGYfpEJ+w6qQ;U zH@e2kheN$*JiOfBd)V$!)KOYW&o;jBGVy>U+rehclgiy{G*{w}0=3<4M)a!V<$KzE zeyVzBOVz2YGim?&n7|NH(@=Z-J?Tj^y@A+GS|k$K5lGYMJ8PqU!>F=ckb=ow_$h9Qww@S^G=X{8W zZV_-gu+1ps{HdX!YXx)9p4znHOl|Nt03i{te-dpUeb&apI5DZn?llvDUVK zwv_Pur2jvPldRdYm0*|IfB<4Tk(1YXj!mcfg@@iFWTJF@va+@O0c1>gFo=x8*$caH z^>DMR@_o_&)I6oA6hagarqW`Um>DEQjF>*R;JG(y+TKUo5Whl0m9Zqn-lYUk`JwT= z#j#+KWqU>PMci$xl7MHe$r#{&CtGac&{Emk`Xfi_gz^Rc&cRvojOw3J$#xI4G4U|W zEj!L^e%9HRY80L7YW$|>h=6I&l9KMf3;92q6dCNiYMs{RAVqIQ(0|ufrGtybl)htD zRp9eL3FgF|QcCxe9NUOdZEY%9?%F3mJwikq!!;`=`7QRzGaoy11c`W##%NBWa$fIw zgF6r-j8&8iBo03P3V$)Y>QgJ{$fEjVil|M7EU_+n=V=f#NPM%to@TeHR;k#{%+!|@}8pO zIl@aoHD8om(_O0h#N!wjV|ycR+IUnVaZH@mV;E~XEgw>(Bd$$?b8X*={l&AYmJGgK zYa-bIye6NCLhkInJ7o>?HY&u;eKeh#H~tKzb5P-&(ecv zyD%&m+%p(SCgxmb;KhLKW>n)45}n4CqQtel2Ni z;@tCjM4a;6NPt>gh}1;>%dvcB8 zPTo^9N~*jf^;y?&u``RJ_c$RL@>?8_==yyIf&D|(%9&<9QIS$|&M;tmeFw4L=&Bc5|41UX@p(^Nh`9b}lP}DB-N5Jd5rtcv&To zaGPLP7v;|=IohfVYa<9&^{Y3CY{l6}R_d6RE;qTty|*=A?AXPcX#ysckaRuMLCsC6 z!V|LJ$y50vd5lqeN9Ax?wtuwTeVrgsLT?5GgU|=L7vS))cOeu{(zDwQcsu{TntZ>u zyCt^d{hQ9mGgc6jmx~(t=Jj<0@)smMT&Ml;K>T1&NLN%{e(UrjPoO?S-8#Jv&#P-x znBHfh8v#XmQXEH{ku~l+1x%ctLlFt%s+Vy1$Lmud@icSVGqrepVyv|is_i^ zm|$mZgm^eK1uh=2!`Oy)hFI+`Jh0R#^e1cQGkfx&d#UGIE;_N2L;{6+YA5TO%430v#`Cu@P9bgX+@&HCgh7 zrTg&dC2Mr6O03-^dKN1rCB}LmyQ?(thq?uB;^!q_+u1jxsjjh=A1;C{fAKDMUEJo^ z{tBHN&~XhBhU#}q<*0q8zT?;v)?VBssV}}|zYQ5dm^6M$z{%pH%pRsU_!y~XU6yaJ z(!0$UNQVE*@ICMRW7Yo^s73%}SCC@>`%(xVtsxOjL`h*{tMAgy4&Al-2vUm~D~*hr zlIk?bhe#9HA3I}ry99?dD!~mqfx*jVuj6Z`qn}bFx=Sd0c)`Aafavs@q~5P8)b+MH z9E;mhp%ejP5(KhZPiv?{`>kXL?1QUQ3LCnUN@g93ibrYA2W4OS(Bfz>n19&R&+4@^ z7t5R2?%J|yMWJ;7rn}=RZCXL8;eDjCALE~X;Gx$SsK_B)WYS!sM;XnvKG8M}qj|gE zj139NT4b&-wVIjaOhr8X)v}ClnDckqtyCT!()s7rZfv>N47ld9qb!SPx5FRzwCf~<}X9*Gy-&6_lgb4yF| z3JQLn0M)Zbgn?^FE*V#FW~Ozp3c#m@G#R2m*%^K0;k#k?YGLniiq@O3`n-!K^+-kb zqHG^Hy2sr)8-*+Y^XDiVZZ7m#ETpU54<{XUY^gC-%=g^da~yv)j}Ulzp6eW-tUp=Z zxwAWBpQ=DctLjptwb93&lNC=Ytli93}8f$xVv5Rl#MYR?p>$KgnxQ-X#4U#Af%K!2HCbMul9!ja|W zGOaT+)6;eI0Kb0(_^Z?n@^CO2(u-nlyv;muno-V`i*9hu-rxJ z4{RyKN1C7$M-^$Ao?v+?uty0C?JcjVkb*?n%2#X^E1OR0^Oroeyqs8FB~{(Hz=hX| z-A5)pF-(IumH%FV0AIhg?d{&ey{*;NdBUvzu#6p?9Y;N~ob0aa%^(cblSR@)cS%xn zL?+4Eyk)e63lBrNe}>4tXf2`z0)$;|xy@O_mc;u&an zkE$jgKPM*39huWiQVwI(+&2 z?U88g`}mmO8GW~rk^$e0H+Oz^E-35wPIrz}?XMlPU3eZA8F{MAL-&xivs3EH%|+=n z3mRQA5vvbWcWvoSO-v?MY!7zUfRcO%dO%sBpISS&I$jU(V~dH2UA7*r_5|q58;@<~ zyP0WQT3SBXsO9O30Hptwt{VWcGS|@2;l8SBPHwJ~0A-mOP)0S9Q!hcpozJ>}>s>)X z!P~djsDq6J5@Bi@8UVjPmo`9nt)LLpx6oko`Y!!t{D{cNCPQv~_{Q$()2D%g)oIso ztvhOKYfljguPzqBCP*wo$tfs+?HUtn0vI);iNIfU^f{pRk?Mm7jfMpg5hNNB8yg#I z{O#@S0F}LN9_cL$3s#>GAMRj$zki~zYv;I)T%5q#$LALTvUo}!lb0|!#Z4~{S=m-R z&Poh@#ib%Gr|Q@Q5x8&kntd*V)`PGyZ*GTK{+q3p+dzf#W4ne%HE(Z$H!mwr-BMJ9 z1$%9#ym)~-WN4*qWMrhIq$ErYv9#RWKg?3g>u77k1`2nuu(BEzr~?lGh9{C;DvP|` z2)kKuxVCas-=A5o(PGs#e{p$KMA%<7Y@rS2)Dj5n5?7fV6(c4#*51ypQkT=GxAy@CP;dd@y1&bH znT?Y(xWTYae-$V|us!Uo2hfkd#vAwM9?3WFOR8&+(jW1t=$-KLUhT=j)C%4=%FZ*F$;o-Rm z)UZ5v`Y}KP50p8uvB`h;?%n(Mm%Jx8c7fr?-0`Re3aAGXajtrQ4v(;3ISkEnbgAig z*Y8n#P=g3CrCxG;>ABU%X?hvxr4$kg(ZwOE?e%BNZ_ynumwy%<0oyLjW=Pv)Lk;XuZvvi-*4ct%&KQ zq^tPOTk|~t+xe-prypx*XsD#i)xisaf*ys1*M|(x924+|+v-(**iS*S-u>VuM^WGb z%y}sCJb?T2V*Gi{M*jy$a`Erq#dykKdq+zLdpU?H1}?x*in<2)f&o8peNJPK@SQ*P z+~-2z`9a>9`#)caL4`a+4r1hJ@1UpRU}oiLY2aW6vDQ-tJDJ%7Ke#b)vCm*i0d(G& zakv=RNtnhyxLA&cxENSIBmiqH=8N_K&|Sz%OGF^goFW z(cian(BEIau?L!Cp7q5(4-oJAVgiKXm?6mDnBmAzEFa8EuP_;E|3G#k0D0sT>d3X9 pj$otC!~tl`Utx+ON8mN`ywg6I=>It5gL&l@rVelcK-C?A`(MM1VP*gT literal 0 HcmV?d00001 diff --git a/crates/io/tests/fixtures/origin/test-origin-7.0552.opj b/crates/io/tests/fixtures/origin/test-origin-7.0552.opj new file mode 100644 index 0000000000000000000000000000000000000000..0dbe7f17a211efe6f255191e975a045b7f2d0e64 GIT binary patch literal 282034 zcmeF)30zHG*f9KEl7y6^5JHk63K7ydQdCkWDy5lbr4ec1NTougS^dkM#KvBI1g zv*ym5t)Q+xYohpMhGE2oncm;MzjZ`8B3tK)Fcu_)Uqa;j=MDqQ2n~5Q1jO-MP@kZ% zhy0f^6x8=6WG@unua7eP7}L(`Bmc-SVt*CVT1N6>#v_hWg0ShRtY^t+(5C_1N^=hS-kF)KR2%m4B}w_R`)%!6PZ2=;*>4+!#rAP)%gfFKVD@_--@2=ag+4+!#r zAP)%gfFKVD@_--@2=ag+4+!#rAP)%gfFKVD@_--@{MY4y|2OYh4(Gqc_`mX=rM{7Y zIc?vEfBl|i(joYI`se*nhW)4d|7VVZ+#|?6g52|8mwN>JNwA*;`$@2$1bIM^2LyRQ zkOu^LK#&Inc|ec{1bIM^2LyRQkOu^LK#&Inc|ec{1bIM^2LyRQkO%mA;J@uXOAG%$ zZ$bWb_}A}Q@=LDxj)@Pvw~ke%TIlvXb`PthPV1FrR}xD-c=7qtuoEnm+$Z-2Y(2{= zA6sws!6l0oZaH^Ly-7Hh=YsKBtG+pOgF7R$r;i&^^NC->EjD`ky4m9MDl zQN|j}QIo8_bcbaSA#9e=UC!dqo!}_QrGi{4$fbfjAjkuPJRryef;=F|1A;st$OD2r zAjkuPJRryef;=F|1A;st$OD2rAjkuPJRryef;=F|1A;v8zn=&G+upNW@gKiuSeiyUe7qXPfuT(mw6tXyb-wK2?ZdZ2ak$l#Q zW!cA+SLd_VoJf8DTQZ-Oa&F(dEqAZ8s$Z(DONhA6x>;kOxJ9zav@jJE8@7K#&Inc|ec{1bIM^2LyRQkOu^L zK#&Inc|ec{1bIM^2LyRQkOu^LK#&Inc|ec{1bIM^2mTB5z<>Onr6m3TS^m#{H>@=I zfTWlV{hw3vKQ75X_Wl1XE&lDBmjC*5mWgIJjE=O#vn-GDE;I=hvQi8WZaI45E^9~J zuj1%gk6E+diA7#G{*0AnzVXJ`xJH)HCljm0+9uY%MM-hRli#wOw2d53Z*67y{`7pb zpy30{HaFPoMru22;_`qcMHXLJqSwy8J|Nw}5)+X!*L(Plr934w;ZeeOmgki1sTx+@ zth*w+f;Uh4!P388HYxUNFRR&b(UH*NU#ti9em-YbFl@okNeF&ULhy4E|NB2DA=pQP zeI(dNf_)^&1A;st$OD2rAjkuPJRryef;=F|1A;st$OD2rAjkuPJRryef;=F|1A;st z$OD2r@UQd0zj)6Q-ev4zgqQ>=A^sbVjF3Hj&r*g4+yDGOC;3P6o+T~&8~UUB-m^5) z(feBo@(z8QQ(Re|VZ`}klaqgMCfaJzLhoXB(En* zFLhN@Dz9{qV#)C86yC>|CBIeDj_~dz=6h$dlX$C)Dz=EJCi1l9;&W_99pKeT?Gqaq_V9ili4v~by_@IZqBz!PPZaN&gGne) zr77i2eO)N8wQ%F7bziwW{`dq(K`s^KQb8^i;D`XhW}n=-?5Iw#bs3cD*fB<_@3p;#jEZ7-8`NCUi`PuSsIQZUGU3f z*nBBc&7hh?HJ?h0FYaQ#7#+U2OZj5-`Qk3;i!tJh&Ebn{!WVBPUyLbVY%{*N=6vy1 z^Tk;5#a_b~*NQKmHD8P^Uu-)nd%hZ}xDNc7!dkv~RQBumF~*Uv6e_k8Kh{Xab>_zu zT=?Qq*}L*%j2mAmRBU&CtdWYlo*z^2;EP9P@5zrb8~940VtespjZ|!Jek_)%kxJEv zU&^JbqEhhXmwHq2s3iUPrS??WRE$5rltYz5)lS9U$gdYm)kvkfiC@a4s-jZZ%rEt( z;!#Nk@JsEfvZgmDut?@ioK0rFP5s2N{-7fWmCCQ#Zu)_HByOf=eJd*vY_HprBPK;bx|pV@_W&v z@}^3p;!(9wN$y}XeP0?>_EeEn*;KVuOc=kdG8Km^fGUNmjH;bVE}Y*Bo640cmMV{` zkxDFr-&U2%f{II(MpZ@CMWwKl--{lVH&r4PkE(@AGLqj`gUX&Nk}8|3mWqkuw^gR% zPz6w>P?b@&Q_1b(_rj)frHZA>qiUoQ+s$vQN@YRCrAni!qUxeji01dAN99eGNX4US zp_1IgZ>vFNPZdd(O;t<9#PHiHQ*o#Qs8XoPsM@LI_VRmSQ@K*bQsq%KQi;U!+saW* zr(#o?Qn^wEQ^iu9q{^eJplYP*pc0GYk3gPEl}d-og35!6OO-&CMwL%hMb%8zMKyRI ze*_9tbEx#Ftf{=I!l)9dGN^b|kEvRydZ{Gi`6E!I(x5V=vZwN=ilj=W%BH$aRZI1r zib>#~SB6TNN|TC1Ph9VN{7! z8B{!~$5bs;y;PEk{1GTpX;2wb*;DybMN%bGWmDazs-=2Q#T??FSB6TNN|TC11ywawBULL^2h}esu~hyX45N~#Qle6&T0o^kWkh8`wU)|* zY7-TgY8O=k)lsT6svN3(suHRys%KQqR3E9jsDzL6uY54o2r31tsZ?{Qw5ar`R#I70 zIa7I41yY4k#ZVI+pbmFNjJ)AuDwHHu1+Y6g`Cl{S?j zl{u9?l{=L`)mExVs(n<+RHv!3sjgAorn*N}OZAHCJyjW>INU zEv4d6t)X(H+CUXR6-pINb$}{`>KxT2szRzVs)tndRBx!-sd}hHPVwhJf=Z5R0@ZY? zxm0W_11eK0TPjy7U#eiL2&!1BB&w5CnN)dHH>oP9s;L^OTB$myeo={?=Fh<}DtRg; zDpjflR60~fR2EcgsXVAQQE{nuQ6*3vrAnj9p~|N!p{k;KM%7I9k*bSI_zeHb2UCrp zQlOejHHS)zN{?zKl{J+!l{Zx&RTxzaRU*}Kstl?tR6MG3s>f8%samMMQ1w!Yrt#-M zl4=x{BGn8k4JvIaLn?DBdn$J-f2yrikyQJrlBrHpWm8?Fx=nSDs+Q^%)qAQ=D&{Qz z%EhTm3e`EPOH_qaWmFHT>Z#sPwNv#_ ziJarlfdrKt)dZ^PRCB4=R0dS0RJK&ERK8TfR1s9MR7q4PsWPeZsBThKP*qbkQngZb zQ2nA3OXttQFe-T}B`Q^_1ynjzMpPD5YpFb_Hc@e@c2Olz9i>X6%Av}qDxs>PdPdbu z^^vNJN;rdm<%6k4P$^JNrJ6&fMWsizlFFLOnaZ0gkSdHShAI)n|EYujb0p4?FYzGw z#*j~;_PwwB_s^n^cldvNUssJZ@$YL*zW=MQ0r+S7zV15je_388$^WxB|9#!YIYnPz zx)+gn)i=*evz$PBwi0$Gn?dyrn?2f|&Wbr?0Pt(z1$F{Qw_OBFDjcLm;Sqc>gm#K~Rq)u!-xTwDGgmnqfG*rkO(14QNLrIH4(SMUfb|vE-6)AKff(r zzZvQ|jyb_it|#JMaTZRrybG}my=yNdia?%AXK@U(K2xKH+AbY1AmCc_BTzQ?vABesSZ(- zP~o+OZkABb-gt*l^ahZ^x2j?@8!TOBnBlw}RUM?Fx&Z5CP2Kcp4Juyu!VgrVl}miF zCDdb|-RK|f3{pt^c&NW8i03e8hqx(-n{hEMXaT6J-pqQ6cM;L|59#*Rmx#QrJNDi2 zenrHtxLog){)~us=8Dsqz&k|T>S*PU>eq=F$G|5x^KhOmcJBGOJ_2God%vt)g!4RV zNp}acA7c$ghjnqTG?tq*9}8xYdW{pyr(eY##opila+)s2EX52@!#!tGR39}72Z1Cn zmzeSbi-;0mo$istkv5+hk=ux8O9{F?_?J19GO--#ZQ{^^S-+;d#0Fv()tfA&@N5C; zjFWWV&T|~QGzr(ex7<9lFSsTZqT|LD;1139Sa<6Z&YGllmD^&>0`_n7kBG0sbF57L z?yC*wVBe@P)W-XfV&FVT%^YINwl&W!l0b>l=OW@mpoPYdQ(kXz2T1;ywOj&rJU3SJ z`JRkyDD7S*`ss8dQR8l>jv%F{MC|&)ib>U_Q182A%g=_ZQ1A1+wsqGsL5f=vwHKv= ze*N(75y9EjTXeBa1JA~cX`SF^$|9wW=`yD@FgLNwlumZyb!hyS4E~Ax+@kWn&jerS zrE1viwn}G^h5D_|%iBQQ@LuzEI795HBlqv(Dpr+Ejh1qSo_Uexjt#a@&py)h%XGYQ zM*Qal>%~xNub?<0vu_`L$}bASeZ*5cEV9)B+jeXmyAXF2Z{h7sSsjS6gD0?@@NV&{ zG<#F^p=Zgn%^ADhpp-e}bN-D9#2S;fzg%w(VwlV8jj;vu-on*J0qXJgev7(-rQC=U zFDE)eY(c0}$f*U5Bw>i9lh}}>!{ascKXxyL<;~gbL%+hmO>$7m4e@}TR;+_sM z?@PI5uTwzD#$UP><3Lpt)vsu7Vi7f8dOR@?XIJ6K$o{i%b#hsww&ZPvdV0<3a@N5h zg$GKX54(ZbyzrPz18B=Ux2?M>i7~sl4F$OVxI#^{?3Y5!{_-!MiR+-AysGFsGuRMgwvSIcy$t02Y{^~`dyJ*+ z_b46>ay^*Za9}5kD0_F@zDamRRcEibt-!nAKCO6m66SX1!PB13@zBD)ZC+X%?hl^f zjpO4n)1<7-eg6UP!quK-+k{S&diJ}f?XsEx*@C^Zti%JjGK(Qewdq#wg;h-th zHz3AKFI*vU2{calx9-^t&ZLT+O6W~8y| zaVELqtcXo&EGV6I&F#VxP?y)OG2%E<1&cU${k}c@P}bMR5=wbrZ(n@Eg9CN-IFTRCnE=6)Ym}RW%6}xb9;)bC44~v`^UKm>F@00i!U}WLKo^=(iT?HP5`JdM*+AV_H3y`{RnJ z{U9_v1#@vg7ok+CD5Pm8>;NW4r76Nc5oE78COHRpXxS9?Hov_PV}x9! z=VE3^eDLT-R0xZNINMjwRC0$>J%{tbl{hMe*3DZ20wAVvL%RFTDs1cZA+W|0#2vXN zClN==t_fZ%j(bp2(s(Wl??+;z+4(r^nXS^X{F5zN15Rc;JSI9bvJ|3s{BBU zz`i?OmtfhAIfrRFoG^YOl%_}+bC#`wp4lP$gXEWjcoib-`6eiyLf5YZQqMk7^ikVt znBB(AS?%T#RYZ&syYy&p5$N0$^FrGk(8w^+Cijz|%+i6oeK1>8NOY#B;VQOJ?Cida z^LIPKHFETJsOP=raO(TbEFxz5xdJsDGczYu^)k+1+F)s+{ac`3w$sI`IUXQYw^Y5K z_}bH`6MWvn8Dj0edWR~o)PA6Io3ItcI4@r0nr#NL&-q+ijx)rz>0(^mAjZ3Cu<$gl zN!7U1%Rkt&NG!YlP`iyQNRpE@vCav^`#4)i+8bJMtFEd2n2gO?E9Uqzvl6>G|D18DHFi7;srUM{3`_CFG z5g=2QTX#copUZ{6EfK|hz?`zq(o2l}3Y1JZ0Aq;PGd%sH-k3g=IO``V_2 z1NE4kStrA-L7aCV=k+}wFbd6AhmC|7@5jwwea~0CjI~E~amLva$6gd-f85{OCne%u zU^InQ0_Q`E#(TPrzkEQErbmn(;x6N+*|Jk`Z?(MeeDe@zNG^MTmd(Z@Qma!c2Y2STSeqGXsdZIZ27$NpzDpn-7iu=w}l_G zzV88LFZ9_nFdCFRRVT_Q9Mt%^MSGV&h+UlfA;1e%^(}Sb3yd*yW|?gq!y+-xtxa3* z;uA&Mwc;79E!bj4>ZE9#4fe5wW%uyPd8r2$RQAp3-S)vp@#&%Q`4?`A4fMj@c=)B^ zIuI`;vP1%7u8EyDA~7TJ?lm{>(uPt-r*+kEHLN$mbLT2tb!Ah(2ad*hw%=A%GSv=B zC7Y_mu48}f-SJ`(X4p12J;D|5TYK(A4FlX|%q$_R7R@5)nYYE(8y_Z8>W;&y~Fe7YuM(2Zo2JKK8UNhy0oq5CT0sc z{kMzd;T|zvDjVN-U{-eBv(auPt}CWpoaNVdr*~dk+_f6&v27cg%9en*{W+RkcNS5~ z=F8qPIFpRwmHQ47P+BEFHP^=h#JO&FJpeO^#hFbr5;j7M*tv6mcj0TE++#Pt6S%Gz z&djX2%b=8r9G@$^1jGxLbQsik7b+Ueym2%F-r^O1$#g zw~og$cS4LEzx?}MFI3jPj%&uyLd4~(ymk!4G``ApKMR3atZ3k_qCilJ_^}FcT*WEl zrf%zA1NBm*qvt)?1L70~D15>(>*a0s2*KG^D6A{^f%lh@e>%uP9D1p0-zA#p3@w=X zCraJbAf~X3{rbvE5F<3{lN{blrmS>Ne1{ep=x*D9tKmFS$}M54R++<$bL;nLZjrqN zErvZka#i&tNPNfX53ZP%Go1E*n;#A_p5HviX9tKYW>RT4i$xTf_NwTWAILjxcwIS; zMKWk!)h1jyjpy3V%6UNxN%5Grxj6ciS-;+G#2uPqD$6`Ffl}VV*9VSaX?CdN+BC@OgtP!u~|X#UIuX)996&IThA3H}poh?5=zwo_mv$X(!A$ z(=^!Twsbb=@sCTUH8{_$uX`Lf9)y^@NdJIYVW8NRGlxIM^~{ZNTHj~^F z)R?%(Z~H1JWe>0G>W}x*{uys13+I%roSoTZ3H200+}|cxKt0}pmtPcMuX2YyN=u1{ z*E}xgXJ+fK0+4uQzS>%x=RyVblIKTBjA?b3QZZ(Uz+VStM2^cX^)&?g~!t@MGgPL1|+7qqO&aAp3V?EYvYi zs=jotnZE&A#7eCW6?X(R9=dur4tEc4L&hXmHndQ^xQC~S&pxppvy>0v-DkE9-FE?J zTp`0Gz8UiY`=h>cH@0Oy8XBLR3vD&L%2My*c@>svX53i~v8p#i66WH0nXRV>7Qq_g zrfBB}_5IDd)1lnXHXrJ((tNnHH3K9Zn4CQX=h?S%|HFMa&yjMj53@p{H09@Lf0eDE zv={l``u-i5r@8U|KwKpjnn~(gP?35T=j3ty=^ei+WrCwhdzGwd5e#ja1!w2~#JlAk zvq*&J$s!@P-|Sz#xEt9QL~@03ttpICoSB2UIQ!XCt}?Fd*v+538}T`oH*4}>W$Yz( zxUSYCviQlx<@2t!>%uuC(~V}jBTUij} zhB|0IHz(rmpIV=A=o0j2UC=GL2Ua#y@c6QYSvsidXvD!2hd}k3epR{ApzNT$(O0;r zTSfMfcx~+Q`u(rty<}G&7d?*G##!S~9)Z`(#Kgo6z^mju^uNHveXdZocDe_s^yyX1Dx4E`Gxp-^KHKr_82!H0q;`ot-Wtwp|CQ^axE@+?8+@EL;VWl$ zr|8QUeQVsw`{O~JAqDZb=3C(^8D789w!_EH)crw> zZvD+Ne1hkhZZYb6?zH!{%6p4@p}p!s+I3u~DdRoV-{20&Uix#dNZ;%>wx))?9&@yULW~*Uo^p8dSfoP6T!3_LR=w1L9mO z-d==x!~WUP_1|zkD}PC0?Z#{4c0~6(iMyL?m2~k~Uly2gaia}pxReFyUwh55zabw+ zC1H#FBds2}+abo>w@RCV&nnEP)JV(OEQsBCxxUjF6)Y=LwG6~jV^#ie1*JUPt2z{W z;q<)GpMlq_dQ)X)w`&n8<=$1^_tFOHaSeWUNX>&-_Du=)wZ3^-{aGt@JjBvQs*ao6 zm#JSb`FtGLc;c!hMQ*-O%8o6ptFwS!+Q$yQs*YKO+ak^k!@IBWFxDby2b3zfEqGzV z0cB5f9ukQ;kzLRU+nof*bn|~hq8+hZ(rTw{}5j9P^Mu$PamM^;4YjK?g%T%W{E!27GM6sd2|wrj(Y4=2!dr8#r`{C%(=wAtSdGFJO^+dLiFr^TVJ8`^8%K7<>(o zw4WJMi&xImHM-M*d&Yaqv?a%Jm$g4x@pVeyj@nYaMi1|=nCopFaU7rbMOR*A-#VD? z`Z+`o&cXBDtKeb)=U`IEbLA`YM8;^O1@s2`LaDu>ui zc&F^{tGGjH4!_@5(LfN<61i+>~! z;qv?@eI@u^48P<1y4t$S^o{;j1pM3Qbz@0ue2bZ+a&})ey+22@KS$fXBX`}OKfC_< zv&WyKci)kIK67K=pV^!LlyCV{9@KZ_f4)$G`={RazN39;-;ukc@5ujr951}@&*U>| z{JZ>r`w?#l|2=;Ozy8hIaO{gS%72c3t%?8q+E}^RXqhf;--o}g4TXMmZ78s@lzswx zY2TmOrhP~LXR6sYfByXEsK8${e~u|GedXLgNBT+QKWmBeXD#tRI~?o#r~aQ~m4Dx# zd4GZM|DQGcfBT;NpIzU-*sDU!zx#U^`azt0@sI!BUKN+= ztMl)C$2`8o%gNc<&h%gW=7qoQ)v;t8I233v!{+y*NHv3M4%K`rEx!JDe(%CxzZu}4 z|IGY9|7r4n{&Uv<{3nqA`OoJ5^PgJ%=ReQ+&wtYKpZ^TtKmX~tfBti0|NJMa{`t=u z{qvu~`R6~M^3Q*AQ%QqF?KCz!n|8Uy0A1R zYtcnTdx#m%i&|`t^|IPz-$&R$Y?96FZf7fyx7~nyBOO2!T^x)TV%yCI1KNJ0bZgr? z3@o9Pod5YK3y^jWcVPV*(B+@grma9F9U@;?Z>#zMu{T&R*TyyT5z20H(gjzncPajc z$rqF@Db&Le=%lT@H~?cIcavxA#$Jv-ywSs24d*bJ($cxl93(YYvtT7^-P*4{p4jtN zX60cGZ1J?MVcaJ(s8{Q_ComH0eM#J2-i5tzr;*vh7N6Hu*epk_Nc}qMx+An`xEvN- zwF)!`PXcoDY7f6-3fkc&r!@ilbKIEz_9XUKV_5j~4aRDQeOY`THM2+jVKk0l$4tZ1 z;(aY_>X-JzvlTEuS{`Gm>jaO0K-9C09&1ljzV+?RYq5ni>CPEr*VGm{MB&+X^cIbvmU#1xN5Rv?1Hm1g_}h z&06hPyxwpzjj*ZMg1pfhQi3u{uK&cr{<_uPWKKd&b5|X53&%3_B`9|Q5=x7J~wF%w)hrCE*;g7cr^PqUdQYaD_ka{c6+@2aTfdI zjW<2}6k|6{zO>4t=x;prB`Xj$M{BBYd%KCwrQzBaTY`(6V3lSg&lbX-N^b z^*ge3%~X_p+pD>wuyhVLmmP|2O}-21d7>uH%6PTM5U#_&wcKS3UJ=>R%U|P)fT{9E z{WVu-E|SfUZO>T0w3&f=yVBaM6YIUXJ1BcE_TqcSKl>MsVC?OCG7C_Djd_8^@3*QL z^PiGA^#`^PnXl|{8hg2LhD zRCWStoPTt4@f>f@hZli?7#m-6I(pU z)GjwRfVKy62K!{-2-dV8&r-lKFAjZGnuxK3eq^Fi?{c&Hy~SRX&gF_;!M1T9f?4V~ z(gb(1(XsUY`uD@0V2gN$EK2N8>+aXak63ys=A2R_wyg+MOOVFW*72hn7Gr4$OWpPw zUI%U?>}7w_qAGDL9oU`XauM5poV4e`0@U=Rdq$!-8-dM3MUP^O3$w%Tt;ZJH0hfGk zV?CQQaR%8a-`IlWtt-_yzHLtzZLz?!1+BMF7s0ViVV&)Khq1{kYbmdP^zlDY+m~EPZOX$#^X4udBSaqxO(Cp6&A#rerquCn9_xj*yEa8)J@=<;>=dyVsXZSRHerk72Ti8uVoZHX{{~xaI7j`F%XKp_ zR&Z7?`~tRJ(RFI;bBu+Pmk(`>;p;x0TLQgA^c!!Jb(H6x0<^F2Np)hSEXy+q~cr3KcY#8RsbDp4w{J?Hd2 zOJxf`QND_?#1jRBvQXn^ow(HZjQQg8jUR_F*7zi{Fa{;EW^_*|%J@j_pv@@Hvo$*F zQ3VzQrX}E$5%Z#;-yYN>js5S!QJHG`jayL`23nstqpamLC;OmYHQxw#M~$BrF5!q; zF{iMy@43_LN>;^cj7<<}6z_WqAN*Kxn<2(L7IMZc>uakVIrJPpTa>R}dFvP|NVvE+ z0o8lp>a{4;j<$l%5Y(CH<*7cX50WjH`<_c?j2&XX8e`KB1isNnwT)5o(?t23eI7Lv z^{s+wACJnjFdr}+HK+6WDI0uVQ*d5TV2xV5?sQt8W-eQMqOUY}U09YKmd^ilI%zHH zPNm9JXVgmFvlHA=V)l3RHlQXAs`K_msdV=SZ$jCp*Bb_+ZvTGRZ!2n>YuT?5)S-vT zt{YJ@UtU+QNA0m+&hoIhP>P^~+k}whL@j$8G9bxB$ z`Za`YWQ$_|T5@|e>cvUJ>b~c-xLxt_h8P<*zPxQ2YD?tY+e=WJJ9~xLDDClndo@x0 z#E0*#!spJJT?x)5s4RoV!0RaTV@J=kP|TU+%o8a0%7tAAP%SzmilR`7@>j)zQ73(> z-u3<6Wre5cemji0o?bEE1ofiw{ugbOuJoDGIjARR!j`Gv)9ZrSDP5CM1!wKnO+@XR zerEeP)RwI2>hh?GZNGI#qKt=*bsK>iH`(4@8Wq2(RaFvoK;P%mU{r!^LW3CU;!@pk z5mdQ*;kMuS3LE~tDC-SA8Kutp{<9V(UwCfkT~y+&jcfXzKntQ~ZN7l9O;f8YPofSK zDwrmsPCtwt9F3})kt)PROe&SSWvESa8$2{o!aAM`YN*!Bqt;GBDLjtU9Es{`luI3q zdL{kX_cuOi)h=YGen#!|Ix?dPCCRf&JcqBG85)D;rJ+Q3K3RAM^>I~!N1rB`YzXWt zon7y#)YqcoTeer)d#NQT16;`A>{KV3v?L|L2QSR*Fhd!a+8=5b9i%L-Qk$R4bj7ysE7&U+^ z8{79RFnrh4phAqzFkCd?BC1(kPp|KadY7me9>&;L+xJts`0R5gagxqv)ShPTk`1UV zg~0vJeX-A;$+oEPRspr{t4I&wrERB5$0+Cz*b7 zsYXXpYI$LI51>xp9z1C;sx;HeY$vK>`gW^rD6zOu-OZ>lNFIdsB3iV?4gw;lSRLB9!^>gVX1tCW-eq?|UjWkgAhY!q~H6E1k!n3J008WKesDmY) z$)I?*Cf9t!=gz#|AwjLEn>U-RpQFb4NJKwCJ^xT`xD}rlxAyV%Jy? z#6>}ued$Pcv0A@xFkZ_qi12b+fKg8Xg7y3E#mCW?5pW~bzXb|UGQI;-l}g9_qI!ir~u z5_w8d$|FF-0w)+hLK&H?NPJ=fvG<4aHTK;HU3uo==Ntw7tv4-P*a^^z5Z9$@_Mi&~GixNe zUlYZ6M!#(c0L>k`yKrJxGtsj6k?zOsL8k9c9UXeCi6~AcCV17oS45ttuX=wU3JM#! zt-8wZB~fqsxP0q@po`&eq#R^n^m|V}P41`zeP}weHtiT($EE80q03xhq#rs055;Q3 z=&$><-!<3hAd-yV`%Yi_8&P`B#VHM;pq6G|^UHCdkEKe@3;jVmce%)qUkF<3H(+h_ zOK5A9T(x|^8|dS(#-@40ctm-buKD9F(uqQrod0y@+9@safuO@Gbylgjhz!^{JDn$1 zfp*?oBh~YmXmE_A>+v`GA4=|(k|8n|8; zr@hwWJV0e<#%&lm8dPDDxvJ_u%*&^>ewTL#fVTMc|D-kq)bUhhUPB_xWB@xz{?JsA z&hw4c^9x{IojZ}9I^P5&Y@!$~GY;-TVX2p>dM0Rh%M+y=GvR)GzZ`k%V*+UX_y^X3 zQqaP~)O*XITu`^QE-T0fdVW7~RJ8OMIIr#SqM_SLK*0$c5A92Wkxu&Z)bBiATs?DBer|y9ilL`$jSF^R}qy}{dl#0;1i-pmu{Y`$blVtCw652w&*&du`6Z; zthw7jWO+(DB;m*lqBpy{hPwDS5 zW~f2NRhWqfWqr(KfBOx(e}MV4*$n1t`T@?&vC|+69LS8TUU?VxV4#A~8O79ABH=-o zPDfV5c~?JkTik*psNqezw1V40Vw*d4rA$x34lR7LYVze>Fj7YRt7?C9*sE{bZbp=i zg1!2)H{?UpL%5=J8P{js@v#1^ZU&hLg)|Z!oOakqb^-LF!I2v9HWPYQ*caF|;w7AS z@e9K#H5o9LtF_sOoWx;#^X;m(tsV!u@qOXgB@;n+a<|TRmIC>&=vp`HDYRJ9xW8XR z0El_9GpqFt^tZ7gEAo^nXtYh!q{g3cHirhWMzc~F!6}by$*pfeOH@lLsuw`1?&(%- zPCn=thtr~^4Q)46BvwY<0ck6oySo~7O8m^bmL{n8#O7Ggl}*r#GoyCM^Bv5_$CAiN zq4>Hr;e2+3L^1CD!lR~xX2NW+3*@Vh7S$3R56sIG`vJT5c+vG6GRt7s+Fdzsb+rxF z#wo>L6ou)fE^O8gkqrp*2YYl!Qp>2-Yv>|_UDhu3S5!J5>GSe=)87}n&7 zuP3xcOJV(8%UhDRyBpTuYq9QCGlgJ>p5>n5s*6DBYnh0a-4CIy6x(v?bv5Yk0AqG4 zFb&Q&`{i&s1tYkt8ba(E32zwR%%>hXS@JOEgvFQIqfOyDPETNtFBk`Ei}ftXG6Ct$ zfBkF2cFw8V$t^>mP&az5Ggr7sJNHwM^v<&H}_x|s8wjAg8#u_W4_s ztk$q6^*(!LN-SUOOJch}zA@xA`Vi>~iSM(1yPk*_P_m(CBeYnTQ0=xU*^x-?`pr-F zj02H$|K`%XXj3Bgwuit=)g`vyPwEhv&;2Rcx*QMY_By$`@LNgx`)7ZT#e8fDN_RT(&`xE__7`5s^UY|F^hX3Cnix* zQU^*W>P!{R{Q&o)J7?Q@wh>%g_oTqw^apT%myDa9F>o7bA=fgq^(5%2wf^o0f&N6_ zvO8yosB9)mJC+ur)C1QhF}W^vSd|Y^(+sm2#=g z=I621n$|?WuM`?&TyY}uzHIvIrmj2DXOHBZx8nkd%Jm%Ydxvm|why?eP<}gtC_$=! z!@_eQ>15&Zgv#wiOxt?<(pAtxZ-aSo#v5-UNrl@_!&bnxneQ7rd~Uv6A;||$^PZnD zUQV=Zb{F$tj~h|wz6)oIlMRSo-wPTt!va>s#qFmvUFU|ASpUbz?WVhh5nWK~uXW

NU*a?5xNqA%+oTg_RqifHHEx>T(qn6;wdZB3%r;fiKBsP1vt89?-k z=ay#{x0z@_=~C^_GeN=wjUz-gHW3|uYP>2s%%5nS=SjCc)3y*P{!j{8l@LtCl8Th9 ze+#|X4-0&59|iS7!@hru8vt`P^mpszA^Nb|H_y}ZRH|4<#Iv3-{n!V%O4ZFJ`u3MV zrPr=h$Da-+(iT%+-zfoizral9!K2x*#utT7@=+B6?fR`gO3VaS$>QE-tKsdiB6L2i z3{yX7PjvZ5%z_i^CKAbu=NwjE%+VsRmuI6>J3g!=^4U=A{QA2-(T;*?$>T1rS%P zBF{O!&jVk6T}$*Me4Ol>k+4^jo(7I9>IE(CNN~CH9(I)Z-LnN<-=T$U@Tsa$Z9Ad> zr3Wv6E`>QQmC$X@J?lcmTo6AM_uQH2Sy0--3{BVpKg2xk2kwAa?1&BRTG_ByKaF=0 zow06BAvs&;kk`TuwnX=Iln+@Bfa}mTkPcez1F9POTWpd&5PNKRWBt&da0h=+DjR-3K9Fd~ymaZ!fgsuY%UcI2Y$BR6 z>&(RGuVS=FUfadh-ZOO%5j$FZSFL#@Q9|O0ry7y42jxB&dA|^WHM!d;C?w_y+!^tH zVY<>sVg0!b=)c0PAFOAQlmjCM)WQ1O*ZiWc#}MwD*S3qNVijTU_oi&tst$xJpM2wC z<`8|zW_s^0U9&zGT1fU~!0pztH^PYYl#1ECHK2hOH)p(^1*+~k`+N0y*m+5F90zYJ zhC9ey=;@gDAcQDGzT~Y0UgaJ+8H@h?V4Z#+ynIor63p6$nXg*yOI(PaKAx`Dc5fY# z#g5g(E|*&qsjxo&I%4Xh#W2UL#w;^=3YwT_d1H1v+{7w zJO-sg_B)jJOGDe)GP4`)uKE>{)g&a_l^6p%YKH6F!uWCSL~}lu`F5~vh|CLug?!A} zL`vz6Dc!eQ3rJ?2CUY;Ua5#xQ9->t>UKldmu?LNZCUh((de=GbOi+vk5$jjQ+WGHb z^h!MITgh1v>+BcZ8Z=0UC{#z$c=)>6g$y%qzv6RsF*t|vtKJI-R$3A*l(`w3xEa>z z*sN~%DjsrJC|9)I+$l<7N+jtM|5r- zhZXe>bgbPc>fpkSM27Y9`>bst))M+nO*RmAt&-0h$&u$ki+$styqyRu`@!M*Qt4Z; zvVC=i4l8JZx%#;<t$9gb0+#~e(K`BLk>iaLWSH&1rE`?eU)PdEi}<$ z7_$dihTr4izG+U`>pE)*?8Y_51*ar`!Vdl2oM(4j4AzUB$NMuD+3Sf8TspR`vL9$y zUcZ(59pP-fC$g=zU10>VJ2g4sXJAy`I}RO~Rs#3U?)cr4G1E5^E&BY_XxVt!gF9_Q zv$9Me`^fGQTbI0yOC%xrz%s2cnCL{zGu87ypk6_F9w*`WRxR>tMCNXMXTA-zH0sp( zn6X@0ylPn@oxJP!SV2(goOvd$mUDLz zdA>R|!1(54X)Thke7EFWje9~?o!OhM*H(74l2UPDixmr}Od+Kg3cmaAc`=@7n%z00 zLvuSw={V_EWmSeUB)0Rzb(QLhSETKzaK}|gi@uQ3C~qn0q{62pc0Sgo!H_3eNcPPA z^0p@(!bDs*Q=8c)U8FS0`(({_T_Mbvq1)5Vf5! zk~@C0jntd@@kjos>i$|J!-YqVy;D4tXw|Our6YDq5sm6CiBTI;yNIlV*N-_LT;XiR z7dkUy_d$Q}@5oGVdk2TUPqCG;S#e`AMwYDy*BQfU>du6UB)Q}ch zElK;s*NT!-&BF_7YD9mKQbw{YMS6V>DP7_4`{EIGn6+opIZxM5?-^cbNi03nA!0`Od!owc@elL*4HY(Vta3Tlg|j7zlu8VT zCiwLu%20i8_dehiIqz7vN~t%jF(kIO*7I2Z@~K2-;q(22R#lJ@%zI?FKdex(ko@f~ z{{EEUyAz2vdzX&PHaLG*{PRZb8BR)vtQaUU$qvS>Poil}q`RJZ)gYNQ2kOXZLAYN|wsEOa6| zo{T11yUc#X;bhnql?U{e#D2I*_gAa|_39-#7fsvEMeiB6;R9tlQDXgY}&y`HlWrq;OZOU2Cu^dWznfYVe?U^TG ze42(nOIsI66Aiz8D6;eo-2KXkxwTsEa4)Y863VkKg#I>v7glTEd2bPU#r~o?&2p%? z7HMHWMSFfd%$1a;gL-$yZ&FIK&aDa|$P8wCXFJvgza@IpZFAnrP=wUGR=WJihM2yN4dVAuX228;*Ko(_KLRX7hX1QH4*RNG!}rap}w& znET45zl$Y4_K?!J*ZUj_KEmjALtKvyshmLcHSJx0OGg!=<`v63TiN4@w!Y&$39N)& z`^hNdLet13(smjvM&jOWi1o}nG|Dtyp2&}-@@+^n=)SvGQ%?f4z36%}@Jrw?GJ@vV z*D1m^usUZ>X{=QL1{q}b@SfUyljo%LXw%~*o#Q1)i^>OP-=P;B%@%P@bpGZtJ zdq(n5xfY__#@hIoa$kvFh4inTeWV{b8{^m>%4Ne|SiWOhYqA8)26xLub*tBKr$2pk z`n;^90a`>#zMD4$)`4GJvuBq%?0|t~A=aO+VQw1u_>jM>D2b)FM%>vYc6||9&xgku zJli*nw0QE(!gn`$gdlg-O6&a9HL%8WYiwLC5@DC+E9`%vHV9^%dA@Oke$Et#$u;{X z^}wvP-MfFzLJ@Y#DAgxN8Z_V>y$9^XPY8`58j@qYd#gIAZMf^SVj(CsY1x#weTo>7 z(8OnT3VCpkw#{D>(C-K2tM6{?{E2F?cAZD0eTrFEMq*~)_sza93_B(N8%O5Rz0X9? zU(|_z*M{>Z<$Erfwn2_4s_?CjQO=& zqci0_Lv(cPp2$L3$Wp7#+zu@Dfu3~~EM>P}cuBNeP5ScU5hDx9d8hU$89LyMAN?S^ z?BHdnr}8>{(`U%pN;<(abwnjdFBkp8j+yF1{xjQo&F}l_sseJSMH`PR?EFaT&5rj< zzp(l#QIyoQ@A+xpNa=7E+c^3x>_!Qhfs1E;hF6?8tNcp2t+0lIk8RQ$I|Fvlidn;J zk|W?M?auow-5)Z7sKn<)&yXK*9i>*cr@uS)h_u};6Ht>Z4r}*9>WyWGEx(hq9XXx& z^=lklxt7E5m1?(PjXO^pq#qUnD@?ZcOzvVmxHkP)Y6C7$Dki;rcjy*hHwad2ZjO(| z(~WRHCcN0~mlg@n06QHRSvfB8e4j35&8GY<(y+N&*0TyLH4r&uLeUF7%S`$ ztn&z7b)OsVeRn+ya#+%;!10DEkok3fA9oFog1eeN`QGH~RgjBsWa@9|dHtOzDsjNT zL;r)l>i~?R=)!ZkBqT=*Rhqa!f&n27LJ5RiNP$2S0tp=rfgEsTTisSxaKD#&NN5Y_Q?3Q07q>d|27j|cy$Mqw|K5W zieLF_b+q*9U(Y9`=y4pKfAHeu4_{`0Qa;pmT5+-tpgDh!ob}ak92IZAdjFL5iP(!n zgJ+HZGYuu3AebLt8He*}o$uNuj(-=Wnj8PT<<#qcLCIr&T)*Ji#W*8PXgRj?#&fu8 zNxG$0)Jsio0pvYx_KrvD;0))LTkYtEL6{ctZCc{c6hz;2{cCHRUr^@l`x<@`6JMhT zTpN7%Roi=iITP*5bL{5OSo$0NPdRYX3r_S+B0?6$uy*^M*f?gQ&EM75C+?m1A7 z;aE2@ZCfm$ff(i{0HQD?3D673+d4%=)6xJ@7&QP8aU^8`!fflp1_7dyZyEw91Igt> z0j)*q>EVF>M)Lc&14>8g=1ei)fsufScY78fWLUR(6d=qtaP4S76rRWiL{cp>0UCtV zIb#5kVyECGq)ikU3HuU=y~Mbfaot|z7_Xc8GyV`B6<#sZaM|fpUAru zQGcwrHtxalEb`7p^at{KO#_sMymum6gS@{Yx`Mm~xLZp<gqwNQX zux$-$-VKP_n}i5;aQ0qAsF$NV5uvV1*(^Y$zi33L)1}i8VYM?~M+B_v#{Yr{hyw?P z&jy6Sn1=}M*7gNNXv+p?5b1!*f^bh6wE5XAMA#3Gu0-So#IjEjQNL=0d(B|KS~?Js z9;hV`A;Nw&{vaaiS2gYh+OzcIMzEAkuT87 zUPJT>P}lv8Xav?9c0ZtMKrP8dv=NBo*CDzcr~|)6gd0*B1Lp#I6{u~q5ZwX91}hQy z1C@P>XcJJ+-ZT$TCQy%NATj{4Y%!wEKwWnL(MX^!t^NR@>Od{&jp#KXj=u*{7ElN7 zKvV;$#&Sfj1GR1U`G7_NvB6YCHG#@rL$n2`XMaL88mLD@7XYdS#IhVjTYe=@Z zpQ#4lPQBK`G5ggV_n5)tazI1>@-e)AKE&>n5yM1(e4eH;RvK8xar(4!n5@ARw;W-whFt zqnKfcaBOX!f(XZ3+eZ=M7+kd(5su3SM-bsyo%a(W8ovP#1Hv&~9Eu3X`PmdiIQBDh z5aE1qbRHs{Bl@gGg!5JD9z-}_MV>}P^A&wv9_OoDS}lb%oUfL~BEtE~FcJ~YS2OQI zg!9#<#}VOtHU14mIA5Lo1QE?wzahf;YJbB=0O5QU6NxApcsHjbdJ?E@3laSa)K!ZR z^#y8!mk_N6>b&<6T>@(PIYfPcT5Na}&?=yw4My|}5HourN&@Op6QY$s?L*&t{~4&I zPhnbbphj*-^aN1Xoka8#P;dDgQ7@n_ZMF>1<3KcYLv#_SGlwEd1nQ;9h*kh~{3D1i z0QKZ1L55&zOh?WDjZ3?0vfx2n}qBx*7n1^T?Q0G05 z=m(&dzk{eJP>W9?dK9Q<_4p<9?}3=v5>YHrkJ5M79|3BgOiVim)Y2JH*YC0}w3%>iCI>&H(k~gNUMmI&cG` zhk&~OJw)FEHRc?m?m*n^|2UwcF{(<^gs8bBIm?HRc^e z9f7#{8$@$~+E%{`(C0v1)dEompf;dyliv^2d3RviXFx5#6Hx?Ei&r4J52$CiB02%Y z%#RU;1NG>yh_IKeF{}oJJ?CU&MA(~}v_XVDY+x5e*y|qbg@}6MAVkXBgftwDyXni|I6}*B!!#VVQQZ*HNbZ9ONB2uZ5#fkGI}Q=f0zp#{;mnYEA0nJB z9(x24&LBtEAi`Ot_GUyl)AZSe2xp&r4kN-Dsq`yEI7?l*hzMt{NZnI_a5l>iK!h{g zx?2$8tamB|5zT}>5aH}NECmtHm`g_>!ddj-1VlKq8m1${**0MwB3dynM})KTw&xMy zOnvEfL^ykg?m>hz`uL9!;Vi%A6e67YPyT`k&jwArp9X|yh=KJG0kQnSmWY73Zhtr; zB>Kc6!eC58glC~?nTYVrv^fuv4yYGqAi^_P+Xaa5tY)IGXL|v4^$VDWXTRh0&1^g) zHrS78dZ4C#f(Xx?^Uff`v+3^3i0}-{syzb;&$>|!5z(1A2oav0U+##AR$p<5@GKsb zj>s3NBS#^^v;AWe5#b8pXbB=*6$C6oge!$Us}SMp;hqhMa79t}79w0_TzL->uD&Aa zd)EFyEjWj1xcXX0U$Ms3*D3!skcO)-p)n#{eGLmnM60jPh;a4wK_VhteHjKJ!qr!T z2~iE8mRJzs>TBCmm(Sm)COx1)dOnUCPZ%ob>2=yW}xo=5K(=gvab-80yXL)qVYg2 zX3qg?0K}JTBPs*x+2)8Q05vECQA41Pj7GE_sE;Kh$^q)p;fQVmYQO|UJAm4UzVw_6 z)O+S)S|gyA(f6Hq0`H z11bRO(z=M60`-Fyh~5FJAso>}pe7g*H3Mo%KSXV*!7-UDjeI7E|yY8rs3B~VvqA$lLE#|scm0cwL1M6H0D_8_7I zK%KV|(Nv)Bei2b1P}y6E4gxjmJw(%hTKp-ZARxYc7SRVlJ^MSN=|BzgUk9i)P)E`? zkPiX%v0E|iPM{v8FCX6u)PO`x`w*yo1|pgP)O)fK1p~FL2+?7nUYUjHE}%v}gs2Tr z3!X%D1gPs?Ml=(sr%DmE1*&im(NQ1{`y5dTP?!FI=r*8!@F${=fNH4m0-(Ern$Q$c zJD`@dMRW|P+qxo}1=LHu5w!@pe}k9(Q%;e-+|~JAo_fWC=95^FA;qT)M*zG-3!#s<%q(8dZE^OKqr9OwmG8v zfNE-wC<3UfyCeDxsK@&vx*wiPF>b~*A?1} z=Gcs+i^mB4xMuUl&4g|-NBy~k(B)fR+FnS=zty>4`x82~fA@pogc=t#EU81N%Y;pD zT%?_s4%DCA=|e*KyCc8ZNNC;BJzJI#s=dUg`6B$TULV-u&y?p@hC`-m-UBLS0Lv zqZ$#)pXK}Zue4uKx3J^i9wD^g@81q?B-Fpv(+d|7Dn8Pvdmf?Fy&k*Jm(WA6j@{LU z(3q6%2dfcE>-%ln)ASC0^w*7l+(oEa-{s-Y5c+t_nwoP6WsaKpST><2XNlqO2a>2yJQH?b+K1y&3TK%4US- z|CM!1EkeJ{S$PX=>|}(sMc;Fe{6uf}KmTj#ou>%h*Y@GP9}^PhpPcv}p&n(83(5$s zo3L%$CPKG$=$f>a&;xJOthbWT%HtbqFCi4xGNc3Fsq3q=?@yaSX(xVv$GeD7_f1RB zjwAH*{K-+7gl^ruywM;+>*M#%N+Hy`+oJ;#2z}LQ=b&gpFMB;(97*V{?$0i7PpI?k zwl4+|>SI~{*v*7)-7qw{0ii1g4y~_2=%QuZvwA{NJ?6B!Lc0p+_B~MR0-?#Tp6Y*! z(4G~)9s7h(3*oj0-Y2y2^REY%5qh?GZp)2?{#^IRr%w@DJ~4jkLxhfe{`@Z`gz|R$ z9-K$0_S)*rhY|W}j^By|Ld9%QWH_N);7iUo6Z-ws%%lEG2(<2KB^~xH4Y&M}k>vm4! z`x?AieD3l@N;`CJ{CU17!ac{H>t>>~)+-)9Gm=pGvQ}Pq5bAMY=#zZ6g}ksc1Nd$W zUe$h_#`AU9nKhI1ZvCs^LB1!#&&zeEc^%8LExIXG%EdbtY~;HrEWLeMHkWFI*O1zL zr?atv`{r_)tB?A2-4fzhF?2)HGD73mHjd`IkiB<4aKtJ~Yw+&7GoB>0>=XZ~PZ4^4 z>+wL2R`^E^Sxsp>#`V zy7JHWtnI!0hc21Y4ldY|)`igh4^+!;NhoS$`}Ylm8uw`R=ufm~L*UEn8XPAS?tjm) z?S!6;YJK+((Q4*dQjRk zk0+#cB?O;SJji!H+p*!fC&DPL=boP)ZbxYFmoF|1CRFlkW_BQ<&TP^7TL}Grd6$1v zLNUkgncslWJ&z9UUz<>T#GY_JLRVG~tj70b6Fz(3gU|L}lH#vW}Qe|NhmMfG+`d0r*A1X6%!ggeAqL5XNo^BH$IR= zX_r^}hV%U>dXAs-ZgWcOw5-=Ue?sF2U;gqJ+BIVOM^X2ELC8CP>xTCTtv=8=cPpXq zp3xtBme90IH|v%X`tirwY!0E5r6p&Ic)mw&I+#u9*{7!O9YpArIvL+35qhR>@5CO2 znjZQ4=MIE!oid@?ZG>7E)ZM`M#@OG+^wZ6h_DSc`L3~Gyt#8%3P@mFHSBtxu?}M>z z!SuBTN^5fa?-6|Wi{PO}KDS0pd#K=gwwD1$YOU9tLTJI(_JZ7nx@4r#lU}eXZl(yl( z$$|$6`G3~88{ge-X@ud0v6S}g!H2dEBJ}a?okt}SIvm$>X(vLF1K;R*E1_-$xx*U~ z3M_j1pg*CI?t35ngLck%WNqU79|)~jK6mp;LetOw@y8)TmT7)zy9n*RwaHVj5nA`b zim#q0G%Ec!-3mhO7Cv@iKB0unA6DK)==*1e|Hb$HFnzoJ%1BCkziG}O* zDXr;*$Ge9S8vJZvP;)}ZMwc0?6WZ`o`L8ovAmq3JJwHGdzWb4Omj>lH%J zo_yiMWrPMB%a8J%=QfP_=;#Evba`JKrjgB$WAa;I8I`8W+B`)sN80@{RC$ zU|erxT@1_pl2G5*M(zH9P@lJpX73cc>Xi~esQ=43%X!+U5rczxzOG{~?d7`ae&ON}zLUwa zllxcvPIc&8&0F{*q37Pn9>nGQ@={*vhm>~53r~N-+t%li^LZ~*+G_{Sjo`cG?HqN- z(FZ7P$`3t$n?h*dg1{3dLUnbk9vwhvY_%S-F@$PoE<>aQi6d~Rntr zB=pLaE=ML3+HL+g)kNs%udR}Y^0ZsKevm?_>3gAd;|K|Rc6I1VsP49%F=2$d2JUae z_t%?!=Y8)8loq;i$)(1G#(y8XydI$u15PceNhqxPp*__I)$1|jCNDxEbq{3!NxPe@ z-*C@EzY=;a>cd$V2u=CSx6xTb&EB5W@oPc{^~KvhCA8&e#{9#Cyoa0T?xM{)A%kN`FouSPmexA@Hu|K~4IH9~p+J`PCw5;ckN%s)iw)koPsf1d;@#3F3g#J3W@?;jF zc5nQ7axkGa4G#JCBQ)>+=svv&z5Y$Zc}7Aja*LjeBJ|{Q=Gl>iy4(~{9!6-tamF9* z2)*m@r63UzX`n+2RrPLYHtr?-sMTxIAB2*`I`QiG64(u5- zBY@Db9lIA-C-i#mp^H9*@-nvGQ%?J}4F2KgsNV>^{+R#Qd?(LSM@}t2O=$y8RD1md zp(a0mzx+c&ap#X8;QPsx>DR3&rL@48{MqY-w&~tGwSmx{<5MG_BXob`$B(Td6!XLz zo0bu3myor2F`-2l8tj-yXv1BjcFrcW^T5F;?<7>G>~umgp-;lYmgN!}u8+Li#MA2C z_1y?U5A}R_YdWDldF*%+q1O^ZJH`MbA$rtU0(k&q19i#aQ7}k^ZOrY`VyfhzwP+)GD3a_`xf0z=v?^B zL+A^A;GIJVZT5ca-ef{wef9c7 zv4kdmHPyQ_p=yWXM)DnUE#d10Dbw!pR*pExEj#; zwcY2mp7{izeOn*isUPwfppl;)do;NCQ9vhlnp9tt?y?LI(D@*aA3;_f+s>O@W~@p)nvpxSfpt3G>6 z37})Eo?`oMz6;Q`e}7TcZVC^j%IwAy z_XSk;(uo6K-5-i+uYEsd$m+MA867asAtY6 z{`70FR6w3!w~;#ZtHa>M5AbV0-Y^+`I-P_JQ;}tGW`=2sa=@Lj-2*dugXt}hQ7Ez$ zn)AjA;SuN=2O$r1ke`<`Rj^r1x^kT6~W0dc1yEM*=<#E(bLa1lXQF+On z7fa?GYBsVj*(S4+!(L`v$sF0Fven_NWF<%CB`Y~9FPU>-$;x)f^vsQ>{c0NUPz|_q zqN-@ZQ(a*2Hrpn=)Z$!Ip?S2Z3gwAP$7oU`@-6qA>%XPBzwvrZ`c<)xO6{iOCn|gt zqZbvfa-#B+XbyneZH&3F$TA>*vWzUV+c+rSAf7nB2aDZvuD+3HZB{0xYe#;H{kM8> z#pVoF4iw<1tj|iJ*NsH1IHJu{6loOQpC+anm>^wGhzlQPRMz!hk8^GK8Tm2b|E~T2 zb^B`VyE=20rbkVWc=4(A1Fauu{XiQB+B~4m1KK>G%>&vzpv?o?JfO`3+B~4m1KK>G z%>&vzpv?o?JfO`3+B~4m1KK>G%>&vz;Mwy)rM?Jy*Bm89cLn|&-R_tBpCsnwXDK`- zWsZ`YfWz?qd+Xzxr0v{e$L-;fO7*83=aQporTR#Ec!bVVU!tO`lssVnOUR6sCvhy(A0|{TN|w8#*gWlUgICx% znMCPH*zV`ri(+#glZz5(Cj~{Wy(qT6E*B;KMu@^)@UdpOC=S~iz(-|mPTiRM8uS%k zESG{@lteMncD(kYRFf)`i_%LDtkIux`qjgH%EIytJMqW5H?mp<4Pj65Uf!L(N~(P{KnDT)n{?*~^#3CnRUY(r9d_gQ+STzFOlo#7+-7>qg92=k^xu_rAWZXY8>J;dB)1 z<~(29@rksGjs_nP2O4sR^Gt$~(=No|Oyb9kVgnpDw<(E}vfcx&_O;I%0Q3KMIk?rn z(6S9|ZXW9UrX;She)|$u;vj{o2B?%71RtOASR${duC4^M%VvlKQjR#tft*ysk>#?o% z*v@(ku^z*$$8hU0!g}mrWI`hye)r&nm!Ix8uSU86wz%FiY<+|FuqD&tdV2x&*U8>S zW1b!=?<9^bFk4KyLWE;_MJdE%eQ&O!d~zD`bd;x5j)^C-@_h0f#M4QhQaL7`&Xwnr z=OCUg@|4Ok@pP>`pF9WggonwKD#e@!KNsM7;5_bi1kU4LN8mi}bp+1iUPs_O?sWvt z<6cMLJnnS_&f{K3Af5>K`T@T6CAilO=uI(q=L4$8y>7tkajzS29{0Kd=W(waa31%% z0q1eA8*m=?x&h~Lb3Wkxz`c&ZdEA>1IFEZBf%CZ65jc-~9f9+>*AY06dmVxExYrRl zk9!?~c-)*1BK4j0o%LPxUG-7=Zu;)}XnhZTjNYh^)%VoL>ErbY`b2#%eQ&+q*DtpT_Qhf7`Ya(trKF_a=e^Lm*Z*`N|)pK!7b;P zDJO+Qwe=+7D{D)9WChlPo}Yr`I=t zeqjO9O&(Eh&pqZ9=N9Cf^DOQdtS~EYY^8lm(d2||vn9=xg+1KmK$jElt-2WTxVP%! zJnpT!IFEa)F3#iLs*Cfux9Z|NZZ5z1<(_-XZ_eZ1@|*LxxBTWj?sWvt<6cMLJnnS_ z&f{K3;5_bi1kU4LM<57sH-0KFM$GvXAdEDy; zoX5Rxzj<33y^g?n z-0KL$gBlO#kem#rg01^SK~TYF2ymAYZv2!islS zY;;p_5`x)w*mPETIc}FZGjniKzl>D7G=~Vk9M^C0J8pmiE31(5kf2l$T{#D_E-6h| z*yQDu6y{AI+_u#-9cDcFpUq{dUeeNWG45jy-6=`~M9K71_g?U=QH_sC!J<5zHx?=p00 z^`36!$w(NIAhJ6>@}3U;mtldGRcOFGh9Z+i7@eOJoEqP|jnLjCgmtTF5aZDZAYGOe zN@?c2@ss4KZKJ~zI)tVV9gx~9GcGfzH!##RWEbb=PNnQaESQ<%oIiyU#zNJUi1p?b zSc)K%D9#YbB8sC-3&y;>`huxK`%yxKwHOqaGma9%k(+Y152xBqIXoB1X7$e%Z^EL+ zDxUQrgHG3j^7;Wn&fNl3O%MVXn&SyN4gNqJ-P@qTQU{H~dFiNFoyhqFGf5`WbK z-mJCCXhWx=S_4+6<2!xgTyhh5>Gj4evH*@|BQYvDI1x>tU)j%S8P=d@X@JiklEJ;W z!5eTPlT!N(vJ+P{xLqO>?4w%DlQp0)xZcj-Lo5o>HG?Y@QuU$=gQL4N1nUS-(F|Vo zELw@g3WNK|4X%e(i;^9@e`SHMI0RXW20dH@3WNJ{gX5wN0xY*V5Py|Zx)XzIi#AbV zo_ACjyq=xG<=3qG`3fnq{Tf%0|vSnSJ=8bSer5Qk+ZY4(NShW znWWiTsS(eKtJ<2;7gzB0&en2Cjcp;;THi{RCde?sGZ&@nG^=e}q z5IRWTQYN|}!vwW)t<;F;#8qw0SVK5>*99s6EdR3{pGLx)$f`k5rfTC_sS(eKTWw7O zNozA-I$!?+aryaL?eVdoY>76ml^XG!xV9Mg99vu$NvwnWeymIy+}dLNy6FCTSE6cb zXNz&z--NYg+wiK)8UzuDNA097YQpo5s-r%lRA}F(}U7{)TXz?4P z0oB3nx+sD&t7dDZMm#63+1hh#ajhlMY<;aY>^bFYw)PxbTx&@*TVHDpdrtX$F^=-s z?dd!qcCdqt_2DJ8Lfy|ywx@GFR*&_^i~`T?1CPMH>aczTl6oc4`;G4G>HNfPwNnjm z^^E03iTgnJlHI-$DHR`G*wdLAc@cb1XD|DWn(a&PA!Yp$B&lFe=W(ow-JZ^@Mw$zI zI_vSdB4Z`=EB_Jqbgs!7QnH(SI+xQsfzl>YhuG-yNZlUmi-L@e!EpLW%=sTN94QTE zN%+>P>ibJpsoZ^Q6$)gPRA|6F)B~vwJ=1x!GY!vM_X?=%mEZLRz2dD^t1!|*{?;n* z%T@H1t!k_~HtGa*cfG}&D{ZRe&l}sKj?`!X{3eMYPtvewYe07213mZs3v9xZ z!K=CE{cGdWkf&)tc5X${RF_qnGpMIo?W)M5YU_HMGw4-Ou7?$(YHMfDpUohAR~`Sl z3~z^^Y>xJvN~sagi94}1F4`QOLB~OA7hMyX^nwf%)SSkY8u6UC6I)k@bvnOjlO9y6 z$}A{TwQ;S~i08yrZLK|j_8bFUMM-4WVf>O>8pqmV{3@#1bBaV-n4osjR%*m^;!bR>z8Gh4dwv|-B$Hl{VS?IXT&WSyi94~iwix#u z6J0e)RBf#-#;=-wJ*gn)i*eVb6qH#t2SKGqJSVO?xLp@dP-fL^t<;F;#5G%cjxDaW zB$}%Y&DjlI{|y0fiq~Kd8|1PiR27g$XFu$lLwu3i*&5><5}(ZzP_zz?D?y*@tu}Ymm{TY8^;i5C>7IG8!C?QTHN4x5``dN zUuXs=l~nm3#R#v>4Q_2#s>!yQy=L$#Z)<7*H#kC>=@tGpa6JGIXPZMF`R;neuJ{oY zafoUg^*HLpGg4G<5!C(X&hDD6J;SbQQmERRXTbSGC>($P*mf&RO<=(>Q5)Be zI`NDYC$`3M{f~A&*3Q?SVOKRNw8glZR{q0js)JiwjQ=Msu4-%EQpd$OemgRV$sS2D z!BHvNN!w8;o{{3j*6NFK29F;DS)^>urebStG42_bb(F%1tudpv7SDYi%)J zrLC$id_{|Kbq>c=!7)io@l*{s8QiXlE4Z3b%Xi~7pxOGy+u(XhpxOF*Y5K<3quKh# z+u(XhpxOF*Y5K<3BXd|m$8(61vwM-u`%n~e6dT1_dcAtpo!vzzckw9JcMYB0nUPoG z!p_GwXZJ7RMv}_T$5_(8c6R6D$bZG1kN+>7-2?a+tx)6eK7i)zF7Fus4J^9}fUdB0 zeLn}2HP1-Y!hbBtZH;$yDQFL4MNRytj;gliEp&Vs zi=L62;eo6z2sS7p?P08_2~RsZv9(Hf7zJT<{1XYD!&vt}B_^{MftGC4alt}^|Rle(5ZN?I_<=uCppjeH&o0K8IF1 zdPauZ3FIWwPArh{ZLDSY7K>>g*)gn>eJLF}hJ{25qqA~C zbMtdlxzk1N@Q{eEV$RTP)7UmPmGcp6HEgTJES6njYb5f~=YtMUd~kc|%8%zh@$uZ! z(qLxRvjANW)&n}@j&k{yT_R71co9`_Gz0_DppViI^@0K`tI&XXNYDz3bd%Iz^@(0K zmJfS&=rFsCv4ZWkG{(+6Gbtt}_o26%wx81j(+7-tXzO>7_HOIB6hGX0?%Va+&w0E0 z>RcN6jZgR5wV`vTUGr-k+QnE2rNOaOUM8iNJs8i#KPew8vE@_eEs3Fd5oX5P3G`=U zsIo}sCCwt(h9d{QPwY7AQeq4LgEI3tYu$YU6KvW*tyC10=ES`2n6tJ|MWuT-EzoF{ zkB+{~HgahqwG-ZcN@!iJjxNJ5TgujK&%|bXE?$Va_2w@gd#7P(OfWlDi`5}AObBAZ zJiB$SPs)hPm|9@6tHdF~rv#tv{@?5N)`nf?b&|H#Dl|Z)%pmyqjK_j_J#}>@@U8*7 zpViedZ(rtDm+4%p*-jK&Z`4Um<{6|ju93(6tj7TBvA*@#$a-vMJqp%iOY2cIMQdw3 z*m`VhJ+`wRL#)Rz>rs5EBElN)U}Qog9ez;!(`v1C9sDwM0c>%-XW04%?R8AIxZYks z{dL?&oK=ZXc_(pf2Aschg$T#=ic*NDLq$0pX~fgfF}(_X@M%0!g<{52%N{gj=*``>j<33y^g?n z-0KLO$GwigdEDy=oX5S6Ks*ud^#hA(ibZg*8!+N=uN!b4_qqY+ajzS29{0Kd=W(wa za31%%0q1eA8*m=?x&h~Lb3Wkxz`c&ZdEDy=oX5S6zj<33y^g?n z-0KLO$Gwg~JZ{bhk@`;h&iXF;u5jVlP2XJ~t?!|a(Hr%#`kwkYeY`$FpQ!Jp@2%JS zItJ-PAp<&<;0sGgSN`FZhb#Z!4v&lfWP)i>(}1P{O#`F>yUTI*w?2jC`MfFV=KgnI zC|vXYS22@D-3tHTE)9fPi3z&ytQXvR-HI3Fw!klwS+Z%YDKFcWtcq;6Uh!@;Gm5uf z^K_Fbzd?Y5VMI=FcpD)Ie&Ft9PH+U`2xoT}S(j>k*lm6##E(CCVL!3i#hAHQF74QM z+iJLNTp!+QzPl3}%-*xzHm1+KSTFHe1}q6Bei^X4R#(U_-8Q}mw=FRkpc-cAS76E$ zicFT`0!pMG!$_o<#{}o)6paZn7iF8Xa`MOKW6svC1#9A{tn6&K*$hpdUeiEnpw8e< zk&zuD+jO(0jLy%q6z1mykIu<2GGXeBnq)^_i+CU1uiD$}1;5Lbh(9mjV}o@UVpjIu z+@jvon?WH>K_R-1>{U=mifByUR;w{nO_SsD^F~WWh``Ri3WWrS#(Yv2OZg2@4KqMd z0+wY0(X%Y(eERYtFVAomo+ubX+X}HoMa8)$A*jEAEzC0K6$zGcCWsUknnqiMyeuHL z4XtUYX&61uG|q!DPlIBqwJM>X$5pgDNBnrN(7KVd35UW$Ecd2c-7Of{h zHuBnT7pOI1ov0GdmYN^O%`qrHrw0~bt#yb+lx}F~Fq8>JNs$uKH2%;{?M1aQ1&0gm z1$^YtRsjAs{NPwj@G!_WG&G`3D4^R2SqnxHsYan~v>@4@Cl0Y-qRq6BI@z2jw9m(` zHhpq_;e?`bCNOUM!s0wE2MmKYO&=VamKm2Wv;YP1hGrM%=1v8DQALoEw~;3du~-wJ zwrGKJVgfu1k|J0(j8ySS7u-&?4`+twG@W|!#nQy-x8}sW^Fa*GByi7DdLUw-n_~Tv zRT7m+d`b!r3v;+a=F>ms7lmRE>zn!A?L9yo#f(n~Fm2#dn%7<~3&huhcc=>m(?(Ay zYY6^S7VM{u*3vwBPMk-f8fHkz&w)`s0A~zaf%Ye@3^vGnA!{jVGp@y+RjXh7ia!`J zt^$DtSSQ;F{Hr+Ifxz)b#)i72MeFmzIq*4Dp`}gW-B2_Je$wh22hYqvYLk#ZMo7vV z4-;2bp6sk*CwnVDwv_E)ExC@NjE^r}4%zMdLFp&>MC^%;@JQ?u+^M$7&NK2~K1ciI z;^RgV!2U-n6O*m?zjeO-^dEzwT8a;P8B?p!fR!2qKX@t(0q%uEsHdBw1J3&a;QeoS zn69G-okF77Vln59rA(Fm*zSKb=>~jdJU5E23kcBFsf*9c*RU<{57C=&JV@v;pwooT z0{V-PPPc{Cw^KVLdrW}m(Ly>*a?ma5X>v$N<)URw&PSdicbL*m6N}*_4{IBtr#UYh zx<=*o#pUM~m@Hj*goIe@Hcc@XS%QOx2gS8(nc4F8OgMG5iH7C4 zb(v;IAbE=>n=PZq3BfRrnWk_-N5f~Ih43g~fyH7-5uD!PS|%9ev1E;Fhdw_hOa2({ zu7k_JQH7?g2>^&K!U-EF7H3(fd>x$d*;H?xX`R&D(Fwn7+O-ydX0n$#(h0A8`nBVl z(M?)Qw(@6St;71D-1W>0s$RT-|)U!h&$4d#EK3ynq7!2i7lbh6o4Qifq9oHq=u>?R9X!#GR?xUNwQbWCzKj&XhA z?pgsBHny9gc4yEE*9u%zn6hxja%6M5RDjmttfkK7Y=Iy)80IYpM!cCN>D3-c_mY}d zo2)CSwnaKu{}8n4(GNbq5EDKuD1TDOZny;v{_J&hW0;9OR=z^KDZ>lnw>Ma#ImeU? z%QpKO6fxVqq=t5`+-a;YL@I3ZE85g?(H`ruf9k;gHmP2&b+VC%7?Y{$m95k%cz>yEP5^E%4I+-Dk1m?qZ zBQe!Jw6c$4Y@aNt;&U9np;gt-aj(c)>kgDpU3EK5a`8TdJ2hZl@#O=4R2aOT&EVkq z54mw<+U6G@#YXU09%Cv7-^(fVYj1EFfzDM|)8N_^B-gHp)eMd_nd$3L-s&QU4{Zeh z2t;LzRfw74P0~_$lT^&dm|%k(rNPUyTJTH_$X}nA&OqMG3vQxjgT!{WHa-rvp0C;3 zGc8%URI0YVTrR#7?#n(eKMK<&6=bZT+gz=g@2?e)>ob*Cs^z#L8c?-0dW$>mQMcOI zS+}s-MW4;w*0h;`pcGo0!<8EGoVeW9()rppt{FS1dY469gQ?hB8`qv=U3*F7wwBJ< z>f<^M#x?Hp5rk`YYY;>r9<^~TYQpo5&KKiga4`N-5ZP{l#yaaZfx*wf;!O;K2*jgi za8VPUca#rqX(H!?8?T_0wodiF=#whm+XcFyY>{SbrA9m_F1NKbk=st%UTji{v{xGg;G~TUa&Z$dZCtyf>KmR{Zfj`{w~cFO z@@b24XPWWgbh)jiIb3~Qrx}m?@GbYmP5rcS?ZK9H6wP5VF1xC7%&ndxD3jBa30mA^ zG$0?`nyo#?EZ0O3Ra;+Hy=$gczo1Oi#*u%B-5Ll^XG!xT>wSaqT$< zx{8vh+S>K<1!Y!kTq`x=IdQ%iM|li8@rN(g)6dS)Sc}yRM=%5 z%_)1AGi!WXc{tgqSH zvn=SkG^)0iKhZp*dZVHKj4ly-=wWPl;P?y&C0_&CR%I`w@QR<9apo^*&#H zPe8{S;-#lHhojo6Opx0e4NSo{u4}O)6+CkhJ4z%wyLTs9jvw{ zktJJmPX*<;PBZ$_Js}^x?G}=tY^%05Q);(S?7@9WlCjbcd?xwFN%uXnnBPrwK% zpEi+ujx}A7L_U#AHgvWa_w_yvw%#Y*If`&aOi`>@IL~c82d$EVhU;n$jd=;Fu zOM3S9%Jf>SPpbhqhBrY_wnSTuD>dRdaaCKZpR`vS7wgNsa6wMu(<386*%Ixft<;F; z#O1b@=5X7%b|$DcuAOPdgVU|HCJCiETzy=p8D|CX2eg!%9svu=rfK6^sS(eK%NOG$ zm3YJ64G{%5@@iRcumnp?FEpTPYuC#YT+gcIywMs^ zwY4^`Z?sviN&Tv}cD*LS^{iUX8?6Dp825o?8K6u@X+9|BX%Udr?h8+&J?c5ux$< ziSY5g4q;(kY8q1g!$P`*h53&N4)5H#O;|`+hmM{2r>JTg`uc~5bdF5uXeEY+bm^4P zK}Z>KTX1{&+UVHe_%^-$+edZ`>txN+p(B2s5S=n}^K;;Pq?jWeN{Q$yCWdC4#@d?8 zJKF?okcumF?vcrz&USmDbpPemiQ!Qn-;Av?&<&_&_0ScdSc7|C`crQ6YdqGr>B432SRM5@@@;-~SIu8HQ1<~f zUGA@|QX%i9n3JQ5xA~Rt`l=gA<=g$JxNo|*&2Lq8-xRm9*S7h!+v`=lJ#E)XD1zG- z;X5c*-Qe3Do90r=_3_tLbI?ZdWMjR~GngP-7O3wmw#4^*^{86SaiX>cK?LGayA3UB z!t;*Ss~iVggQSi-i>3ISuYPxJ&DNf0MwcX#+giGAusurjW&6vQN!$FM@?O&5yV@BI zH5UPwbn9uERBg>K7k`24Mmv99@at>dU{AmjK8I`D{Cb)#E0f1+>t`{abiTHYYX;+5 ztq5@qCMb!T>yJ_+o)fp)ngo*OaP@JG{<_=`-zL6bgB)pVGo?m6CoW%%+YfFqI2hk? zn_pk=ZngFVe2A|JTZ141@u(SG)P(09RVQ*jxE&YczTWo*>*;=U) z&xy-zEluRMleW@dcVYF#`kAmm(@xr+V?viDQMI-DNgMrj&sWCrl0|>rx1=XVr4vXbq^^S{v6l+APl0u$yW1_O<8<%=B z*b|{X>o*{&S5hjL(N330+x+5|z49{U+x!L>)=AaB)iZXy{s}sY`EYCCf3wZ+=VC#Q z#{eTwyAb0CKww5mI0=YMt);92B>CEBg>8P>CxwjFU_1Ipu=*@WcU0tq#LgwG#D|F` zijG3I3=FpU)ytkR$S7crV^2rppFiVY2~W$mBRh-Ot6B6uZWRdrNcS`FleUKU-@~vz z2DcgieIehHX$m;#?jmc_3Hxj$yG#KKxAlRUWJY55-rsp%ObZ^<& z-c62e6O>8Xa|ESEJSQ$+&Pw-|tI>#e*htmrD7K(X(rm5Ni08yrZ4GaUfrPk5Yb}@5 z*a2cqeXWlNi6Y49h&2Z;rA9m_?!?wO#FgWEwGmI>O?)dneQV>|b8M@WL~d*8d=0O* z)MC~pnRu0@WaM!{nXHX#rA9m_u4-$(*P_zlI!#4tL7AqFYo$g!C(f7Ud?QtoN*dhS zV%&4AbQL9$4{mKSeiaq#fkmm>+Sy{9u?DOIlfOzWD4V36w3Qn1oVXKPt1rfJ(H6lv zO5 zijJ8HYSq_c^;k8pP;ks#D>`Okl5FKI=!d}l^g1kINL)fP72(`5vmViR0{eYy-yMRE znc`8b`=2^y#zl?jS7;t<&g&dDB4NsCQ%-oukmPi+RA{4h+t|BoNlFxK7~P;uPEz4365)TS(uh59a*?nW6KG0lN*i<-jub-l8dexT&|5OffWY7+0Ni`vL}#$7K;P-fLm z+DeUhPF!_xyDpxf%&OU1sS(eKYqs_rTU={NG+SS54SP=cnyo#@7S~!5&DPgi!=6*V z%wc6GNCi=RCw{P+fc0UE!9v~7O!l!N#XIrv0NfkjiFfZ~MJ?h?$Gza4`18NNGz{O3 z7wi6~@5GDmyx$=fO1em06+4N74g0-HW_nNq3`81peK*~2M6I4i`Yc#Gv^i~(dmDYb#r!36vRB@u~x+o zsKspB@djvscTefH>xK>nPc6yQX4czGq|T)v=aFg25(|# z@Z|iwu`+%f_nN_F3aN@vVenh+44#x{F^x49itWOO(wf0VZB)@CH#iC}Grjha!zyY` zx5Sp;2E(yT;a~ZX?G;PWjvl4~Ra-N-o2B+b2i~>#)LZ+=p@-SfRdJly8i%;!3mtgR z<+yi!S=oXN6VyJ2rqqb%#O1bzmpH`mFMP<(h-GL;57U6Et^bt|nRdSRFcVASXp3=4 z75^VVKDf2T_tUrhEXMzV6N;d0ljh{F)QIQA<%3(ZwddI7T1cYV z`dVn!Gt1O$?U{DD7E)=pz7|^b%rZ4wd!}8kg;X+!73&KfxdXB$CuPJrR)#NhyysQR zYj%yL{#Gi!&_O$$jr7A5k&-eXb}%nuOLh3TL^fmXbnvEuEa(EjDdr_s59@s5w%Vx% zc#%UqirN0{3mvzJ1xZKa%re5pmy9Hhai$UD{_qh)%W>m#H7#GoK>Aqzg}>6rmSfl& zBOb-9cw}{+Dub->LdX61LPuTL#c{Rnd$zNBYrmh$J23B>G5+cB`Mn+Go62?8hKnze z5Ig^YtsbpJ?ieN`G4X{CF)b}SLWC6^w@6pdm`i-2gB>(vu#Ghyk~n8i?-=;TU*E@s z3{HCbc;V*KKzvU?D2Zfuu%qQ0rEeWlet2mGLk>8r<$)x&gnd_fb)JxA9_bdbO}=Vn>VqqA~yrV4RDgTb2xqrS+L zW6m={4!bceq?xiN6d_;s_~If9oHM{D90oj2Dp+1kL(ycjW%M{9*qoPbn$o7GVR}u2 z|7a*p2#*5fZ_YD^j5S$8#+Y+Vd0Dxpz-az~=MW%t!wiY>$^CkTq-9yg1x91$cyl3~ zmGTRxhCl-fFjHfMhC@;>4n94Aw_(d7#s_Qx8sXP zY3(SrZNh}xKsnr^Goau56Xa$vYLjV=R7F2D81sc#exWOHrR8KzH5I}LFW@x<4HHMFY2yTNHH5~9#X7EFHVb)}!eV)KCp@jC+gd&rLNF15%GtL3#i zuT;`Fokt?*JmR2Gn8WxQH{DcJ1g9h35$#z!5NozMD<^+!NRcJ0&=Q=Vo1HVZoxm5n zc%o3_>*%aXx^RNNPp3gx@+z3EvvnA#Ziekz@u}Yh{~lK!74J!~-rFDSzd!1`(&dk5 z-(42`lPlMX;nI2D9#nL#_^7{haidnDcCCo2v|cN&|2@pV%n@!vs-wwXmY=UQLE5{m=TiJ|>$z{&Yd`1h>Z@~Okly%ouU#8DciJ_-#-Uw| zl~5WSOOa*^kY4s+y!0pKV>ei^Iq-dA$5EFO zTlgQ8na5cR?-Q6{(*|m#qM$S<=5@!Mwc+V@RSkhgtK@Wq=kU_Vm4MVV?(;GM;(%IJ zXU*91}Rx))rb6cRa;#Rc6;4Nt_Dk) zbo)rHLh1Gqxz{*#HNd9~Si2CHpJyq|ci|batT$E_SA&uUEQGCQJ;6i`SR$Zdgzf-T zNN5tEd6EG=7f82{TzU3yZ3aDc6CWSC*fjAeBDoFlM~sznsy={p!9}6cg0&2q1~d(* zYe4<>(HS^>d5Eil;JGBTnKf6#_csD<{U$dMo-EKEo-Q8EPC{W}ejz=fLN5cz)JDLEb8?pg$vYr46du~6 zj)b-$;nt_OymHCqq7$}&r->dHcy_lrTg0HV1^7?I*k?pX3-U;jvW8PedRS~LuLF8; zz!PaHSA1{E7Ej&_wtO8SUqarrskw0SC@RXD${jtN)Ex<#I`vr=K^PAZw@ zLI~Cn8fufKGw>*$h)WFsO-LFMLPL{d)7uGMLL)kbhDEl6zJ%U(*^)}#ax@u}jSn=z zokH?6!M-HAS@c6ElGrxnryw^r@7Y;s-O#q+8EDEQHyu%#wan2l6!L`P0s%%>K@NHrQGQix?IM8Gj%5qBC#+tz&5ekIO!wa(nbWeg&CYO0ImzgS&+*NEYHKWJn zn@3CTCh`%2-Y@N7YG@ZOjSp^9k%fEKIIvJZcEZwuD_rI;LA79?c3MqGbspZose3!A zEK;2jn-t-MrIR`f$wxd=_ZA#g*qN9^Lmf1t==|1}T0Iy<2L`KSK{xa!;64V}d0-|5 zFEkTEabC1N@OP*P*O|D?q}JKxr>Fc5`N6`p%yi68Pxl@2^Yx`&ZoK0qJ;rBw{)gGH z>Mf;frakBGim%!HOibflyDtrWY;#N?YmDC}2c;!K`x~S)7P2s7EU)ePe zoLMCob@bY2(q-6L-Ed+vKsC&ek~PJgYo2BjxI3M#KtpdNlL4l6vnBsSBs#pq8Q(sP z*d4M=H{qL@?%hn2n)+R&4^PN%OxfqASUt#3iA~B*@(e{#r{W*{Yk4iHr-K zVTN;3aIQJ84JC34k*$~V>>>7!H`6tjlsR4Z>EjzdTu|1r1BSyOFI-JKmp9(X*wAEK z;Z70nLX6GP+sm6S%8M86hRrbgLczDoR*e0PhqeA4*yQRjN#w%I?WcKXeiu`e*m_5e zN4CPf$WyiCCg&rA|5`gd3_@HhNJXdu>y3wBd{=tyt&MNp_=wrmDl|Z)@Q#Cz&sogH z>#3_Nfv=vxI}UZ}9S6UwKkG5TdaQ3fHnJX@S&xGC z*wT6ov>sbqkHOYsTkEl%^%!D3hFOo{)?9) zj_DTH+Y6|_&heEU=CQ#H3kJcNlamvvi6_Dd(MdA#bZ|m+l1w}uoe-TQ6HlZQqLXCe z>EwjyB$;?RJ0UtrCY~-%h)$A;r>hg9lVsuv4|76QPv$)Cbp+1iUPs_O?sWvt<6cML zJnnS_&f{K3;5_bi1kU4LN8mi}bp+yxaIYU&V9D?9d_X+zbpu8`?sWss<6bx5JnnS^ z&f{J;;5_bi1J2`KH{d+(bpy`h=6t~WfqU}-=W(wia31$M0_SnBBXAz~Is)f$uOo0C z_c{XSajzqA9``x|@who3MCv=~JL|jXyXvF#-Spk{(fS_x7`;&+tM93g)5q%*^ojaj z`rdk9m7sQ}(WwMqSVFq;4F8))KP)pM^plRU$TLbo&<5~Jcy50InFHt9T zCqJF?QqVgw6S9O}eP1pGKL8B2n;rZ@oag43$t*K78Q&sG${UlP4zE~djg?)QOD79{ zIqrA2?q;gWRv}|56>=G|7LMq9fLOI3zy3Tyg`Pa!hQQ+~_VdyPQ$H5yFoOgV*H-{}M!iyAV*Oy}==Mq+><@w+2TA{S$+W2EIN9 zuWwv15JGWtBbrWitk*#blXi81H4lvVM<8i+ZBbUt5a#tg>j1mYAm%|3fq2wDJR)ks z^N#W@w68^(1(PhPsf54PVlcE%RbkVk1i)z*0Bs`l|HzJpkAKXC&x#sp4ik!#XwM!m6vEr zx)#528jufe&DJ;04%ee*Ra^5rXs%Q*C^Ko}TB#AwiL2V$b@2pcR?XH*jd)I6)z;d$ z_8bFUMM+d`?RxowGOISOl^XG!xYA)IVZZ=NV(dbUAvqznS4MAZJlbgJ)5nVAjfSKo zTfW|jNy!O3S~49-seJ}z#>ZyFhFi0MUrhSY0ja$*iMGi^NIUo3>gV2X;%IXE0=uUxk*>5Dl>IpinX4Oyl$~pqoH4V zTvBp!Y(`SQRGw&j&}e`r$^@GMubuacjZaTYPD)Kk7!;d4Fg7D$KwQ6grv#l?J~Ip% zGQ=7IZ9%yaHd1;g1x7z?1|-BMi5(PVNjH4) z+=o#qtDq-YU9XHVv@{foN2@i|t7=im#FoO|u!Qa0M z2E5TF)|{L_mKl~}!tf5Ek)g2FMpxl~?Ok1LR7Dg%yWNG|g@r8zMSkjK3)0`^w!1B* zq;%2Nlo%+XA{dIbv`gBwQqwMpCM2v1Zx}T^h=#=SWQ+-7Of)9Ov@gb(G!hLD_@J%f z9X0-u0QLLM-Mf3YY;hs}MC@$7nYm~F&YU^-%$;;*S{rSO)*0bVt%;#zPkY!XuZz{k z8n@I}91JRpO?tONZTGI$_F!P=a9?7m$>=ze=)@5t{Ew1H`Ujq>+tfPzsL^ahO|vl= z*yT4Pjiw3TjK&&*fyex{kH=L20D>Yx5er)dm7-j z(!en+>jELhzHJC)Pl=LX(3`#6^g?Kj4nTif*>}Yl0F_&Shgt*RGnpPsty%?w1hEE% zF*>V+#&ob4YIXBx69%RKV% zwndu78Bf+c>G|R~_Vmi*kXsv`|IA(dvUuv;`>UpB=i4Sla&yLyy-Qg~ffDM_9}1Gs zHyzuA5uJDcAqm_7uN*&u>O`gHm{TdKaa1;@&AWS30()c_zoZz_CPxsC$nzNS!rgt) z7+W&52ZoP6rdaiHx1YoP@Nx6y-#RAsimdozD`6j6ACW;hqBDTo|2t zn=k)dbk>?L|Gi9iiNWrroh|1oOAf!!rdP~CubzWmIS1W;3;nd)BcW8$48!8o0u7g? zJQ|jy7HTLd(QtXnJM*+!l9RAc)BKdiJK5QvNF-}X;VqyUN*yUz`I&9!iIVawKe}y7 z>=bMsj|NV_`b|L1!k@b2x_*~47i zV?i~9i{bZSa;H%&hR;dfC)eNk@vEJOcDHmZy;A8_>c6Ozl!K)W5EL1E0Vxw6`;*`$ z&3!`sbF?kU!H(hfsSLX!Q!N%#Q!S>s7kSYqH$OI2ZKBdpkh$vmfAEpda8`Aq8-~a0=l1 z;0!>Mas`FB24ER}R3gaxOe3{(pm$_=bog-6XnU?VF{Ev#K08pde#@jQ&VoCNGHXgc z-l@A?|J|1+FP)974xN4-{ja`up8dZ9aOueY?*_OIV!yfG=>xb<8Uo0B5@5fN0bCQE z0Bm_sKI4E>9uSM{ePX*BvZWgrXeo8MaCJlh=L2 UBZH%yp++-HKv3u1|3w0S0K(43mjD0& literal 0 HcmV?d00001 diff --git a/crates/io/tests/origin_fixtures.rs b/crates/io/tests/origin_fixtures.rs new file mode 100644 index 0000000..bcb4718 --- /dev/null +++ b/crates/io/tests/origin_fixtures.rs @@ -0,0 +1,238 @@ +use plotx_io::origin::{ + OriginCell, OriginError, OriginFormat, OriginLimits, OriginProfile, OriginProject, + probe_origin, read_origin, +}; + +const MIXED_NUMERIC_VALUE: f64 = 314.0 / 100.0; + +fn cell<'a>( + project: &'a OriginProject, + workbook_name: &str, + column_name: &str, + row: usize, +) -> Option<&'a OriginCell> { + project + .workbooks + .iter() + .find(|workbook| workbook.name == workbook_name)? + .worksheets + .iter() + .flat_map(|worksheet| &worksheet.columns) + .find(|column| column.name == column_name)? + .cells + .get(row) +} + +fn assert_float_cell( + project: &OriginProject, + workbook: &str, + column: &str, + row: usize, + expected: f64, +) { + let Some(OriginCell::Float(actual)) = cell(project, workbook, column, row) else { + panic!("expected a floating-point cell at {workbook}/{column}/{row}"); + }; + assert_eq!(*actual, expected); +} + +fn parameter(project: &OriginProject, key: &str) -> f64 { + project + .parameters + .iter() + .find(|entry| entry.key == key) + .unwrap_or_else(|| panic!("missing project parameter {key}")) + .value + .parse() + .unwrap_or_else(|error| panic!("parameter {key} is not numeric: {error}")) +} + +#[test] +fn imports_openopj_origin_7_v552_fixture() { + let bytes = include_bytes!("fixtures/origin/test-origin-7.0552.opj"); + let project = read_origin(bytes, OriginLimits::default()) + .expect("the licensed OpenOPJ fixture should import"); + + assert_eq!(project.probe.profile, Some(OriginProfile::Origin7V552)); + assert_eq!( + project + .workbooks + .iter() + .map(|workbook| workbook.name.as_str()) + .collect::>(), + ["Data1", "Data1Coeff", "Data1spline", "TestW"] + ); + assert!(project.workbooks.iter().all(|workbook| { + workbook.worksheets.len() == 1 && workbook.worksheets[0].name == "Sheet1" + })); + assert_eq!( + project + .workbooks + .iter() + .map(|workbook| { + let worksheet = &workbook.worksheets[0]; + ( + workbook.name.as_str(), + worksheet.row_count, + worksheet.columns.len(), + ) + }) + .collect::>(), + [ + ("Data1", 21, 5), + ("Data1Coeff", 774, 4), + ("Data1spline", 481, 1), + ("TestW", 3, 6), + ] + ); + assert_eq!( + cell(&project, "Data1", "INJV", 0), + Some(&OriginCell::Float(0.4)) + ); + assert_eq!( + cell(&project, "Data1", "INJV", 19), + Some(&OriginCell::Float(2.0)) + ); + assert_eq!(cell(&project, "Data1", "INJV", 20), Some(&OriginCell::Null)); + assert_eq!( + cell(&project, "TestW", "TextNumeric", 0), + Some(&OriginCell::Text("text".to_owned())) + ); + assert_eq!( + cell(&project, "TestW", "TextNumeric", 1), + Some(&OriginCell::Float(MIXED_NUMERIC_VALUE)) + ); + + assert_float_cell( + &project, + "TestW", + "Float", + 0, + f64::from(f32::from_bits(0x43ac_cccd)), + ); + assert_float_cell( + &project, + "TestW", + "Float", + 1, + f64::from(f32::from_bits(0xc7c3_501a)), + ); + assert_eq!( + cell(&project, "TestW", "Long", 0), + Some(&OriginCell::Integer(345)) + ); + assert_eq!( + cell(&project, "TestW", "Long", 1), + Some(&OriginCell::Integer(-100000)) + ); + assert_eq!( + cell(&project, "TestW", "Integer", 0), + Some(&OriginCell::Integer(34)) + ); + assert_eq!( + cell(&project, "TestW", "Integer", 1), + Some(&OriginCell::Integer(-1000)) + ); + assert_eq!( + cell(&project, "TestW", "Text", 0), + Some(&OriginCell::Text("test string 123".to_owned())) + ); + assert_eq!( + cell(&project, "TestW", "Text", 1), + Some(&OriginCell::Text("only text".to_owned())) + ); + assert_eq!( + cell(&project, "TestW", "firstRow", 0), + Some(&OriginCell::Null) + ); + assert_float_cell(&project, "TestW", "firstRow", 1, 5.23); + assert_float_cell(&project, "TestW", "firstRow", 2, -7.0); + + assert_eq!(project.parameters.len(), 41); + assert_eq!(parameter(&project, "ERR"), 1.0); + assert_eq!(parameter(&project, "SYRNG_C_DATA1"), 1.25); + assert_eq!(parameter(&project, "CELL_C_DATA1"), 0.1246); + assert!((parameter(&project, "S") - 1.28889201142965).abs() < 1.0e-14); + + assert_eq!( + project + .notes + .iter() + .find(|note| note.name == "Results") + .map(|note| note.content.as_str()), + Some("Data1 Temperature:\t25.10242\r\n\r\n") + ); + assert_eq!( + project + .notes + .iter() + .find(|note| note.name == "ResultsLog") + .map(|note| note.content.as_str()), + Some( + "[3/5/2009 13:32 \"/DeltaH\" (2454895)]\r\n\ +Data: Data1_NDH\r\n\ +Model: OneSites\r\n\ +Chi^2/DoF = 3008\r\n\ +N\t0.800\t0.0346\r\n\ +K\t1.75E4\t1.86E3\r\n\ +H\t-5406\t340.5\r\n\ +S\t1.29\r\n\r\n" + ) + ); + + assert_eq!(project.resource_usage.workbooks, 4); + assert_eq!(project.resource_usage.worksheets, 4); + assert_eq!(project.resource_usage.columns, 16); + assert_eq!(project.resource_usage.cells, 1889); + assert_eq!(project.resource_usage.metadata_records, 362); + assert_eq!( + project + .unsupported_objects + .iter() + .find(|summary| summary.kind == "worksheet columns") + .map(|summary| summary.count), + Some(23) + ); + assert!(project.diagnostics.iter().any(|diagnostic| { + diagnostic.code == plotx_io::origin::OriginDiagnosticCode::UnsupportedColumnSkipped + })); + assert!( + project + .unsupported_objects + .iter() + .any(|summary| { summary.kind == "project tree" && summary.count == 1 }) + ); + assert!( + project + .unsupported_objects + .iter() + .any(|summary| { summary.kind == "embedded attachments" && summary.count == 1 }) + ); + assert!( + project + .unsupported_objects + .iter() + .any(|summary| { summary.kind == "window presentation records" && summary.count > 0 }) + ); + assert!( + project + .unsupported_objects + .iter() + .any(|summary| { summary.kind == "note properties" && summary.count == 2 }) + ); +} + +#[test] +fn recognizes_public_opju_fixture_without_partial_output() { + let bytes = include_bytes!("fixtures/origin/RawData_Locust_Revision1_TIS_Mechanism.opju"); + let probe = probe_origin(bytes).expect("the public fixture has a recognized OPJU header"); + assert_eq!(probe.format, OriginFormat::Opju); + let error = read_origin(bytes, OriginLimits::default()).unwrap_err(); + assert_eq!( + error, + OriginError::UnsupportedOpjuVariant { + message: "This OPJU file uses a record layout that PlotX does not support yet. No data was imported." + .to_owned(), + } + ); +} diff --git a/xtask/about.hbs b/xtask/about.hbs index 699b3b0..a87ca52 100644 --- a/xtask/about.hbs +++ b/xtask/about.hbs @@ -39,17 +39,46 @@

Third Party Licenses

-

This page lists the licenses of the projects used in cargo-about.

+

This page lists licenses for Cargo dependencies and other third-party projects whose material is distributed with or adapted by PlotX.

+ +

Non-Cargo third-party projects:

+
    +
  • +

    OpenOPJ

    +

    Origin project format documentation and parser structure adapted by PlotX; public test fixture redistributed unchanged.

    +

    OpenOPJ source repository

    + 
    Copyright (c) 2012 Juliusz Gonera, Minor Laboratory, University of Virginia
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
    +
    • +
-

Overview of licenses:

+

Overview of Cargo dependency licenses:

    {{#each overview}}
  • {{name}} ({{count}})
    • {{/each}}
-

All license text:

+

All Cargo dependency license text:

    {{#each licenses}}
  • diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 6564361..f45490e 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -424,3 +424,25 @@ fn format_duration(duration: Duration) -> String { format!("{seconds:.0}s") } } + +#[cfg(test)] +mod tests { + const ABOUT_TEMPLATE: &str = include_str!("../about.hbs"); + const OPENOPJ_LICENSE: &str = + include_str!("../../crates/io/tests/fixtures/origin/OPENOPJ-LICENSE.txt"); + + #[test] + fn license_template_includes_the_complete_openopj_notice() { + assert!(ABOUT_TEMPLATE.contains("OpenOPJ")); + assert!(ABOUT_TEMPLATE.contains( + "Copyright (c) 2012 Juliusz Gonera, Minor Laboratory, University of Virginia" + )); + assert!(ABOUT_TEMPLATE.contains(OPENOPJ_LICENSE.trim())); + } + + #[test] + fn license_template_describes_cargo_and_non_cargo_projects() { + assert!(ABOUT_TEMPLATE.contains("Cargo dependencies")); + assert!(ABOUT_TEMPLATE.contains("other third-party projects")); + } +} From 5a065175d03f6a5a721f7783e1b24acc62a008a2 Mon Sep 17 00:00:00 2001 From: Dongcheng Lin <256920284+Limdongcheng@users.noreply.github.com> Date: Sat, 1 Aug 2026 01:02:51 +0800 Subject: [PATCH 2/2] docs: document Origin import support --- .../src/content/docs/guides/importing-data.md | 21 +++++++- .../content/docs/reference/file-formats.md | 53 ++++++++++++++++++- .../docs/zh-cn/guides/importing-data.md | 21 ++++++-- .../docs/zh-cn/reference/file-formats.md | 42 ++++++++++++++- 4 files changed, 130 insertions(+), 7 deletions(-) diff --git a/docs/src/content/docs/guides/importing-data.md b/docs/src/content/docs/guides/importing-data.md index eeef0e4..cf8b0ef 100644 --- a/docs/src/content/docs/guides/importing-data.md +++ b/docs/src/content/docs/guides/importing-data.md @@ -17,6 +17,7 @@ no conversion step is needed. | JCAMP-DX | `.dx` / `.jdx` / `.jcamp` | 1D frequency-domain NMR spectra | | Axon Binary Format 2 | `.abf` | int16/float32, multiple channels and sweeps, embedded DAC/epoch stimuli | | Tabular data | `.csv`, `.tsv`, `.txt`, `.xlsx` | Column types and empty cells preserved; one table per XLSX worksheet | +| Origin project (experimental) | `.opj`, `.opju` | Worksheets from the verified Origin 7.0552 and Origin 9.51 OPJ profiles; graphs are not imported, and `.opju` is detection-only. See [compatibility details](/reference/file-formats/). | | Zip archive | `.zip` | An archived dataset folder | | PlotX project | `.plotx` | Full project: data, processing, and layout | @@ -24,7 +25,7 @@ no conversion step is needed. Drag a file onto the PlotX window, or use the toolbar's open menu: *Open File…*, *Open Folder…* (for acquisition directories such as Bruker -TopSpin and Waters MassLynx RAW), *Open Project…*, or *Import Table / CSV…*. +TopSpin and Waters MassLynx RAW), *Open Project…*, or *Import Table…*. Each imported dataset appears in the Primary Side Bar and is placed on the board automatically. The file picker accepts several ABF files at once. Opening a folder recursively @@ -87,7 +88,7 @@ import** dialog. It shows each column's inferred type and unit, whether the column allows empty cells, a preview of the first rows, and any import diagnostics. Choose **Import table** to add it, or **Cancel** to leave your project and recent-file list untouched. An XLSX workbook with several sheets -adds a **Worksheet** selector so you can preview each one; a single **Import +adds a **Table** selector so you can preview each worksheet; a single **Import table** brings them all in as separate tables. PlotX keeps Boolean, whole-number, decimal, text, and empty cells distinct. A @@ -111,6 +112,22 @@ formula cell with no cached value imports as empty and is listed in the diagnostics. Exported XLSX files hold plain values, so they never depend on Excel recalculating them. +## Origin project import (experimental) + +Origin `.opj` and `.opju` files appear in the file picker for both *Open +File…* and *Import Table…*. Both routes identify the format from file +content and signatures rather than relying only on the extension. + +When a supported `.opj` yields worksheets, PlotX opens the existing **Review +table import** preview so you can inspect every candidate table. Confirm once +to import all candidates, or cancel to leave the current project and recent-file +list unchanged. While a preview is pending, selecting a second table path is +rejected with a clear message; finish or cancel the current preview first. + +Origin does not need to be installed or launched, and PlotX does not automate +or invoke it. See [File formats](/reference/file-formats/) for the exact, +evidence-limited compatibility boundary. + ## Pseudo-2D experiments DOSY, T1, and T2 experiments are detected automatically from the acquisition diff --git a/docs/src/content/docs/reference/file-formats.md b/docs/src/content/docs/reference/file-formats.md index 6fefd2c..b46c58b 100644 --- a/docs/src/content/docs/reference/file-formats.md +++ b/docs/src/content/docs/reference/file-formats.md @@ -1,6 +1,6 @@ --- title: File formats -description: What PlotX's own files contain and how safely they can be shared. +description: Native PlotX files, imported formats, and their compatibility boundaries. --- ## `.plotx` projects @@ -52,6 +52,57 @@ are covered in [the command line](/reference/cli/). A workflow is not a recipe: a recipe holds one processing pipeline, while a workflow describes a whole run and may reference a recipe as one of its steps. +## Origin project import (experimental) + +Origin project import is experimental. Successful import is limited to two +exact, content-detected OPJ producer profiles: + +- Origin 7.0552 (`CPYA 4.2673 build 552`) imports verified `f64`, `f32`, signed + `i32`, signed `i16`, fixed-width ASCII text, mixed numeric/text cells, nulls, + and nonzero row offsets. Project parameters and notes are retained as source + metadata. +- Origin 9.51 build 195 W64 (`CPYA 4.3268 build 195 W64`) imports worksheet + names, column names, numeric `f64` values, nulls, and validated empty columns. + This modern profile does not yet import text cells, project parameters, or + notes. + +Compatibility claims are limited to the committed Origin 7 regression fixture +and the exact Origin 9.51 profile checked against two real projects, companion +CSV exports, and an independent parser comparison. They do not extend to other +files merely because the extension or major Origin version is the same. + +PlotX preserves validated Origin window or group names and column names. Each +supported window is represented as one table under the generated worksheet +name `Sheet1`; this release does not claim to decode original worksheet labels. +Mixed Origin 7 columns are retained as text, and unequal column lengths are +padded with nulls. There is no verified-support claim for long names, units, +comments, column designations, dates, categorical values, or unverified code +pages. + +An `.opju` file is recognized from its CPYUA content signature, but `.opju` is +not importable in this release and PlotX creates no partial OPJU result. + +Unsupported content includes graphs, formulas, scripts, analysis +recomputation, saved analysis results as executable analyses, matrices, +embedded objects, modern OPJ text cells, non-ASCII text without a verified code +page, encrypted or protected projects, unverified OPJ versions or profiles, and +unverified OPJU containers. For the supported Origin 9.51 profile, PlotX stops +after the validated window-list boundary and warns that the remaining project +objects were not imported. + +PlotX never silently or heuristically guesses an import. Corrupt or truncated +files, files above the current 128 MiB input cap, extension/signature-family +mismatches, and malformed or otherwise unsupported files produce a clear error +before any table is committed. Inside an otherwise supported OPJ, an +unsupported worksheet column may be omitted, or an unsupported non-table object +skipped, only when each is independently framed and its outer boundaries are +trusted. PlotX shows warnings for every such omission; an imported worksheet +may therefore contain only the supported columns, not every source column. If +framing is ambiguous or untrusted, PlotX rejects the file rather than guessing +boundaries or silently shifting data. + +Origin need not be installed, launched, or called during import. + ## Data you import and export See [Importing data](/guides/importing-data/) for the supported instrument and diff --git a/docs/src/content/docs/zh-cn/guides/importing-data.md b/docs/src/content/docs/zh-cn/guides/importing-data.md index 280d540..4bddaf7 100644 --- a/docs/src/content/docs/zh-cn/guides/importing-data.md +++ b/docs/src/content/docs/zh-cn/guides/importing-data.md @@ -16,6 +16,7 @@ PlotX 直接读取厂商 LC–MS、NMR、AFM 与电生理格式，无需任何 | JCAMP-DX | `.dx` / `.jdx` / `.jcamp` | 1D 频域 NMR 谱 | | Axon Binary Format 2 | `.abf` | int16/float32、多通道、多 sweep，以及文件内 DAC/epoch 刺激 | | 表格数据 | `.csv`、`.tsv`、`.txt`、`.xlsx` | 保留列类型与空单元格；每个 XLSX 工作表导入为独立数据表 | +| Origin 项目（实验性） | `.opj`、`.opju` | 经验证的 Origin 7.0552 与 Origin 9.51 OPJ 配置中的工作表；不导入图形，`.opju` 仅作识别。见[兼容性详情](/zh-cn/reference/file-formats/)。 | | Zip 压缩包 | `.zip` | 打包的数据文件夹 | | PlotX 项目 | `.plotx` | 完整项目：数据、处理与排版 | @@ -23,7 +24,7 @@ PlotX 直接读取厂商 LC–MS、NMR、AFM 与电生理格式，无需任何 把文件拖到 PlotX 窗口上，或使用工具栏的打开菜单：*Open File…*、 *Open Folder…*（用于 Bruker TopSpin 与 Waters MassLynx RAW 等采集目录）、 -*Open Project…* 或 *Import Table / CSV…*。每个导入的数据集会出现在主侧栏中， +*Open Project…* 或 *Import Table…*。每个导入的数据集会出现在主侧栏中， 并自动放置到画板上。 文件选择器可以一次选择多个 ABF。打开文件夹时会递归导入其中所有 `.abf`、 `.spm`、`.pfc` 和已识别的 `.raw` 数据包。每个 `.raw` 目录会作为一次完整采集 @@ -70,8 +71,8 @@ PlotX 不提供 LC–MS 处理流程。导入的数据、活动功能、检测 无论从文件还是剪贴板导入表格，都会先打开 **Review table import** 对话框。它会 列出每列推断出的类型和单位、该列是否允许空单元格、前几行的预览，以及任何导入 诊断。选择 **Import table** 导入，或选择 **Cancel** 保持项目与最近文件列表不变。 -含多个工作表的 XLSX 会额外提供 **Worksheet** 选择器，可逐一预览各工作表；一次 -**Import table** 会把它们作为独立数据表全部导入。 +含多个工作表的 XLSX 会额外提供 **Table** 选择器，可逐一预览工作簿中的各工作表； +一次 **Import table** 会把它们作为独立数据表全部导入。 PlotX 会区分布尔、整数、小数、文本和空单元格。混合了不同类型、或取值含糊的列会 保留为文本而不会被丢弃。除非文件自带 PlotX 的类型信息（见下），只有毫不含糊的 @@ -89,6 +90,20 @@ PlotX 导出 CSV 或 TSV 时，会在旁边写入一个配套的 `.plotx-schema. 没有缓存值的公式单元格会以空导入，并列入诊断。导出的 XLSX 文件只包含确定值， 因此不依赖 Excel 重新计算。 +## Origin 项目导入（实验性） + +Origin 的 `.opj` 与 `.opju` 文件会出现在 *Open File…* 和 *Import Table…* +两个入口的文件选择器中。这两个入口均根据文件内容与签名识别格式， +而不是只看扩展名。 + +受支持的 `.opj` 成功生成工作表后，PlotX 会打开现有的 **Review table +import** 预览，可先检查每个候选数据表。确认一次会导入全部候选数据表； +取消则保持当前项目和最近文件列表不变。预览尚未处理完时，若再选择第二个 +表格路径，PlotX 会给出明确提示并拒绝该操作；请先完成或取消当前预览。 + +无需安装或启动 Origin，PlotX 也不会自动化或调用 Origin。严格且以证据为限的 +兼容范围见[文件格式](/zh-cn/reference/file-formats/)。 + ## 伪 2D 实验 DOSY、T1、T2 实验会根据采集参数自动识别，并获得专属的分析工具——参见 diff --git a/docs/src/content/docs/zh-cn/reference/file-formats.md b/docs/src/content/docs/zh-cn/reference/file-formats.md index 1fb3faa..0674892 100644 --- a/docs/src/content/docs/zh-cn/reference/file-formats.md +++ b/docs/src/content/docs/zh-cn/reference/file-formats.md @@ -1,6 +1,6 @@ --- title: 文件格式 -description: PlotX 自有文件的内容，以及它们的分享与兼容性。 +description: PlotX 自有文件、导入格式及其兼容性边界。 --- ## `.plotx` 项目 @@ -44,6 +44,46 @@ PlotX（或相反）时，文件会被拒绝并给出明确的"不支持的版 工作流不是配方：配方保存一条处理管线，而工作流描述一整次运行，可以把 配方作为其中一个步骤引用。 +## Origin 项目导入（实验性） + +Origin 项目导入仍属实验性。成功导入仅限两种通过文件内容精确识别的 OPJ +生成配置： + +- Origin 7.0552（`CPYA 4.2673 build 552`）可导入经过验证的 `f64`、`f32`、 + 有符号 `i32`、有符号 `i16`、定宽 ASCII 文本、数值与文本混合的单元格、 + 空值和非零行偏移。项目参数与备注会作为来源元数据保留。 +- Origin 9.51 build 195 W64（`CPYA 4.3268 build 195 W64`）可导入工作表名、 + 列名、`f64` 数值、空值和经过验证的空列。该现代配置目前不导入文本单元格、 + 项目参数或备注。 + +兼容性声明仅限仓库中已提交的 Origin 7 回归样本，以及通过两个真实项目、配套 +CSV 导出和独立解析器对照检查的精确 Origin 9.51 配置。不能因为扩展名相同或 +Origin 主版本相同，就推定其他文件也受支持。 + +PlotX 会保留经过验证的 Origin 窗口或分组名与列名。每个受支持的窗口以单个 +数据表表示，并使用合成的工作表名 `Sheet1`；本版本不声称能够解析原始工作表 +标签。Origin 7 的混合列会保留为文本，长度不等的列会用空值补齐。目前不声称 +已验证长名称、单位、注释、列标识、日期、分类值或未经验证的代码页。 + +PlotX 会根据 CPYUA 内容签名识别 `.opju` 文件，但本版本不能导入 `.opju`， +也不会产生任何部分 OPJU 结果。 + +不支持的内容包括图形、公式、脚本、分析重新计算、把已保存的分析结果恢复为 +可执行分析、矩阵、嵌入对象、现代 OPJ 文本单元格、没有已验证代码页的非 ASCII +文本、加密或受保护项目、未经验证的 OPJ 版本或配置，以及未经验证的 OPJU +容器。对于受支持的 Origin 9.51 配置，PlotX 会在经过验证的窗口列表边界停止 +解析，并警告剩余项目对象没有被导入。 + +PlotX 绝不会静默导入，也不会凭推测猜测导入内容。损坏或截断的文件、超过当前 +128 MiB 输入上限的文件、扩展名与签名家族不匹配的文件，以及结构异常或其他 +不受支持的文件，都会在提交任何数据表之前给出明确错误。在其他方面受支持的 +OPJ 中，只有在相应内容具有独立框架且其外层边界可信时，才可以省略不受支持的 +工作表列，或跳过不受支持的非表格对象。PlotX 会为每次此类省略或跳过显示警告； +因此，导入的工作表可能只含受支持的列，而非源文件中的全部列。若框架含糊或 +不可信，PlotX 会拒绝文件，而不会猜测边界或静默造成数据错位。 + +导入过程中无需安装、启动或调用 Origin。 + ## 你导入和导出的数据 受支持的仪器与表格格式见[导入数据](/zh-cn/guides/importing-data/)；图形