diff --git a/.gitattributes b/.gitattributes index 9fc52fc..bd4afdb 100644 --- a/.gitattributes +++ b/.gitattributes @@ -2,3 +2,5 @@ *.ico binary *.png binary +*.opj binary +*.opju binary diff --git a/Cargo.lock b/Cargo.lock index 7eedb61..a297eaf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4047,6 +4047,7 @@ dependencies = [ "egui_extras", "fontdb", "image", + "libc", "log", "muda", "num-complex", diff --git a/Cargo.toml b/Cargo.toml index d5e0052..09f4a7a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -75,6 +75,7 @@ muda = { version = "0.19.3", default-features = false } rfd = "0.17" zip = { version = "8.6", default-features = false, features = ["deflate"] } image = { version = "0.25", default-features = false, features = ["jpeg", "png", "tiff"] } +libc = "0.2" pdf-writer = "0.12" resvg = "0.47" svg2pdf = "0.13" diff --git a/crates/app/Cargo.toml b/crates/app/Cargo.toml index 30b26e4..7d37318 100644 --- a/crates/app/Cargo.toml +++ b/crates/app/Cargo.toml @@ -38,6 +38,9 @@ uuid.workspace = true raw-window-handle.workspace = true windows-sys.workspace = true +[target.'cfg(unix)'.dependencies] +libc.workspace = true + [target.'cfg(target_os = "macos")'.dependencies] fontdb.workspace = true muda.workspace = true diff --git a/crates/app/src/ui/canvas/mod.rs b/crates/app/src/ui/canvas/mod.rs index fad8fb6..954168d 100644 --- a/crates/app/src/ui/canvas/mod.rs +++ b/crates/app/src/ui/canvas/mod.rs @@ -429,7 +429,7 @@ fn welcome_start(app: &mut PlotxApp, ui: &mut Ui) { if welcome_action(ui, icon::FOLDER_OPEN, "Open project…") { crate::ui::file_dialogs::open_project(app); } - if welcome_action(ui, icon::TABLE, "Import table / CSV…") { + if welcome_action(ui, icon::TABLE, "Import table…") { crate::ui::file_dialogs::import_delimited_table(app); } if welcome_action(ui, icon::FILE_PLUS, "New empty data table") { diff --git a/crates/app/src/ui/commands.rs b/crates/app/src/ui/commands.rs index bf86b9a..3f1644a 100644 --- a/crates/app/src/ui/commands.rs +++ b/crates/app/src/ui/commands.rs @@ -400,6 +400,10 @@ pub fn describe(app: &PlotxApp, id: CommandId) -> CommandDescriptor { !app.session.recent_files.is_empty(), "Open a file or project to build the recent list.", ), + CommandId::ImportTable => requires( + app.session.ui.table_import_preview.is_none(), + "Finish or cancel the current table import preview before importing another table.", + ), CommandId::ExportData => requires( dataset().is_some_and(|dataset| { !plotx_core::data_export::DataExportAvailability::for_dataset(dataset).is_empty() diff --git a/crates/app/src/ui/commands/identity.rs b/crates/app/src/ui/commands/identity.rs index 6f5ef6f..5637f9d 100644 --- a/crates/app/src/ui/commands/identity.rs +++ b/crates/app/src/ui/commands/identity.rs @@ -52,7 +52,7 @@ pub(super) fn command_identity( ), CommandId::ClearRecentFiles => plain("Clear Recent Files", None), CommandId::HelpManual => plain("User Manual", Some(icon::BOOK_OPEN)), - CommandId::ImportTable => plain("Import Table / CSV…", Some(icon::TABLE)), + CommandId::ImportTable => plain("Import Table…", Some(icon::TABLE)), CommandId::PasteTable => plain("Paste Table from Clipboard", Some(icon::CLIPBOARD_TEXT)), CommandId::SaveProject => plain("Save Project", Some(icon::FLOPPY_DISK)), CommandId::NewTable => plain("New Empty Data Table", Some(icon::TABLE)), diff --git a/crates/app/src/ui/commands_tests.rs b/crates/app/src/ui/commands_tests.rs index 550f8a5..5521a5f 100644 --- a/crates/app/src/ui/commands_tests.rs +++ b/crates/app/src/ui/commands_tests.rs @@ -253,6 +253,34 @@ fn spacing_commands_are_registered_checked_and_execute() { ); } +#[test] +fn origin_import_reuses_import_table_command_identity() { + let app = app(); + assert_eq!(CommandId::ImportTable.stable_id(), "file.import_table"); + assert_eq!( + describe(&app, CommandId::ImportTable).label, + "Import Table…" + ); +} + +#[test] +fn import_table_is_disabled_while_a_table_preview_is_pending() { + let mut app = app(); + crate::ui::file_dialogs::import_delimited_text( + &mut app, + "x,y\n0,1\n", + crate::ui::file_dialogs::DelimitedTableSource::Clipboard, + ); + assert!(app.session.ui.table_import_preview.is_some()); + + let command = describe(&app, CommandId::ImportTable); + assert!(!command.enabled); + assert_eq!( + command.disabled_reason, + Some("Finish or cancel the current table import preview before importing another table.") + ); +} + #[test] fn automation_is_a_global_menu_and_palette_command() { let app = app(); diff --git a/crates/app/src/ui/file_dialogs.rs b/crates/app/src/ui/file_dialogs.rs index d3102d9..59e1b1c 100644 --- a/crates/app/src/ui/file_dialogs.rs +++ b/crates/app/src/ui/file_dialogs.rs @@ -8,6 +8,7 @@ use plotx_core::state::ProcessingSchemeDialogState; mod delimited; mod discovery; +mod origin; mod path; mod preview; mod recent; @@ -21,26 +22,23 @@ use xlsx::import_xlsx_table_path; pub(crate) fn import_delimited_table(app: &mut PlotxApp) { let Some(path) = rfd::FileDialog::new() .add_filter( - "Table (*.csv, *.tsv, *.txt, *.xlsx)", - &["csv", "tsv", "txt", "xlsx"], + "Table (*.csv, *.tsv, *.txt, *.xlsx, *.opj)", + origin::IMPORT_TABLE_FILTER_EXTENSIONS, + ) + .add_filter( + origin::ORIGIN_PROJECT_FILTER_LABEL, + origin::ORIGIN_PROJECT_FILTER_EXTENSIONS, ) .add_filter("Excel workbook (*.xlsx)", &["xlsx"]) .add_filter("CSV (*.csv)", &["csv"]) .add_filter("TSV (*.tsv)", &["tsv"]) .add_filter("All files", &["*"]) - .set_title("Import a comma, tab, or semicolon delimited table") + .set_title("Import a table") .pick_file() else { return; }; - if path - .extension() - .is_some_and(|extension| extension.eq_ignore_ascii_case("xlsx")) - { - import_xlsx_table_path(app, &path); - } else { - import_delimited_table_path(app, &path); - } + open_recent_path(app, &path); } fn import_delimited_table_path(app: &mut PlotxApp, path: &std::path::Path) { @@ -288,9 +286,34 @@ pub(crate) fn import_delimited_text_with_schema( } pub(crate) fn commit_table_import_preview(app: &mut PlotxApp) -> bool { + commit_table_import_preview_with_recent(app, PlotxApp::note_recent_file) +} + +pub(crate) fn commit_table_import_preview_with_recent( + app: &mut PlotxApp, + mut note_recent_file: F, +) -> bool +where + F: FnMut(&mut PlotxApp, &std::path::Path), +{ let Some(preview) = app.session.ui.table_import_preview.take() else { return false; }; + if preview.candidates.is_empty() { + app.session.record_operation(OperationReport::<()>::failure( + preview.report.id, + OperationKind::TableImport, + "Table import failed because there are no supported tables to import.", + Diagnostic::new( + Severity::Error, + DiagnosticCode::TableImportFailed, + "The import preview contains no supported table candidates.", + ) + .with_source("app.table_import") + .with_context("stage", "preview_commit"), + )); + return false; + } for candidate in preview.candidates { app.import_table_dataset_typed( candidate.name, @@ -305,7 +328,7 @@ pub(crate) fn commit_table_import_preview(app: &mut PlotxApp) -> bool { // through the status line. Recording the import first keeps that // diagnostic: `record_operation` replaces the status unconditionally. if let Some(path) = preview.recent_path { - app.note_recent_file(&path); + note_recent_file(app, &path); } true } @@ -321,8 +344,12 @@ pub(crate) fn load_and_note(app: &mut PlotxApp, path: &std::path::Path) { pub(crate) fn open_file(app: &mut PlotxApp) { if let Some(paths) = rfd::FileDialog::new() .add_filter( - "All supported data (*.spm, *.pfc, *.abf, *.jdf, fid, ser, *.zip)", - &["spm", "pfc", "abf", "jdf", "fid", "ser", "zip"], + "All supported data (*.spm, *.pfc, *.abf, *.jdf, fid, ser, *.zip, *.opj)", + origin::OPEN_FILE_FILTER_EXTENSIONS, + ) + .add_filter( + origin::ORIGIN_PROJECT_FILTER_LABEL, + origin::ORIGIN_PROJECT_FILTER_EXTENSIONS, ) .add_filter("Bruker NanoScope AFM (*.spm, *.pfc)", &["spm", "pfc"]) .add_filter("Axon Binary Format 2 (*.abf)", &["abf"]) @@ -334,7 +361,7 @@ pub(crate) fn open_file(app: &mut PlotxApp) { .pick_files() { for path in paths { - load_and_note(app, &path); + open_recent_path(app, &path); } } } diff --git a/crates/app/src/ui/file_dialogs/origin.rs b/crates/app/src/ui/file_dialogs/origin.rs new file mode 100644 index 0000000..8592d62 --- /dev/null +++ b/crates/app/src/ui/file_dialogs/origin.rs @@ -0,0 +1,527 @@ +use std::io::{Read, Seek, SeekFrom}; +use std::path::Path; +use std::sync::Arc; + +use plotx_core::operation::{ + Diagnostic, DiagnosticCode, OperationId, OperationKind, OperationReport, Severity, +}; +use plotx_core::origin::{ + ImportedOriginWorksheet, ORIGIN_IMPORT_OPERATION, OriginImportError, import_origin_project, +}; +use plotx_core::state::{ + PlotxApp, TableImportCandidate, TableImportPreviewState, TableImportSource, TypedTableState, +}; +use plotx_io::origin::{ + OriginDiagnostic, OriginDiagnosticSeverity, OriginError, OriginLimits, OriginProject, + probe_origin, read_origin, +}; + +pub(super) const IMPORT_TABLE_FILTER_EXTENSIONS: &[&str] = &["csv", "tsv", "txt", "xlsx", "opj"]; +pub(super) const ORIGIN_PROJECT_FILTER_LABEL: &str = + "Origin projects (experimental: OPJ import; OPJU recognition only)"; +pub(super) const ORIGIN_PROJECT_FILTER_EXTENSIONS: &[&str] = &["opj", "opju"]; +pub(super) const OPEN_FILE_FILTER_EXTENSIONS: &[&str] = + &["spm", "pfc", "abf", "jdf", "fid", "ser", "zip", "opj"]; + +const ORIGIN_MEDIA_TYPE: &str = "application/x-origin-project"; +const ORIGIN_READ_CHUNK_BYTES: usize = 16 * 1024; + +#[derive(Debug)] +pub(super) struct OpenOriginSource { + file: std::fs::File, + metadata_len: u64, +} + +impl OpenOriginSource { + pub(super) fn new(file: std::fs::File, metadata_len: u64) -> Self { + Self { file, metadata_len } + } + + fn into_parts(self) -> (std::fs::File, u64) { + (self.file, self.metadata_len) + } +} + +struct OriginFailure { + stage: &'static str, + message: String, + detail: String, +} + +#[derive(Clone, Copy)] +struct SourceByteLimit { + resource: &'static str, + limit: usize, + maximum: u64, + sentinel: usize, +} + +impl OriginFailure { + fn io(stage: &'static str, message: impl Into, error: impl ToString) -> Self { + Self { + stage, + message: message.into(), + detail: error.to_string(), + } + } + + fn parser(stage: &'static str, error: OriginError) -> Self { + let message = match &error { + OriginError::UnrecognizedFormat => { + "The selected file does not have a recognized Origin project signature. No data was imported." + .to_owned() + } + OriginError::UnsupportedOpjuVariant { message } => message.clone(), + OriginError::NoSupportedWorksheet => { + "The Origin project contains no supported table data. No data was imported." + .to_owned() + } + OriginError::LimitExceeded { .. } + | OriginError::InvalidLimit { .. } + | OriginError::ArithmeticOverflow { .. } + | OriginError::AllocationFailed { .. } => { + format!("The Origin project could not be imported safely: {error}. No data was imported.") + } + _ => format!("The Origin project could not be read: {error}. No data was imported."), + }; + Self { + stage, + message, + detail: error.to_string(), + } + } + + fn core(error: OriginImportError) -> Self { + let message = match &error { + OriginImportError::NoSupportedWorksheet => { + "The Origin project contains no supported table data. No data was imported." + .to_owned() + } + _ => format!( + "The Origin project could not be converted into PlotX tables: {error}. No data was imported." + ), + }; + Self { + stage: "convert", + message, + detail: error.to_string(), + } + } +} + +pub(super) fn import_origin_project_source( + app: &mut PlotxApp, + path: &Path, + source: OpenOriginSource, +) { + let limits = OriginLimits::default(); + let result = read_origin_source(source, limits).and_then(|source_bytes| { + probe_origin(&source_bytes).map_err(|error| OriginFailure::parser("probe", error))?; + let project = read_origin(&source_bytes, limits) + .map_err(|error| OriginFailure::parser("parse", error))?; + Ok((source_bytes, project)) + }); + match result { + Ok((source_bytes, project)) => { + import_origin_project_model(app, path, source_bytes, project, limits); + } + Err(error) => { + let operation_id = app.session.begin_operation(); + install_origin_result(app, operation_id, path, Err(error)); + } + } +} + +pub(super) fn import_origin_project_model( + app: &mut PlotxApp, + path: &Path, + source_bytes: Arc<[u8]>, + project: OriginProject, + limits: OriginLimits, +) { + let operation_id = app.session.begin_operation(); + let result = preview_from_project(operation_id, path, source_bytes, project, limits); + install_origin_result(app, operation_id, path, result); +} + +fn read_origin_source( + source: OpenOriginSource, + limits: OriginLimits, +) -> Result, OriginFailure> { + let (mut file, metadata_len) = source.into_parts(); + read_origin_handle(&mut file, Some(metadata_len), limits) +} + +fn read_origin_handle( + mut reader: R, + metadata_len: Option, + limits: OriginLimits, +) -> Result, OriginFailure> { + let source_limit = checked_source_byte_limit(limits) + .map_err(|error| OriginFailure::io("limits", limit_message(&error), error))?; + if let Some(length) = metadata_len + && length > source_limit.maximum + { + let error = source_too_large(length, source_limit); + return Err(OriginFailure::io("metadata", limit_message(&error), error)); + } + reader.seek(SeekFrom::Start(0)).map_err(|error| { + OriginFailure::io( + "rewind", + "The selected Origin project could not be rewound for import. No data was imported.", + error, + ) + })?; + read_bounded_origin(reader, metadata_len, limits).map_err(|error| OriginFailure { + stage: "read", + message: limit_message(&error), + detail: error, + }) +} + +pub(super) fn read_bounded_origin( + mut reader: R, + metadata_len: Option, + limits: OriginLimits, +) -> Result, String> { + let source_limit = checked_source_byte_limit(limits)?; + if let Some(length) = metadata_len + && length > source_limit.maximum + { + return Err(source_too_large(length, source_limit)); + } + + let mut bytes = Vec::new(); + let mut chunk = [0_u8; ORIGIN_READ_CHUNK_BYTES]; + loop { + let remaining = source_limit + .sentinel + .checked_sub(bytes.len()) + .ok_or_else(|| { + OriginError::ArithmeticOverflow { + resource: "Origin source sentinel bytes", + } + .to_string() + })?; + let request = remaining.min(chunk.len()); + let read = match reader.read(&mut chunk[..request]) { + Ok(0) => break, + Ok(read) if read <= request => read, + Ok(read) => { + return Err(format!( + "the bounded Origin project read failed: the reader returned {read} bytes for a {request}-byte buffer" + )); + } + Err(error) if error.kind() == std::io::ErrorKind::Interrupted => continue, + Err(error) => { + return Err(format!("the bounded Origin project read failed: {error}")); + } + }; + let next_len = bytes.len().checked_add(read).ok_or_else(|| { + OriginError::ArithmeticOverflow { + resource: "Origin source bytes", + } + .to_string() + })?; + if next_len > source_limit.limit { + return Err(OriginError::LimitExceeded { + resource: source_limit.resource, + limit: source_limit.limit, + actual: next_len, + } + .to_string()); + } + reserve_source_capacity(&mut bytes, next_len, source_limit, limits)?; + let read_bytes = chunk.get(..read).ok_or_else(|| { + "the bounded Origin project read failed: the reader exceeded its buffer".to_owned() + })?; + bytes.extend_from_slice(read_bytes); + } + + let conversion_peak = bytes.capacity().checked_add(bytes.len()).ok_or_else(|| { + OriginError::ArithmeticOverflow { + resource: "Origin source Arc conversion", + } + .to_string() + })?; + if conversion_peak > limits.max_total_owned_bytes { + return Err(OriginError::LimitExceeded { + resource: "total owned bytes", + limit: limits.max_total_owned_bytes, + actual: conversion_peak, + } + .to_string()); + } + Ok(Arc::<[u8]>::from(bytes)) +} + +fn reserve_source_capacity( + bytes: &mut Vec, + required_len: usize, + source_limit: SourceByteLimit, + limits: OriginLimits, +) -> Result<(), String> { + let old_capacity = bytes.capacity(); + if required_len <= old_capacity { + return Ok(()); + } + let doubled = old_capacity.checked_mul(2).unwrap_or(source_limit.sentinel); + let target_capacity = required_len.max(doubled).min(source_limit.limit); + let additional = target_capacity.checked_sub(bytes.len()).ok_or_else(|| { + OriginError::ArithmeticOverflow { + resource: "Origin source allocation", + } + .to_string() + })?; + bytes.try_reserve_exact(additional).map_err(|_| { + OriginError::AllocationFailed { + resource: "Origin source bytes", + requested: target_capacity, + } + .to_string() + })?; + let actual_capacity = bytes.capacity(); + if actual_capacity > limits.max_total_owned_bytes { + return Err(OriginError::LimitExceeded { + resource: "total owned bytes", + limit: limits.max_total_owned_bytes, + actual: actual_capacity, + } + .to_string()); + } + Ok(()) +} + +fn checked_source_byte_limit(limits: OriginLimits) -> Result { + limits.validate().map_err(|error| error.to_string())?; + let (resource, limit_name, limit) = if limits.max_input_bytes <= limits.max_total_owned_bytes { + ("input bytes", "max_input_bytes", limits.max_input_bytes) + } else { + ( + "total owned bytes", + "max_total_owned_bytes", + limits.max_total_owned_bytes, + ) + }; + let sentinel = limit.checked_add(1).ok_or_else(|| { + invalid_limit( + limit_name, + limit, + "the limit must leave room for an oversize sentinel byte", + ) + .to_string() + })?; + let maximum = u64::try_from(limit).map_err(|_| { + invalid_limit( + limit_name, + limit, + "the source-byte limit cannot be represented by the bounded reader", + ) + .to_string() + })?; + Ok(SourceByteLimit { + resource, + limit, + maximum, + sentinel, + }) +} + +fn invalid_limit(name: &'static str, value: usize, reason: &'static str) -> OriginError { + OriginError::InvalidLimit { + name, + value, + reason, + } +} + +fn source_too_large(actual: u64, source_limit: SourceByteLimit) -> String { + let actual = usize::try_from(actual).unwrap_or(usize::MAX); + OriginError::LimitExceeded { + resource: source_limit.resource, + limit: source_limit.limit, + actual, + } + .to_string() +} + +fn limit_message(detail: impl std::fmt::Display) -> String { + format!("The Origin project could not be imported safely: {detail}. No data was imported.") +} + +fn preview_from_project( + operation_id: OperationId, + path: &Path, + source_bytes: Arc<[u8]>, + project: OriginProject, + limits: OriginLimits, +) -> Result { + let store = Arc::new(plotx_core::data::MemoryBlockStore::default()); + let codecs = plotx_core::data::CodecRegistry::with_arrow_ipc(); + let imported = import_origin_project(project, store.as_ref(), &codecs, limits) + .map_err(OriginFailure::core)?; + preview_from_imported(operation_id, path, source_bytes, store, imported).map_err(|error| { + OriginFailure::io( + "revision", + "The Origin project could not be prepared for preview. No data was imported.", + error, + ) + }) +} + +pub(super) fn preview_from_imported( + operation_id: OperationId, + path: &Path, + source_bytes: Arc<[u8]>, + store: Arc, + imported: Vec, +) -> Result { + ensure_candidate_count(imported.len())?; + let candidate_count = imported.len(); + let project_diagnostics = imported + .first() + .map(|worksheet| worksheet.diagnostics.clone()) + .unwrap_or_default(); + let file_name = path + .file_name() + .map(|name| name.to_string_lossy().into_owned()) + .unwrap_or_else(|| path.display().to_string()); + let mut candidates = Vec::with_capacity(candidate_count); + let mut candidate_diagnostics = Vec::with_capacity(candidate_count); + + for worksheet in imported { + let row_count = worksheet.snapshot.row_count; + let worksheet_name = worksheet.name; + let typed_state = TypedTableState::imported_with_operation( + worksheet.snapshot, + Arc::clone(&store), + ORIGIN_IMPORT_OPERATION, + ) + .map_err(|error| error.to_string())?; + let mut source = TableImportSource::new(Arc::clone(&source_bytes), ORIGIN_MEDIA_TYPE); + source.name = Some(file_name.clone()); + source.metadata = worksheet.source_metadata; + candidates.push(TableImportCandidate { + name: worksheet_name.clone(), + retained_sources: vec![source], + typed_state, + x_binding: None, + series_bindings: Vec::new(), + }); + candidate_diagnostics.push( + Diagnostic::new( + Severity::Info, + DiagnosticCode::TableImportSucceeded, + format!("Prepared Origin table '{worksheet_name}' with {row_count} row(s)."), + ) + .with_source("core.origin") + .with_context("path", path.display().to_string()) + .with_context("table", worksheet_name), + ); + } + + let mut diagnostics = project_diagnostics + .iter() + .map(origin_diagnostic) + .collect::>(); + let warning_count = diagnostics + .iter() + .filter(|diagnostic| diagnostic.severity == Severity::Warning) + .count(); + diagnostics.extend(candidate_diagnostics); + let summary = if warning_count == 0 { + format!("Imported {candidate_count} Origin table(s).") + } else { + format!("Imported {candidate_count} Origin table(s) with {warning_count} warning(s).") + }; + let mut report = if warning_count == 0 { + OperationReport::success(operation_id, OperationKind::TableImport, summary, ()) + } else { + OperationReport::warning(operation_id, OperationKind::TableImport, summary, ()) + }; + for diagnostic in diagnostics { + report = report.with_diagnostic(diagnostic); + } + Ok(TableImportPreviewState { + candidates, + selected: 0, + report, + recent_path: Some(path.to_owned()), + }) +} + +pub(super) fn ensure_candidate_count(count: usize) -> Result<(), String> { + if count == 0 { + Err("the Origin project contains no supported table candidates".to_owned()) + } else { + Ok(()) + } +} + +fn origin_diagnostic(diagnostic: &OriginDiagnostic) -> Diagnostic { + let severity = match diagnostic.severity { + OriginDiagnosticSeverity::Info => Severity::Info, + OriginDiagnosticSeverity::Warning => Severity::Warning, + }; + let mut result = Diagnostic::new( + severity, + if severity == Severity::Warning { + DiagnosticCode::TableImportWarning + } else { + DiagnosticCode::TableImportSucceeded + }, + diagnostic.message.clone(), + ) + .with_source("io.origin") + .with_context("origin_code", format!("{:?}", diagnostic.code)); + if let Some(location) = &diagnostic.location { + if let Some(workbook) = &location.workbook { + result = result.with_context("workbook", workbook.clone()); + } + if let Some(worksheet) = &location.worksheet { + result = result.with_context("table", worksheet.clone()); + } + if let Some(column) = &location.column { + result = result.with_context("column", column.clone()); + } + if let Some(offset) = location.byte_offset { + result = result.with_context("byte_offset", offset.to_string()); + } + } + result +} + +fn install_origin_result( + app: &mut PlotxApp, + operation_id: OperationId, + path: &Path, + result: Result, +) { + match result { + Ok(preview) => app.session.ui.table_import_preview = Some(preview), + Err(error) => { + app.session.record_operation(OperationReport::<()>::failure( + operation_id, + OperationKind::TableImport, + error.message.clone(), + Diagnostic::new( + Severity::Error, + DiagnosticCode::TableImportFailed, + error.message, + ) + .with_source("app.table_import.origin") + .with_context("path", path.display().to_string()) + .with_context("stage", error.stage) + .with_context("error", error.detail), + )); + } + } +} + +#[cfg(test)] +#[path = "origin_allocation_tests.rs"] +mod origin_allocation_tests; + +#[cfg(test)] +#[path = "origin_tests.rs"] +mod origin_tests; diff --git a/crates/app/src/ui/file_dialogs/origin_allocation_tests.rs b/crates/app/src/ui/file_dialogs/origin_allocation_tests.rs new file mode 100644 index 0000000..814613c --- /dev/null +++ b/crates/app/src/ui/file_dialogs/origin_allocation_tests.rs @@ -0,0 +1,114 @@ +use super::*; +use std::cell::Cell; +use std::io::{self, Read}; +use std::rc::Rc; + +const NON_POWER_OF_TWO_BYTES: usize = 5_003; + +struct FiniteCountingReader { + remaining: usize, + consumed: Rc>, +} + +impl FiniteCountingReader { + fn new(remaining: usize, consumed: Rc>) -> Self { + Self { + remaining, + consumed, + } + } +} + +impl Read for FiniteCountingReader { + fn read(&mut self, buffer: &mut [u8]) -> io::Result { + let read = self.remaining.min(buffer.len()); + buffer[..read].fill(0x5a); + self.remaining -= read; + self.consumed.set(self.consumed.get() + read); + Ok(read) + } +} + +struct InfiniteCountingReader { + consumed: Rc>, +} + +impl Read for InfiniteCountingReader { + fn read(&mut self, buffer: &mut [u8]) -> io::Result { + buffer.fill(0x5a); + self.consumed.set(self.consumed.get() + buffer.len()); + Ok(buffer.len()) + } +} + +fn source_limits(max_input_bytes: usize, max_total_owned_bytes: usize) -> OriginLimits { + OriginLimits { + max_input_bytes, + max_total_owned_bytes, + ..OriginLimits::default() + } +} + +#[test] +fn unknown_non_power_of_two_source_rejects_conversion_peak_over_total_limit() { + let total_limit = NON_POWER_OF_TWO_BYTES * 2 - 1; + let consumed = Rc::new(Cell::new(0)); + let reader = FiniteCountingReader::new(NON_POWER_OF_TWO_BYTES, Rc::clone(&consumed)); + + let result = read_bounded_origin( + reader, + None, + source_limits(NON_POWER_OF_TWO_BYTES, total_limit), + ); + let error = match result { + Ok(bytes) => panic!( + "a {}-byte Arc bypassed the {total_limit}-byte conversion-peak budget", + bytes.len() + ), + Err(error) => error, + }; + + assert_eq!(consumed.get(), NON_POWER_OF_TWO_BYTES); + assert!(error.contains("total owned bytes"), "{error}"); + assert!(error.contains(&total_limit.to_string()), "{error}"); +} + +#[test] +fn unknown_non_power_of_two_source_succeeds_at_exact_conversion_peak() { + let total_limit = NON_POWER_OF_TWO_BYTES * 2; + let consumed = Rc::new(Cell::new(0)); + let reader = FiniteCountingReader::new(NON_POWER_OF_TWO_BYTES, Rc::clone(&consumed)); + + let bytes = read_bounded_origin( + reader, + None, + source_limits(NON_POWER_OF_TWO_BYTES, total_limit), + ) + .expect("an exact source allocation plus Arc copy must fit"); + + assert_eq!(bytes.len(), NON_POWER_OF_TWO_BYTES); + assert_eq!(consumed.get(), NON_POWER_OF_TWO_BYTES); +} + +#[test] +fn unknown_source_consumes_only_the_one_byte_oversize_sentinel() { + let input_limit = NON_POWER_OF_TWO_BYTES; + let consumed = Rc::new(Cell::new(0)); + let reader = InfiniteCountingReader { + consumed: Rc::clone(&consumed), + }; + + let error = read_bounded_origin(reader, None, source_limits(input_limit, input_limit * 3)) + .expect_err("an unknown-length source must stop at one byte over the input limit"); + + assert_eq!(consumed.get(), input_limit + 1); + assert_eq!( + error, + OriginError::LimitExceeded { + resource: "input bytes", + limit: input_limit, + actual: input_limit + 1, + } + .to_string() + ); +} diff --git a/crates/app/src/ui/file_dialogs/origin_tests.rs b/crates/app/src/ui/file_dialogs/origin_tests.rs new file mode 100644 index 0000000..39c9208 --- /dev/null +++ b/crates/app/src/ui/file_dialogs/origin_tests.rs @@ -0,0 +1,781 @@ +use super::*; +use crate::ui::file_dialogs::recent::{ + OPEN_HEADER_BYTES, OpenPathEntryType, classify_open_path, classify_open_path_with_header, +}; +#[cfg(unix)] +use crate::ui::file_dialogs::recent::{ + classify_open_handle, dispatch_classified_path, open_file_for_classification, +}; +use crate::ui::file_dialogs::{RecentOpenKind, open_recent_path}; +use plotx_core::operation::{OperationId, OperationOutcome, Severity}; +use plotx_core::origin::{ImportedOriginWorksheet, ORIGIN_IMPORT_OPERATION}; +use plotx_core::state::PlotxApp; +use plotx_io::origin::OriginLimits; +use std::cell::Cell; +use std::io::{self, Read, Seek, SeekFrom}; +use std::path::{Path, PathBuf}; +use std::rc::Rc; +use std::sync::Arc; + +const OPENOPJ_FIXTURE: &[u8] = + include_bytes!("../../../../io/tests/fixtures/origin/test-origin-7.0552.opj"); + +struct PanicOnRead; + +impl Read for PanicOnRead { + fn read(&mut self, _buffer: &mut [u8]) -> io::Result { + panic!("bounded reader must reject before reading") + } +} + +struct SeekFailure; + +impl Read for SeekFailure { + fn read(&mut self, _buffer: &mut [u8]) -> io::Result { + panic!("a failed rewind must stop before reading") + } +} + +impl Seek for SeekFailure { + fn seek(&mut self, _position: SeekFrom) -> io::Result { + Err(io::Error::other("injected rewind failure")) + } +} + +struct ReadFailure; + +impl Read for ReadFailure { + fn read(&mut self, _buffer: &mut [u8]) -> io::Result { + Err(io::Error::other("injected full-read failure")) + } +} + +impl Seek for ReadFailure { + fn seek(&mut self, _position: SeekFrom) -> io::Result { + Ok(0) + } +} + +struct CountingRepeat { + bytes_read: Rc>, +} + +impl Read for CountingRepeat { + fn read(&mut self, buffer: &mut [u8]) -> io::Result { + buffer.fill(0); + self.bytes_read + .set(self.bytes_read.get().saturating_add(buffer.len())); + Ok(buffer.len()) + } +} + +fn temp_origin_file(extension: &str, bytes: &[u8]) -> PathBuf { + let path = std::env::temp_dir().join(format!( + "plotx-origin-app-{}.{}", + uuid::Uuid::new_v4(), + extension + )); + std::fs::write(&path, bytes).unwrap(); + path +} + +fn duplicated_fixture_import() -> ( + Arc, + Vec, +) { + let limits = OriginLimits::default(); + let project = plotx_io::origin::read_origin(OPENOPJ_FIXTURE, limits).unwrap(); + let store = Arc::new(plotx_core::data::MemoryBlockStore::default()); + let codecs = plotx_core::data::CodecRegistry::with_arrow_ipc(); + let imported = + plotx_core::origin::import_origin_project(project, store.as_ref(), &codecs, limits) + .unwrap(); + let first = imported.into_iter().next().unwrap(); + let second = ImportedOriginWorksheet { + name: format!("{} copy", first.name), + snapshot: first.snapshot.clone(), + source_metadata: first.source_metadata.clone(), + diagnostics: first.diagnostics.clone(), + resource_usage: first.resource_usage.clone(), + }; + (store, vec![first, second]) +} + +#[test] +fn origin_import_filter_retains_tables_and_adds_experimental_projects() { + assert_eq!( + IMPORT_TABLE_FILTER_EXTENSIONS, + &["csv", "tsv", "txt", "xlsx", "opj"] + ); + assert_eq!( + ORIGIN_PROJECT_FILTER_LABEL, + "Origin projects (experimental: OPJ import; OPJU recognition only)" + ); + assert_eq!(ORIGIN_PROJECT_FILTER_EXTENSIONS, &["opj", "opju"]); +} + +#[test] +fn origin_supported_file_filter_excludes_recognition_only_opju() { + assert!(OPEN_FILE_FILTER_EXTENSIONS.contains(&"opj")); + assert!(!OPEN_FILE_FILTER_EXTENSIONS.contains(&"opju")); +} + +#[test] +fn origin_routing_uses_signature_before_extension() { + let root = std::env::temp_dir().join(format!("plotx-origin-route-{}", uuid::Uuid::new_v4())); + std::fs::create_dir(&root).unwrap(); + let disguised = root.join("project.dat"); + std::fs::write(&disguised, OPENOPJ_FIXTURE).unwrap(); + + let kind = classify_open_path(&disguised).unwrap(); + + std::fs::remove_dir_all(root).unwrap(); + assert_eq!(kind.kind(), RecentOpenKind::OriginProject); +} + +#[test] +fn origin_extension_routes_signature_mismatch_to_origin_adapter() { + let path = PathBuf::from("not-origin.opj"); + let kind = classify_open_path_with_header(&path, OpenPathEntryType::RegularFile, || { + Ok(([0_u8; OPEN_HEADER_BYTES], 0)) + }) + .unwrap(); + assert_eq!(format!("{kind:?}"), "OriginProject"); +} + +#[test] +fn origin_pending_preview_rejects_a_second_table_path_without_replacement() { + let first = temp_origin_file("opj", OPENOPJ_FIXTURE); + let second = temp_origin_file("csv", b"time,value\n0,1\n"); + let mut app = PlotxApp::new_with_settings(plotx_core::settings::Settings::default()); + + open_recent_path(&mut app, &first); + let first_preview_path = app + .session + .ui + .table_import_preview + .as_ref() + .expect("the first table path should create a preview") + .recent_path + .clone(); + open_recent_path(&mut app, &second); + + std::fs::remove_file(first).unwrap(); + std::fs::remove_file(second).unwrap(); + let preview = app + .session + .ui + .table_import_preview + .as_ref() + .expect("the first preview must remain pending"); + assert_eq!(preview.recent_path, first_preview_path); + assert!(app.doc.datasets.is_empty()); + assert!(app.session.recent_files.is_empty()); + let report = app + .session + .operation_history + .operations() + .next_back() + .expect("the rejected second import should be reported"); + assert_eq!(report.outcome, OperationOutcome::Failure); + assert!( + report + .summary + .to_ascii_lowercase() + .contains("finish or cancel"), + "{report:?}" + ); +} + +#[cfg(unix)] +#[test] +fn origin_dispatch_reuses_the_classified_handle_after_path_replacement() { + use std::os::unix::net::UnixListener; + + let id = uuid::Uuid::new_v4(); + let path = PathBuf::from("/tmp").join(format!("px-{id}.opj")); + let original_path = PathBuf::from("/tmp").join(format!("px-{id}.saved")); + std::fs::write(&path, OPENOPJ_FIXTURE).unwrap(); + let classified = classify_open_path(&path).unwrap(); + assert_eq!(classified.kind(), RecentOpenKind::OriginProject); + std::fs::rename(&path, &original_path).unwrap(); + let replacement = UnixListener::bind(&path).unwrap(); + let mut app = PlotxApp::new_with_settings(plotx_core::settings::Settings::default()); + + dispatch_classified_path(&mut app, &path, classified); + + drop(replacement); + std::fs::remove_file(&path).unwrap(); + std::fs::remove_file(original_path).unwrap(); + let preview = app + .session + .ui + .table_import_preview + .as_ref() + .expect("dispatch must consume the original classified file handle"); + assert_eq!(preview.recent_path.as_deref(), Some(path.as_path())); + assert!(!preview.candidates.is_empty()); + assert!(app.doc.datasets.is_empty()); + assert!(app.session.recent_files.is_empty()); +} + +#[cfg(unix)] +#[test] +fn origin_classification_rejects_non_regular_handle_metadata() { + let device = std::fs::File::open("/dev/null").unwrap(); + + let error = classify_open_handle(Path::new("device.opj"), device) + .expect_err("a character-device handle must be rejected before header reads"); + + assert!(error.to_string().contains("regular file"), "{error}"); +} + +#[cfg(unix)] +#[test] +fn origin_classification_opens_paths_in_nonblocking_mode() { + use std::os::fd::AsRawFd; + + let path = temp_origin_file("opj", OPENOPJ_FIXTURE); + let file = open_file_for_classification(&path).unwrap(); + let flags = unsafe { libc::fcntl(file.as_raw_fd(), libc::F_GETFL) }; + std::fs::remove_file(path).unwrap(); + + assert_ne!(flags, -1, "F_GETFL must succeed"); + assert_ne!(flags & libc::O_NONBLOCK, 0); +} + +#[test] +fn origin_rewind_and_full_read_errors_are_propagated() { + let limits = OriginLimits::default(); + let rewind_error = read_origin_handle(&mut SeekFailure, Some(0), limits) + .expect_err("rewind errors must stop the import"); + assert_eq!(rewind_error.stage, "rewind"); + assert!( + rewind_error.detail.contains("injected rewind failure"), + "{}", + rewind_error.detail + ); + + let read_error = read_origin_handle(&mut ReadFailure, Some(0), limits) + .expect_err("full-read errors must stop the import"); + assert_eq!(read_error.stage, "read"); + assert!( + read_error.detail.contains("injected full-read failure"), + "{}", + read_error.detail + ); +} + +#[test] +fn origin_oversized_metadata_is_rejected_before_rewind() { + let limits = OriginLimits { + max_input_bytes: 4, + max_total_owned_bytes: 8, + ..OriginLimits::default() + }; + let oversized = 5; + + let error = read_origin_handle(&mut SeekFailure, Some(oversized), limits) + .expect_err("known oversized input must be rejected before rewinding"); + + assert_eq!(error.stage, "metadata"); + assert_eq!( + error.detail, + OriginError::LimitExceeded { + resource: "input bytes", + limit: 4, + actual: 5, + } + .to_string() + ); +} + +#[test] +fn origin_lower_total_owned_metadata_limit_is_rejected_before_rewind() { + let limits = OriginLimits { + max_input_bytes: 8, + max_total_owned_bytes: 4, + ..OriginLimits::default() + }; + + let error = read_origin_handle(&mut SeekFailure, Some(5), limits) + .expect_err("known cumulative oversize must be rejected before rewinding"); + + assert_eq!(error.stage, "metadata"); + assert_eq!( + error.detail, + OriginError::LimitExceeded { + resource: "total owned bytes", + limit: 4, + actual: 5, + } + .to_string() + ); +} + +#[test] +fn origin_unknown_length_stops_at_the_lower_total_owned_sentinel() { + let limits = OriginLimits { + max_input_bytes: 8, + max_total_owned_bytes: 4, + ..OriginLimits::default() + }; + let bytes_read = Rc::new(Cell::new(0)); + let reader = CountingRepeat { + bytes_read: Rc::clone(&bytes_read), + }; + + let error = read_bounded_origin(reader, None, limits).unwrap_err(); + + assert_eq!(bytes_read.get(), 5); + assert_eq!( + error, + OriginError::LimitExceeded { + resource: "total owned bytes", + limit: 4, + actual: 5, + } + .to_string() + ); +} + +#[cfg(unix)] +#[test] +fn origin_routing_metadata_errors_are_not_silently_discarded() { + use std::os::unix::fs::symlink; + + let path = std::env::temp_dir().join(format!( + "plotx-origin-metadata-loop-{}.opj", + uuid::Uuid::new_v4() + )); + symlink(&path, &path).unwrap(); + + let result = classify_open_path(&path); + let mut app = PlotxApp::new_with_settings(plotx_core::settings::Settings::default()); + open_recent_path(&mut app, &path); + + std::fs::remove_file(path).unwrap(); + assert!( + result.is_err(), + "metadata errors must be propagated: {result:?}" + ); + let report = app + .session + .operation_history + .operations() + .next_back() + .expect("routing metadata errors must be user-visible"); + assert_eq!(report.outcome, OperationOutcome::Failure); + assert!( + report + .diagnostics + .iter() + .any(|diagnostic| diagnostic.source.as_deref() == Some("app.open_path")), + "{report:?}" + ); +} + +#[test] +fn origin_header_read_errors_are_propagated_by_the_shared_classifier() { + let result = classify_open_path_with_header( + Path::new("unreadable.bin"), + OpenPathEntryType::RegularFile, + || { + Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "injected header read failure", + )) + }, + ); + + let error = result.expect_err("header read errors must not fall back to an extension route"); + assert!(error.to_string().contains("header"), "{error}"); + assert!( + error.to_string().contains("injected header read failure"), + "{error}" + ); +} + +#[test] +fn origin_non_regular_file_classification_never_reads_a_header() { + let result = + classify_open_path_with_header(Path::new("stream.opj"), OpenPathEntryType::Other, || { + panic!("non-regular paths must be rejected before header reads") + }); + + let error = result.expect_err("non-regular paths must be rejected"); + assert!(error.to_string().contains("regular file"), "{error}"); +} + +#[cfg(unix)] +#[test] +fn origin_routing_rejects_non_regular_files_without_opening_them() { + use std::os::unix::net::UnixListener; + + let path = PathBuf::from("/tmp").join(format!("px-{}.opj", uuid::Uuid::new_v4())); + let listener = UnixListener::bind(&path).unwrap(); + + let result = classify_open_path(&path); + + drop(listener); + std::fs::remove_file(path).unwrap(); + assert!( + result.is_err(), + "non-regular files must be rejected before opening: {result:?}" + ); +} + +#[test] +fn origin_opj_extension_rejects_an_opju_signature() { + let path = temp_origin_file("opj", b"CPYUA 4.3668 178\n"); + let mut app = PlotxApp::new_with_settings(plotx_core::settings::Settings::default()); + + open_recent_path(&mut app, &path); + + std::fs::remove_file(path).unwrap(); + assert!(app.session.ui.table_import_preview.is_none()); + assert!(app.session.recent_files.is_empty()); + let report = app + .session + .operation_history + .operations() + .next_back() + .expect("the extension/signature mismatch must be reported"); + assert_eq!(report.outcome, OperationOutcome::Failure); + assert!(report.summary.contains("does not match"), "{report:?}"); +} + +#[test] +fn origin_opju_extension_rejects_an_opj_signature() { + let path = temp_origin_file("opju", OPENOPJ_FIXTURE); + let mut app = PlotxApp::new_with_settings(plotx_core::settings::Settings::default()); + + open_recent_path(&mut app, &path); + + std::fs::remove_file(path).unwrap(); + assert!(app.session.ui.table_import_preview.is_none()); + assert!(app.session.recent_files.is_empty()); + let report = app + .session + .operation_history + .operations() + .next_back() + .expect("the extension/signature mismatch must be reported"); + assert_eq!(report.outcome, OperationOutcome::Failure); + assert!(report.summary.contains("does not match"), "{report:?}"); +} + +#[test] +fn origin_default_limit_accepts_exactly_128_mib_and_rejects_one_more_byte() { + let limits = OriginLimits::default(); + assert_eq!(limits.max_input_bytes, 128 * 1024 * 1024); + + let exact = read_bounded_origin( + io::repeat(0).take(limits.max_input_bytes as u64), + Some(limits.max_input_bytes as u64), + limits, + ) + .expect("the exact default limit must be accepted"); + assert_eq!(exact.len(), limits.max_input_bytes); + drop(exact); + + let error = read_bounded_origin(PanicOnRead, Some(limits.max_input_bytes as u64 + 1), limits) + .expect_err("one byte beyond the default limit must be rejected"); + assert!(error.contains("exceeding"), "{error}"); +} + +#[test] +fn origin_usize_max_input_limit_is_rejected_before_reading() { + let limits = OriginLimits { + max_input_bytes: usize::MAX, + ..OriginLimits::default() + }; + let error = read_bounded_origin(PanicOnRead, None, limits).unwrap_err(); + assert!( + error.contains("invalid Origin limit max_input_bytes"), + "{error}" + ); +} + +#[test] +fn recent_entries_route_to_origin_project_import() { + let classify = |path: &Path| { + classify_open_path_with_header(path, OpenPathEntryType::RegularFile, || { + Ok(([0_u8; OPEN_HEADER_BYTES], 0)) + }) + .unwrap() + }; + assert_eq!( + format!("{:?}", classify(Path::new("project.OPJU"))), + "OriginProject" + ); + assert_ne!(classify(Path::new("project.opj")), RecentOpenKind::DataFile); +} + +#[test] +fn origin_signature_mismatch_becomes_a_user_visible_failure_report() { + let path = temp_origin_file("opj", b"not an Origin project"); + let mut app = PlotxApp::new_with_settings(plotx_core::settings::Settings::default()); + + open_recent_path(&mut app, &path); + + std::fs::remove_file(path).unwrap(); + assert!(app.session.ui.table_import_preview.is_none()); + assert!(app.session.recent_files.is_empty()); + let report = app + .session + .operation_history + .operations() + .next_back() + .expect("a user-visible failure report"); + assert_eq!(report.outcome, OperationOutcome::Failure); + assert!( + report.diagnostics.iter().any(|diagnostic| diagnostic + .message + .to_ascii_lowercase() + .contains("signature")), + "{report:?}" + ); +} + +#[test] +fn origin_opju_is_unsupported_without_preview_or_recent_entry() { + let path = temp_origin_file("opju", b"CPYUA 4.3668 178\n"); + let mut app = PlotxApp::new_with_settings(plotx_core::settings::Settings::default()); + + open_recent_path(&mut app, &path); + + std::fs::remove_file(path).unwrap(); + assert!(app.session.ui.table_import_preview.is_none()); + assert!(app.session.recent_files.is_empty()); + let report = app + .session + .operation_history + .operations() + .next_back() + .expect("OPJU must produce a failure report"); + assert_eq!(report.outcome, OperationOutcome::Failure); + assert!(report.summary.contains("OPJU"), "{report:?}"); + assert!( + report.summary.contains("No data was imported"), + "{report:?}" + ); +} + +#[test] +fn unsupported_classic_opj_reports_its_version_and_supported_profile() { + let path = temp_origin_file("opj", b"CPYA 4.3224 220 W64 #\n"); + let mut app = PlotxApp::new_with_settings(plotx_core::settings::Settings::default()); + + open_recent_path(&mut app, &path); + + std::fs::remove_file(path).unwrap(); + assert!(app.session.ui.table_import_preview.is_none()); + assert!(app.session.recent_files.is_empty()); + let report = app + .session + .operation_history + .operations() + .next_back() + .expect("unsupported OPJ must produce a failure report"); + assert_eq!(report.outcome, OperationOutcome::Failure); + assert!(report.summary.contains("4.3224 220 W64"), "{report:?}"); + assert!(report.summary.contains("Origin 7.0552"), "{report:?}"); + assert!( + report.summary.contains("Origin 9.51 build 195 W64"), + "{report:?}" + ); + assert!( + report.summary.contains("No data was imported"), + "{report:?}" + ); +} + +#[test] +fn origin_core_failure_becomes_a_user_visible_operation_report() { + let probe = plotx_io::origin::probe_origin(OPENOPJ_FIXTURE).unwrap(); + let project = plotx_io::origin::OriginProject { + probe, + parameters: Vec::new(), + notes: Vec::new(), + workbooks: Vec::new(), + diagnostics: Vec::new(), + unsupported_objects: Vec::new(), + resource_usage: plotx_io::origin::OriginResourceUsage::default(), + }; + let mut app = PlotxApp::new_with_settings(plotx_core::settings::Settings::default()); + + import_origin_project_model( + &mut app, + Path::new("empty.opj"), + Arc::<[u8]>::from(OPENOPJ_FIXTURE), + project, + OriginLimits::default(), + ); + + let report = app + .session + .operation_history + .operations() + .next_back() + .expect("core failures must be recorded"); + assert_eq!(report.outcome, OperationOutcome::Failure); + assert!( + report + .diagnostics + .iter() + .any(|diagnostic| diagnostic.message.contains("no supported")), + "{report:?}" + ); +} + +#[test] +fn origin_recent_file_is_recorded_only_after_confirmed_full_success() { + let path = temp_origin_file("opj", OPENOPJ_FIXTURE); + let mut app = PlotxApp::new_with_settings(plotx_core::settings::Settings::default()); + let mut noted_paths = Vec::new(); + + open_recent_path(&mut app, &path); + + assert!(app.doc.datasets.is_empty()); + assert!(app.session.recent_files.is_empty()); + assert_eq!(app.session.operation_history.operation_count(), 0); + let candidate_count = app + .session + .ui + .table_import_preview + .as_ref() + .expect("valid OPJ should produce a preview") + .candidates + .len(); + assert!(candidate_count > 0); + + assert!( + crate::ui::file_dialogs::commit_table_import_preview_with_recent(&mut app, |app, path| { + let path = std::path::absolute(path).unwrap(); + noted_paths.push(path.clone()); + app.session.recent_files.push(path); + },) + ); + assert_eq!(app.doc.datasets.len(), candidate_count); + assert_eq!(app.session.recent_files.len(), 1); + assert_eq!( + app.session.recent_files[0], + std::path::absolute(&path).unwrap() + ); + assert_eq!(noted_paths, app.session.recent_files); + std::fs::remove_file(path).unwrap(); +} + +#[test] +fn origin_cancel_leaves_tables_and_recent_files_unchanged() { + let path = temp_origin_file("opj", OPENOPJ_FIXTURE); + let mut app = PlotxApp::new_with_settings(plotx_core::settings::Settings::default()); + open_recent_path(&mut app, &path); + assert!(app.session.ui.table_import_preview.take().is_some()); + + std::fs::remove_file(path).unwrap(); + assert!(app.doc.datasets.is_empty()); + assert!(app.session.recent_files.is_empty()); + assert_eq!(app.session.operation_history.operation_count(), 0); +} + +#[test] +fn origin_candidates_share_source_allocation_and_stable_operation() { + let (store, imported) = duplicated_fixture_import(); + let source_bytes = Arc::<[u8]>::from(OPENOPJ_FIXTURE); + let source_pointer = source_bytes.as_ptr(); + let preview = preview_from_imported( + OperationId(41), + Path::new("selected-project.opj"), + source_bytes, + store, + imported, + ) + .unwrap(); + + assert_eq!(preview.candidates.len(), 2); + for candidate in &preview.candidates { + let source = &candidate.retained_sources[0]; + assert_eq!(source.bytes().as_ptr(), source_pointer); + assert_eq!(source.media_type, "application/x-origin-project"); + assert_eq!(source.name.as_deref(), Some("selected-project.opj")); + assert!( + source + .metadata + .contains_key("space.nmrtist.plotx.import.origin.resource_usage") + ); + assert_eq!( + candidate.typed_state.envelope.revision.operation.name, + ORIGIN_IMPORT_OPERATION + ); + } +} + +#[test] +fn origin_preview_reports_each_parser_warning_once() { + let (store, imported) = duplicated_fixture_import(); + let expected_warnings = imported[0] + .diagnostics + .iter() + .filter(|diagnostic| diagnostic.severity == OriginDiagnosticSeverity::Warning) + .count(); + assert!(expected_warnings > 0); + + let preview = preview_from_imported( + OperationId(43), + Path::new("selected-project.opj"), + Arc::<[u8]>::from(OPENOPJ_FIXTURE), + store, + imported, + ) + .unwrap(); + let actual_warnings = preview + .report + .diagnostics + .iter() + .filter(|diagnostic| diagnostic.severity == Severity::Warning) + .count(); + + assert_eq!(actual_warnings, expected_warnings); +} + +#[test] +fn origin_zero_candidates_fails_without_indexing_candidate_zero() { + let result = std::panic::catch_unwind(|| ensure_candidate_count(0)); + let error = result.expect("zero candidates must not panic").unwrap_err(); + assert!(error.contains("no supported"), "{error}"); +} + +#[test] +fn origin_selector_changes_preview_only_and_confirmation_imports_all_tables() { + assert_eq!( + crate::ui::file_dialogs::preview::candidate_selector_label(), + "Table" + ); + assert_eq!( + crate::ui::file_dialogs::preview::all_candidate_import_summary(2), + "All 2 candidate tables will be imported." + ); + let (store, imported) = duplicated_fixture_import(); + let mut preview = preview_from_imported( + OperationId(42), + Path::new("selected-project.opj"), + Arc::<[u8]>::from(OPENOPJ_FIXTURE), + store, + imported, + ) + .unwrap(); + preview.selected = 1; + preview.recent_path = None; + let mut app = PlotxApp::new_with_settings(plotx_core::settings::Settings::default()); + app.session.ui.table_import_preview = Some(preview); + + assert!( + crate::ui::file_dialogs::commit_table_import_preview_with_recent(&mut app, |_, _| panic!( + "a preview without a recent path must not persist settings" + ),) + ); + assert_eq!(app.doc.datasets.len(), 2); + assert!(app.session.recent_files.is_empty()); +} diff --git a/crates/app/src/ui/file_dialogs/preview.rs b/crates/app/src/ui/file_dialogs/preview.rs index e4cce5e..760ad32 100644 --- a/crates/app/src/ui/file_dialogs/preview.rs +++ b/crates/app/src/ui/file_dialogs/preview.rs @@ -3,10 +3,31 @@ use plotx_core::state::{PlotxApp, TableImportPreviewState}; use super::commit_table_import_preview; +pub(super) fn candidate_selector_label() -> &'static str { + "Table" +} + +pub(super) fn all_candidate_import_summary(count: usize) -> String { + if count == 1 { + "The candidate table will be imported.".to_owned() + } else { + format!("All {count} candidate tables will be imported.") + } +} + pub(crate) fn table_import_preview_window(app: &mut PlotxApp, ctx: &egui::Context) { let Some(mut preview) = app.session.ui.table_import_preview.take() else { return; }; + if preview.candidates.is_empty() { + app.session.ui.table_import_preview = Some(preview); + let committed = commit_table_import_preview(app); + debug_assert!(!committed); + return; + } + if preview.selected >= preview.candidates.len() { + preview.selected = 0; + } let mut import = false; let mut cancel = false; let modal = super::super::modal( @@ -20,7 +41,7 @@ pub(crate) fn table_import_preview_window(app: &mut PlotxApp, ctx: &egui::Contex ui.label("Confirm the inferred schema before the table is added to the project."); ui.separator(); if preview.candidates.len() > 1 { - egui::ComboBox::from_label("Worksheet") + egui::ComboBox::from_label(candidate_selector_label()) .selected_text(&preview.candidates[preview.selected].name) .show_ui(ui, |ui| { for (index, candidate) in preview.candidates.iter().enumerate() { @@ -64,12 +85,7 @@ fn import_summary(ui: &mut egui::Ui, preview: &TableImportPreviewState) { let candidate = &preview.candidates[preview.selected]; let snapshot = &candidate.typed_state.envelope.revision.snapshot; ui.label(format!("Name: {}", candidate.name)); - if preview.candidates.len() > 1 { - ui.label(format!( - "{} worksheet(s) will be imported", - preview.candidates.len() - )); - } + ui.label(all_candidate_import_summary(preview.candidates.len())); ui.label(format!( "{} row(s), {} column(s)", snapshot.row_count, @@ -85,6 +101,7 @@ fn import_summary(ui: &mut egui::Ui, preview: &TableImportPreviewState) { fn schema_table(ui: &mut egui::Ui, preview: &TableImportPreviewState) { ui.strong("Inferred schema"); egui::ScrollArea::vertical() + .id_salt("table_import_schema_scroll") .max_height(170.0) .show(ui, |ui| { egui::Grid::new("table_import_schema_grid") @@ -133,6 +150,7 @@ fn value_preview(ui: &mut egui::Ui, preview: &TableImportPreviewState) { return; }; egui::ScrollArea::horizontal() + .id_salt("table_import_value_scroll") .max_height(210.0) .show(ui, |ui| { egui::Grid::new("table_import_value_grid") diff --git a/crates/app/src/ui/file_dialogs/recent.rs b/crates/app/src/ui/file_dialogs/recent.rs index 6595cf1..12e834e 100644 --- a/crates/app/src/ui/file_dialogs/recent.rs +++ b/crates/app/src/ui/file_dialogs/recent.rs @@ -1,42 +1,393 @@ use super::{ PlotxApp, import_delimited_table_path, import_xlsx_table_path, load_and_note, open_folder_path, + origin, }; +use plotx_core::operation::{Diagnostic, DiagnosticCode, OperationKind, OperationReport, Severity}; +use plotx_io::origin::{OriginError, OriginFormat}; +use std::fmt; +use std::fs::{File, OpenOptions}; +use std::io::{self, Read}; +use std::path::Path; + +pub(super) const OPEN_HEADER_BYTES: usize = plotx_io::origin::MAX_PROBE_BYTES; +type OpenHeader = ([u8; OPEN_HEADER_BYTES], usize); #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum RecentOpenKind { Project, DelimitedTable, XlsxTable, + OriginProject, + Folder, + DataFile, +} + +impl RecentOpenKind { + fn is_table_import(self) -> bool { + matches!( + self, + Self::DelimitedTable | Self::XlsxTable | Self::OriginProject + ) + } +} + +#[derive(Debug)] +pub(crate) enum ClassifiedOpenPath { + Project, + DelimitedTable, + XlsxTable, + OriginProject(origin::OpenOriginSource), Folder, DataFile, } -pub(crate) fn recent_open_kind(path: &std::path::Path) -> RecentOpenKind { +impl ClassifiedOpenPath { + pub(crate) fn kind(&self) -> RecentOpenKind { + match self { + Self::Project => RecentOpenKind::Project, + Self::DelimitedTable => RecentOpenKind::DelimitedTable, + Self::XlsxTable => RecentOpenKind::XlsxTable, + Self::OriginProject(_) => RecentOpenKind::OriginProject, + Self::Folder => RecentOpenKind::Folder, + Self::DataFile => RecentOpenKind::DataFile, + } + } + + fn is_table_import(&self) -> bool { + self.kind().is_table_import() + } +} + +#[cfg(test)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum OpenPathEntryType { + Directory, + RegularFile, + Other, +} + +#[derive(Debug)] +pub(crate) enum OpenPathError { + Io { + stage: &'static str, + error: io::Error, + }, + OriginProbe(OriginError), + OriginFamilyMismatch { + extension: &'static str, + detected: &'static str, + }, + NonRegularFile, +} + +impl OpenPathError { + fn io(stage: &'static str, error: io::Error) -> Self { + Self::Io { stage, error } + } + + fn stage(&self) -> &'static str { + match self { + Self::Io { stage, .. } => stage, + Self::OriginProbe(_) => "origin_probe", + Self::OriginFamilyMismatch { .. } => "origin_family", + Self::NonRegularFile => "file_type", + } + } +} + +impl fmt::Display for OpenPathError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Io { + stage: "metadata", + error, + } => write!( + formatter, + "the selected path could not be inspected: {error}" + ), + Self::Io { + stage: "open", + error, + } => write!(formatter, "the selected file could not be opened: {error}"), + Self::Io { + stage: "handle_metadata", + error, + } => write!(formatter, "the opened file could not be inspected: {error}"), + Self::Io { error, .. } => { + write!( + formatter, + "the selected file header could not be read: {error}" + ) + } + Self::OriginProbe(OriginError::UnsupportedVersion { raw_version }) => write!( + formatter, + "PlotX recognized classic Origin project version {raw_version}, but this build imports only the verified Origin 7.0552 and Origin 9.51 build 195 W64 OPJ profiles. No data was imported" + ), + Self::OriginProbe(error) => { + write!(formatter, "Origin project detection failed: {error}") + } + Self::OriginFamilyMismatch { + extension, + detected, + } => write!( + formatter, + "the .{extension} extension does not match the detected {detected} project signature; no data was imported" + ), + Self::NonRegularFile => write!( + formatter, + "the selected path is neither a regular file nor a directory" + ), + } + } +} + +impl std::error::Error for OpenPathError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Self::Io { error, .. } => Some(error), + Self::OriginProbe(error) => Some(error), + Self::OriginFamilyMismatch { .. } | Self::NonRegularFile => None, + } + } +} + +pub(crate) fn classify_open_path(path: &Path) -> Result { + let metadata = std::fs::metadata(path).map_err(|error| OpenPathError::io("metadata", error))?; + if metadata.is_dir() { + return Ok(ClassifiedOpenPath::Folder); + } + if !metadata.is_file() { + return Err(OpenPathError::NonRegularFile); + } + let file = + open_file_for_classification(path).map_err(|error| OpenPathError::io("open", error))?; + classify_open_handle(path, file) +} + +pub(crate) fn open_file_for_classification(path: &Path) -> io::Result { + let mut options = OpenOptions::new(); + options.read(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.custom_flags(libc::O_NONBLOCK); + } + options.open(path) +} + +pub(crate) fn classify_open_handle( + path: &Path, + mut file: File, +) -> Result { + let metadata = file + .metadata() + .map_err(|error| OpenPathError::io("handle_metadata", error))?; + if !metadata.is_file() { + return Err(OpenPathError::NonRegularFile); + } + let (header, length) = + read_open_header(&mut file).map_err(|error| OpenPathError::io("header", error))?; + let kind = classify_open_header(path, &header[..length])?; + Ok(match kind { + RecentOpenKind::Project => ClassifiedOpenPath::Project, + RecentOpenKind::DelimitedTable => ClassifiedOpenPath::DelimitedTable, + RecentOpenKind::XlsxTable => ClassifiedOpenPath::XlsxTable, + RecentOpenKind::OriginProject => { + ClassifiedOpenPath::OriginProject(origin::OpenOriginSource::new(file, metadata.len())) + } + RecentOpenKind::Folder => ClassifiedOpenPath::Folder, + RecentOpenKind::DataFile => ClassifiedOpenPath::DataFile, + }) +} + +#[cfg(test)] +pub(crate) fn classify_open_path_with_header( + path: &Path, + entry_type: OpenPathEntryType, + read_header: F, +) -> Result +where + F: FnOnce() -> io::Result, +{ + match entry_type { + OpenPathEntryType::Directory => return Ok(RecentOpenKind::Folder), + OpenPathEntryType::Other => return Err(OpenPathError::NonRegularFile), + OpenPathEntryType::RegularFile => {} + } + + let (header, length) = read_header().map_err(|error| OpenPathError::io("header", error))?; + classify_open_header(path, &header[..length]) +} + +fn classify_open_header(path: &Path, header: &[u8]) -> Result { + if header.starts_with(b"CPYA") || header.starts_with(b"CPYUA") { + let probe = plotx_io::origin::probe_origin(header).map_err(OpenPathError::OriginProbe)?; + reject_origin_family_mismatch(path, probe.format)?; + return Ok(RecentOpenKind::OriginProject); + } + + Ok(extension_open_kind(path)) +} + +fn reject_origin_family_mismatch(path: &Path, detected: OriginFormat) -> Result<(), OpenPathError> { + let extension = path.extension().and_then(|extension| extension.to_str()); + let expected = match extension { + Some(extension) if extension.eq_ignore_ascii_case("opj") => Some(OriginFormat::Opj), + Some(extension) if extension.eq_ignore_ascii_case("opju") => Some(OriginFormat::Opju), + _ => None, + }; + if let Some(expected) = expected + && expected != detected + { + return Err(OpenPathError::OriginFamilyMismatch { + extension: match expected { + OriginFormat::Opj => "opj", + OriginFormat::Opju => "opju", + }, + detected: match detected { + OriginFormat::Opj => "OPJ", + OriginFormat::Opju => "OPJU", + }, + }); + } + Ok(()) +} + +fn extension_open_kind(path: &Path) -> RecentOpenKind { let has_extension = |target: &str| { path.extension() .is_some_and(|extension| extension.eq_ignore_ascii_case(target)) }; - if path.is_dir() { - RecentOpenKind::Folder - } else if has_extension("plotx") { + if has_extension("plotx") { RecentOpenKind::Project } else if has_extension("csv") || has_extension("tsv") || has_extension("txt") { RecentOpenKind::DelimitedTable } else if has_extension("xlsx") { RecentOpenKind::XlsxTable + } else if has_extension("opj") || has_extension("opju") { + RecentOpenKind::OriginProject } else { RecentOpenKind::DataFile } } -pub(crate) fn open_recent_path(app: &mut PlotxApp, path: &std::path::Path) { - match recent_open_kind(path) { - RecentOpenKind::Project => app.request_project_transition( +pub(crate) fn recent_open_kind(path: &Path) -> RecentOpenKind { + if path.is_dir() { + RecentOpenKind::Folder + } else { + extension_open_kind(path) + } +} + +fn read_open_header(mut reader: R) -> io::Result { + let mut header = [0_u8; OPEN_HEADER_BYTES]; + let mut length = 0; + while length < header.len() { + match reader.read(&mut header[length..]) { + Ok(0) => break, + Ok(read) => length += read, + Err(error) if error.kind() == io::ErrorKind::Interrupted => continue, + Err(error) => return Err(error), + } + } + Ok((header, length)) +} + +pub(crate) fn open_recent_path(app: &mut PlotxApp, path: &Path) { + let classified = match classify_open_path(path) { + Ok(classified) => classified, + Err(error) => { + record_open_path_failure(app, path, error); + return; + } + }; + dispatch_classified_path(app, path, classified); +} + +pub(crate) fn dispatch_classified_path( + app: &mut PlotxApp, + path: &Path, + classified: ClassifiedOpenPath, +) { + if classified.is_table_import() && app.session.ui.table_import_preview.is_some() { + record_pending_table_import(app, path); + return; + } + match classified { + ClassifiedOpenPath::Project => app.request_project_transition( plotx_core::state::ProjectTransition::Open(path.to_owned()), ), - RecentOpenKind::DelimitedTable => import_delimited_table_path(app, path), - RecentOpenKind::XlsxTable => import_xlsx_table_path(app, path), - RecentOpenKind::Folder => open_folder_path(app, path), - RecentOpenKind::DataFile => load_and_note(app, path), + ClassifiedOpenPath::DelimitedTable => import_delimited_table_path(app, path), + ClassifiedOpenPath::XlsxTable => import_xlsx_table_path(app, path), + ClassifiedOpenPath::OriginProject(source) => { + origin::import_origin_project_source(app, path, source); + } + ClassifiedOpenPath::Folder => open_folder_path(app, path), + ClassifiedOpenPath::DataFile => load_and_note(app, path), } } + +fn record_open_path_failure(app: &mut PlotxApp, path: &Path, error: OpenPathError) { + let operation_id = app.session.begin_operation(); + let message = error.to_string(); + let (operation_kind, diagnostic_code) = open_path_failure_classification(path, &error); + app.session.record_operation(OperationReport::<()>::failure( + operation_id, + operation_kind, + format!("The selected path could not be opened: {message}."), + Diagnostic::new(Severity::Error, diagnostic_code, message) + .with_source("app.open_path") + .with_context("path", path.display().to_string()) + .with_context("stage", error.stage()) + .with_context("error", error.to_string()), + )); +} + +fn open_path_failure_classification( + path: &Path, + error: &OpenPathError, +) -> (OperationKind, DiagnosticCode) { + let table_like_extension = path + .extension() + .and_then(|extension| extension.to_str()) + .is_some_and(|extension| { + ["csv", "tsv", "txt", "xlsx", "opj", "opju"] + .iter() + .any(|candidate| extension.eq_ignore_ascii_case(candidate)) + }); + if table_like_extension + || matches!( + error, + OpenPathError::OriginProbe(_) | OpenPathError::OriginFamilyMismatch { .. } + ) + { + ( + OperationKind::TableImport, + DiagnosticCode::TableImportFailed, + ) + } else { + ( + OperationKind::DatasetLoad, + DiagnosticCode::DatasetLoadFailed, + ) + } +} + +fn record_pending_table_import(app: &mut PlotxApp, path: &Path) { + let operation_id = app.session.begin_operation(); + let message = + "Finish or cancel the current table import preview before importing another table."; + app.session.record_operation(OperationReport::<()>::failure( + operation_id, + OperationKind::TableImport, + message, + Diagnostic::new(Severity::Error, DiagnosticCode::TableImportFailed, message) + .with_source("app.table_import") + .with_context("path", path.display().to_string()) + .with_context("stage", "preview_pending"), + )); +} + +#[cfg(test)] +#[path = "recent_report_tests.rs"] +mod recent_report_tests; diff --git a/crates/app/src/ui/file_dialogs/recent_report_tests.rs b/crates/app/src/ui/file_dialogs/recent_report_tests.rs new file mode 100644 index 0000000..7895a36 --- /dev/null +++ b/crates/app/src/ui/file_dialogs/recent_report_tests.rs @@ -0,0 +1,117 @@ +use super::*; +use plotx_core::operation::{DiagnosticCode, OperationKind}; +use plotx_core::settings::Settings; +use std::path::{Path, PathBuf}; + +fn unique_path(extension: &str) -> PathBuf { + std::env::temp_dir().join(format!( + "plotx-recent-report-{}.{}", + uuid::Uuid::new_v4(), + extension + )) +} + +fn write_temp(extension: &str, bytes: &[u8]) -> PathBuf { + let path = unique_path(extension); + std::fs::write(&path, bytes).unwrap(); + path +} + +fn assert_latest_failure( + app: &PlotxApp, + expected_kind: OperationKind, + expected_code: DiagnosticCode, +) { + let report = app + .session + .operation_history + .operations() + .next_back() + .expect("the open failure must be reported"); + assert_eq!(report.kind, expected_kind, "{report:?}"); + assert!( + report + .diagnostics + .iter() + .any(|diagnostic| diagnostic.code == expected_code), + "{report:?}" + ); +} + +#[test] +fn missing_table_like_paths_report_table_import_failures() { + for extension in ["csv", "tsv", "txt", "xlsx", "opj", "opju"] { + let path = unique_path(extension); + assert!(!path.exists()); + let mut app = PlotxApp::new_with_settings(Settings::default()); + + open_recent_path(&mut app, &path); + + assert_latest_failure( + &app, + OperationKind::TableImport, + DiagnosticCode::TableImportFailed, + ); + } +} + +#[test] +fn origin_probe_errors_report_table_import_even_without_an_origin_extension() { + let path = write_temp("bin", b"CPYA invalid\n"); + let mut app = PlotxApp::new_with_settings(Settings::default()); + + open_recent_path(&mut app, &path); + + std::fs::remove_file(path).unwrap(); + assert_latest_failure( + &app, + OperationKind::TableImport, + DiagnosticCode::TableImportFailed, + ); +} + +#[test] +fn origin_family_mismatches_report_table_import_failures() { + let path = write_temp("opj", b"CPYUA 4.3668 178\n"); + let mut app = PlotxApp::new_with_settings(Settings::default()); + + open_recent_path(&mut app, &path); + + std::fs::remove_file(path).unwrap(); + assert_latest_failure( + &app, + OperationKind::TableImport, + DiagnosticCode::TableImportFailed, + ); +} + +#[test] +fn missing_dataset_and_project_paths_keep_dataset_load_failures() { + for path in [unique_path("abf"), unique_path("plotx")] { + assert!(!path.exists()); + let mut app = PlotxApp::new_with_settings(Settings::default()); + + open_recent_path(&mut app, Path::new(&path)); + + assert_latest_failure( + &app, + OperationKind::DatasetLoad, + DiagnosticCode::DatasetLoadFailed, + ); + } +} + +#[test] +fn later_origin_failures_remain_table_import_failures() { + let path = write_temp("opju", b"CPYUA 4.3668 178\n"); + let mut app = PlotxApp::new_with_settings(Settings::default()); + + open_recent_path(&mut app, &path); + + std::fs::remove_file(path).unwrap(); + assert_latest_failure( + &app, + OperationKind::TableImport, + DiagnosticCode::TableImportFailed, + ); +} diff --git a/crates/app/src/ui/file_dialogs/tests.rs b/crates/app/src/ui/file_dialogs/tests.rs index 3f648be..94b0a0f 100644 --- a/crates/app/src/ui/file_dialogs/tests.rs +++ b/crates/app/src/ui/file_dialogs/tests.rs @@ -1,6 +1,49 @@ use super::*; use std::path::PathBuf; +fn origin_9_initial_structure() -> Vec { + const HEADER_LEN: usize = 115; + let mut bytes = b"CPYA 4.3268 195 W64 #\n".to_vec(); + bytes.extend_from_slice(&(HEADER_LEN as u32).to_le_bytes()); + bytes.push(b'\n'); + let mut header = [0_u8; HEADER_LEN]; + header[0x1b..0x23].copy_from_slice(&9.510195_f64.to_le_bytes()); + bytes.extend_from_slice(&header); + bytes.push(b'\n'); + bytes.extend_from_slice(&[0, 0, 0, 0, b'\n']); + bytes +} + +#[test] +fn table_import_preview_uses_unique_scroll_area_ids() { + let mut app = PlotxApp::new_with_settings(plotx_core::settings::Settings::default()); + import_delimited_text( + &mut app, + "time,value\n0,1\n", + DelimitedTableSource::Clipboard, + ); + let ctx = egui::Context::default(); + + let output = ctx.run_ui(egui::RawInput::default(), |ui| { + table_import_preview_window(&mut app, ui.ctx()); + }); + let painted = format!("{:?}", output.shapes); + + assert!(!painted.contains("use of ScrollArea ID"), "{painted}"); +} + +#[test] +fn recent_path_classification_reads_enough_bytes_for_origin_9() { + let path = + std::env::temp_dir().join(format!("plotx-origin-9-probe-{}.opj", uuid::Uuid::new_v4())); + std::fs::write(&path, origin_9_initial_structure()).unwrap(); + + let classified = recent::classify_open_path(&path); + + std::fs::remove_file(path).unwrap(); + assert_eq!(classified.unwrap().kind(), RecentOpenKind::OriginProject); +} + #[test] fn mixed_columns_import_as_typed_text_without_being_discarded() { let mut app = PlotxApp::new(); @@ -139,30 +182,43 @@ fn clipboard_schema_restores_typed_contract_and_is_retained() { #[test] fn recent_entries_route_to_their_import_path() { let file = |name: &str| PathBuf::from(format!("C:/data/{name}")); + let regular = |path: &std::path::Path| { + recent::classify_open_path_with_header(path, recent::OpenPathEntryType::RegularFile, || { + Ok(([0_u8; recent::OPEN_HEADER_BYTES], 0)) + }) + .unwrap() + }; + assert_eq!(regular(&file("session.PLOTX")), RecentOpenKind::Project); assert_eq!( - recent_open_kind(&file("session.PLOTX")), - RecentOpenKind::Project + regular(&file("results.csv")), + RecentOpenKind::DelimitedTable ); assert_eq!( - recent_open_kind(&file("results.csv")), + regular(&file("results.tsv")), RecentOpenKind::DelimitedTable ); assert_eq!( - recent_open_kind(&file("results.tsv")), + regular(&file("results.txt")), RecentOpenKind::DelimitedTable ); + assert_eq!(regular(&file("results.XLSX")), RecentOpenKind::XlsxTable); + assert_eq!(regular(&file("run.abf")), RecentOpenKind::DataFile); + assert_eq!(regular(&file("fid")), RecentOpenKind::DataFile); assert_eq!( - recent_open_kind(&file("results.txt")), - RecentOpenKind::DelimitedTable + format!("{:?}", regular(&file("project.opj"))), + "OriginProject" ); assert_eq!( - recent_open_kind(&file("results.XLSX")), - RecentOpenKind::XlsxTable + format!("{:?}", regular(&file("project.OPJU"))), + "OriginProject" ); - assert_eq!(recent_open_kind(&file("run.abf")), RecentOpenKind::DataFile); - assert_eq!(recent_open_kind(&file("fid")), RecentOpenKind::DataFile); assert_eq!( - recent_open_kind(&std::env::temp_dir()), + recent::classify_open_path_with_header( + &std::env::temp_dir(), + recent::OpenPathEntryType::Directory, + || panic!("directories must not be opened for header reads"), + ) + .unwrap(), RecentOpenKind::Folder ); @@ -179,9 +235,32 @@ fn recent_entries_route_to_their_import_path() { std::fs::create_dir(&csv_directory).expect("create CSV-named directory"); std::fs::create_dir(&plotx_directory).expect("create PlotX-named directory"); let kinds = ( - recent_open_kind(&csv_directory), - recent_open_kind(&plotx_directory), + recent::classify_open_path(&csv_directory).unwrap().kind(), + recent::classify_open_path(&plotx_directory).unwrap().kind(), ); std::fs::remove_dir_all(&root).expect("remove recent-open test directory"); assert_eq!(kinds, (RecentOpenKind::Folder, RecentOpenKind::Folder)); } + +#[test] +fn project_dispatch_preserves_dirty_document_confirmation() { + let mut app = PlotxApp::new_with_settings(plotx_core::settings::Settings::default()); + app.mark_document_dirty(); + let path = PathBuf::from("pending-project.plotx"); + + recent::dispatch_classified_path(&mut app, &path, recent::ClassifiedOpenPath::Project); + + let pending = app + .session + .ui + .project_transition + .expect("a dirty document must defer the project open"); + assert_eq!( + pending.target, + plotx_core::state::ProjectTransition::Open(path) + ); + assert_eq!( + pending.phase, + plotx_core::state::ProjectTransitionPhase::NeedsConfirmation + ); +} diff --git a/crates/app/src/ui/file_dialogs/xlsx.rs b/crates/app/src/ui/file_dialogs/xlsx.rs index d0ca78c..014a317 100644 --- a/crates/app/src/ui/file_dialogs/xlsx.rs +++ b/crates/app/src/ui/file_dialogs/xlsx.rs @@ -45,8 +45,8 @@ pub(super) fn import_xlsx_table_path(app: &mut PlotxApp, path: &std::path::Path) operation_id, path, "worksheet_selection", - "The workbook has no visible data worksheets.", - "no visible non-empty worksheet".into(), + "The workbook has no visible data tables.", + "no visible non-empty table".into(), ); return; } @@ -68,7 +68,7 @@ pub(super) fn import_xlsx_table_path(app: &mut PlotxApp, path: &std::path::Path) let mut report = OperationReport::success( operation_id, OperationKind::TableImport, - format!("Imported {sheet_count} worksheet(s) from an XLSX workbook."), + format!("Imported {sheet_count} table(s) from an XLSX workbook."), (), ); let mut candidates = Vec::with_capacity(sheet_count); @@ -125,10 +125,7 @@ pub(super) fn import_xlsx_table_path(app: &mut PlotxApp, path: &std::path::Path) Diagnostic::new( Severity::Info, DiagnosticCode::TableImportSucceeded, - format!( - "Imported worksheet '{}' with {row_count} row(s).", - sheet.name - ), + format!("Imported table '{}' with {row_count} row(s).", sheet.name), ) .with_source("core.xlsx") .with_context("path", path.display().to_string()) @@ -150,7 +147,7 @@ pub(super) fn import_xlsx_table_path(app: &mut PlotxApp, path: &std::path::Path) if warning_count > 0 { report.outcome = plotx_core::operation::OperationOutcome::Warning; report.summary = - format!("Imported {sheet_count} XLSX worksheet(s) with {warning_count} warning(s)."); + format!("Imported {sheet_count} XLSX table(s) with {warning_count} warning(s)."); } app.session.ui.table_import_preview = Some(TableImportPreviewState { candidates, diff --git a/crates/app/src/ui/shortcuts.rs b/crates/app/src/ui/shortcuts.rs index e21e270..0f4371c 100644 --- a/crates/app/src/ui/shortcuts.rs +++ b/crates/app/src/ui/shortcuts.rs @@ -520,7 +520,7 @@ pub(super) fn handle_file_drop(app: &mut PlotxApp, ctx: &egui::Context) { painter.text( rect.center(), egui::Align2::CENTER_CENTER, - "Drop a .plotx project, a CSV/TSV table, an .abf/.jdf file, a Waters .raw or Bruker folder, or a .zip archive", + "Drop a PlotX project, table, Origin project, data file, data folder, or archive", egui::FontId::proportional(24.0), Color32::WHITE, ); diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index dd29a69..1273008 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -8,6 +8,13 @@ pub mod export; pub mod fit_model_library; pub mod layout; pub mod operation; +pub mod origin; +#[cfg(test)] +#[path = "origin_hardening_tests.rs"] +mod origin_hardening_tests; +#[cfg(test)] +#[path = "origin_tests.rs"] +mod origin_tests; pub mod project; pub mod properties; pub mod settings; diff --git a/crates/core/src/origin.rs b/crates/core/src/origin.rs new file mode 100644 index 0000000..cfd54a4 --- /dev/null +++ b/crates/core/src/origin.rs @@ -0,0 +1,720 @@ +//! Conversion from the bounded Origin transport model to PlotX tables. + +use std::{collections::BTreeMap, mem::size_of}; + +use plotx_data::{ + BlockStore, CodecRegistry, ColumnChunk, ColumnSchema, ColumnValues, LogicalType, RowId, + SnapshotBuilder, TableId, TableSchema, TableSnapshot, Validity, +}; +use plotx_io::origin::{ + OriginCell, OriginColumn, OriginColumnType, OriginDiagnostic, OriginDiagnosticCode, + OriginDiagnosticSeverity, OriginError, OriginLimits, OriginMetadataEntry, OriginNote, + OriginProject, OriginResourceUsage, OriginUnsupportedObjectSummary, OriginWorksheet, +}; +use serde_json::{Map, Value}; + +mod names; +mod preflight; + +/// Stable operation identifier stored in revisions created from Origin imports. +pub const ORIGIN_IMPORT_OPERATION: &str = "plotx.import.origin.v1"; + +const CHUNK_ROWS: usize = 65_536; + +const FORMAT_KEY: &str = "space.nmrtist.plotx.import.format"; +const VERSION_KEY: &str = "space.nmrtist.plotx.import.origin.producer_version"; +const WORKBOOK_KEY: &str = "space.nmrtist.plotx.import.origin.workbook"; +const WORKSHEET_KEY: &str = "space.nmrtist.plotx.import.origin.worksheet"; +const PARAMETERS_KEY: &str = "space.nmrtist.plotx.import.origin.parameters"; +const NOTES_KEY: &str = "space.nmrtist.plotx.import.origin.notes"; +const DIAGNOSTICS_KEY: &str = "space.nmrtist.plotx.import.origin.diagnostics"; +const UNSUPPORTED_KEY: &str = "space.nmrtist.plotx.import.origin.unsupported_objects"; +const USAGE_KEY: &str = "space.nmrtist.plotx.import.origin.resource_usage"; +const COLUMNS_KEY: &str = "space.nmrtist.plotx.import.origin.columns"; +const WORKSHEET_METADATA_KEY: &str = "space.nmrtist.plotx.import.origin.worksheet_metadata"; +const ORIGINAL_NAME_KEY: &str = "space.nmrtist.plotx.import.origin.original_name"; +const LONG_NAME_KEY: &str = "space.nmrtist.plotx.import.origin.long_name"; +const ROLE_KEY: &str = "space.nmrtist.plotx.import.origin.role"; +const UNITS_KEY: &str = "space.nmrtist.plotx.import.origin.units"; +const COMMENTS_KEY: &str = "space.nmrtist.plotx.import.origin.comments"; + +/// One worksheet ready for the application's existing import-preview flow. +#[derive(Debug)] +pub struct ImportedOriginWorksheet { + /// Human-readable candidate label preserving workbook and worksheet identity. + pub name: String, + /// Typed, validity-aware table snapshot. + pub snapshot: TableSnapshot, + /// Bounded metadata suitable for direct attachment to `TableImportSource`. + pub source_metadata: BTreeMap, + /// Recoverable parser diagnostics retained for the operation report. + pub diagnostics: Vec, + /// Shared cumulative parser and conversion allocation estimate. + pub resource_usage: OriginResourceUsage, +} + +struct PreparedOriginWorksheet { + name: String, + worksheet: OriginWorksheet, + schema: TableSchema, + source_metadata: BTreeMap, + snapshot_metadata: BTreeMap, + diagnostics: Vec, + resource_usage: OriginResourceUsage, +} + +/// Errors raised while validating or converting an engine-neutral Origin model. +#[derive(Debug, thiserror::Error)] +pub enum OriginImportError { + #[error(transparent)] + Origin(#[from] OriginError), + + #[error(transparent)] + Data(#[from] plotx_data::DataError), + + #[error("the Origin project contains no supported worksheet data")] + NoSupportedWorksheet, + + #[error("invalid Origin project model: {detail}")] + InvalidModel { detail: String }, + + #[error( + "Origin column {column:?} row {row} contains {actual}, but its declared type is {expected:?}" + )] + InvalidCellType { + column: String, + row: usize, + expected: OriginColumnType, + actual: &'static str, + }, + + #[error("Origin table conversion size calculation overflowed for {resource}")] + ArithmeticOverflow { resource: &'static str }, + + #[error( + "Origin table conversion {resource} is {actual}, exceeding the configured limit of {limit}" + )] + LimitExceeded { + resource: &'static str, + limit: usize, + actual: usize, + }, + + #[error("Origin table conversion could not reserve {requested} bytes for {resource}")] + AllocationFailed { + resource: &'static str, + requested: usize, + }, +} + +/// Converts every nonempty Origin worksheet into an independent typed snapshot. +/// +/// The complete neutral model is validated and conservatively charged against +/// `max_total_owned_bytes` before a snapshot builder or block-store write is +/// created. Source cell vectors are then drained batch by batch, so text cell +/// storage is moved rather than cloned. +pub fn import_origin_project( + project: OriginProject, + store: &dyn BlockStore, + codecs: &CodecRegistry, + limits: OriginLimits, +) -> Result, OriginImportError> { + limits.validate()?; + let preflight = preflight::validate(&project, &limits)?; + if preflight.worksheets.is_empty() { + return Err(OriginImportError::NoSupportedWorksheet); + } + + let OriginProject { + probe, + parameters, + notes, + workbooks, + diagnostics, + unsupported_objects, + mut resource_usage, + } = project; + resource_usage.total_owned_bytes = preflight.total_owned_bytes; + let candidate_count = preflight.worksheets.len(); + let mut worksheet_preflights = preflight.worksheets.into_iter(); + let mut prepared = Vec::new(); + try_reserve(&mut prepared, candidate_count, "prepared Origin worksheets")?; + + for workbook in workbooks { + let workbook_name = workbook.name; + for worksheet in workbook.worksheets { + if worksheet.row_count == 0 || worksheet.columns.is_empty() { + continue; + } + let worksheet_preflight = + worksheet_preflights + .next() + .ok_or_else(|| OriginImportError::InvalidModel { + detail: "worksheet preflight count does not match the retained model" + .to_owned(), + })?; + let source_metadata = source_metadata( + &probe.raw_version, + ¶meters, + ¬es, + &diagnostics, + &unsupported_objects, + &resource_usage, + &workbook_name, + &worksheet, + &worksheet_preflight.imported_names, + )?; + let snapshot_metadata = snapshot_metadata(&source_metadata)?; + let name = candidate_name(&workbook_name, &worksheet.name)?; + let schema = build_schema(&worksheet.columns, worksheet_preflight.imported_names)?; + prepared.push(PreparedOriginWorksheet { + name, + worksheet, + schema, + source_metadata, + snapshot_metadata, + diagnostics: diagnostics.clone(), + resource_usage: resource_usage.clone(), + }); + } + } + if worksheet_preflights.next().is_some() { + return Err(OriginImportError::InvalidModel { + detail: "worksheet preflight count does not match the retained model".to_owned(), + }); + } + + let mut imported = Vec::new(); + try_reserve( + &mut imported, + candidate_count, + "Origin worksheet candidates", + )?; + for prepared in prepared { + let snapshot = build_snapshot( + prepared.worksheet, + prepared.schema, + prepared.snapshot_metadata, + store, + codecs, + )?; + imported.push(ImportedOriginWorksheet { + name: prepared.name, + snapshot, + source_metadata: prepared.source_metadata, + diagnostics: prepared.diagnostics, + resource_usage: prepared.resource_usage, + }); + } + Ok(imported) +} + +fn build_snapshot( + mut worksheet: OriginWorksheet, + schema: TableSchema, + metadata: BTreeMap, + store: &dyn BlockStore, + codecs: &CodecRegistry, +) -> Result { + let mut builder = + SnapshotBuilder::new(TableId::new(), schema, store, codecs)?.with_trusted_row_identity(); + *builder.metadata_mut() = metadata; + + let mut row_start = 0_usize; + while row_start < worksheet.row_count { + let row_count = (worksheet.row_count - row_start).min(CHUNK_ROWS); + let mut chunks = Vec::new(); + try_reserve(&mut chunks, worksheet.columns.len(), "Origin column chunks")?; + for column in &mut worksheet.columns { + chunks.push(drain_column_chunk(column, row_start, row_count)?); + } + let mut row_ids = Vec::new(); + try_reserve(&mut row_ids, row_count, "Origin row identities")?; + row_ids.extend((0..row_count).map(|_| RowId::new())); + builder.push_batch(&row_ids, &chunks)?; + row_start = checked_add(row_start, row_count, "worksheet row offset")?; + } + Ok(builder.finish()?) +} + +fn build_schema( + columns: &[OriginColumn], + imported_names: Vec, +) -> Result { + let mut schemas = Vec::new(); + try_reserve(&mut schemas, columns.len(), "Origin column schemas")?; + for (column, name) in columns.iter().zip(imported_names) { + let logical_type = match column.column_type { + OriginColumnType::Float => LogicalType::Float64, + OriginColumnType::Integer => LogicalType::Int64, + OriginColumnType::Text | OriginColumnType::Mixed => LogicalType::Utf8, + }; + let changed = name != column.name; + let mut schema = ColumnSchema::new(name, logical_type); + if changed { + insert_text(&mut schema.metadata, ORIGINAL_NAME_KEY, &column.name)?; + } + for (key, value) in [ + (LONG_NAME_KEY, column.long_name.as_deref()), + (ROLE_KEY, column.role.as_deref()), + (UNITS_KEY, column.units.as_deref()), + (COMMENTS_KEY, column.comments.as_deref()), + ] { + if let Some(value) = value { + insert_text(&mut schema.metadata, key, value)?; + } + } + schemas.push(schema); + } + Ok(TableSchema::new(schemas)?) +} + +fn drain_column_chunk( + column: &mut OriginColumn, + row_start: usize, + row_count: usize, +) -> Result { + let take = row_count.min(column.cells.len()); + let column_name = column.name.as_str(); + let column_type = column.column_type; + let mut validity = Vec::new(); + try_reserve(&mut validity, row_count, "Origin validity input")?; + let values = match column.column_type { + OriginColumnType::Float => { + let mut values = Vec::new(); + try_reserve(&mut values, row_count, "Origin Float64 values")?; + for (index, cell) in column.cells.drain(..take).enumerate() { + match cell { + OriginCell::Float(value) => { + values.push(value); + validity.push(true); + } + OriginCell::Null => { + values.push(0.0); + validity.push(false); + } + other => { + return cell_type_error( + column_name, + column_type, + checked_add(row_start, index, "worksheet row index")?, + &other, + ); + } + } + } + values.resize(row_count, 0.0); + ColumnValues::Float64(values) + } + OriginColumnType::Integer => { + let mut values = Vec::new(); + try_reserve(&mut values, row_count, "Origin Int64 values")?; + for (index, cell) in column.cells.drain(..take).enumerate() { + match cell { + OriginCell::Integer(value) => { + values.push(value); + validity.push(true); + } + OriginCell::Null => { + values.push(0); + validity.push(false); + } + other => { + return cell_type_error( + column_name, + column_type, + checked_add(row_start, index, "worksheet row index")?, + &other, + ); + } + } + } + values.resize(row_count, 0); + ColumnValues::Int64(values) + } + OriginColumnType::Text | OriginColumnType::Mixed => { + let mut values = Vec::new(); + try_reserve(&mut values, row_count, "Origin UTF-8 values")?; + for (index, cell) in column.cells.drain(..take).enumerate() { + match cell { + OriginCell::Text(value) => { + values.push(value); + validity.push(true); + } + OriginCell::Float(value) if column.column_type == OriginColumnType::Mixed => { + values.push(value.to_string()); + validity.push(true); + } + OriginCell::Integer(value) if column.column_type == OriginColumnType::Mixed => { + values.push(value.to_string()); + validity.push(true); + } + OriginCell::Null => { + values.push(String::new()); + validity.push(false); + } + other => { + return cell_type_error( + column_name, + column_type, + checked_add(row_start, index, "worksheet row index")?, + &other, + ); + } + } + } + values.resize_with(row_count, String::new); + ColumnValues::Utf8(values) + } + }; + validity.resize(row_count, false); + Ok(ColumnChunk::new(values, Validity::from_valid(validity))?) +} + +fn cell_type_error( + column: &str, + expected: OriginColumnType, + row: usize, + cell: &OriginCell, +) -> Result { + Err(OriginImportError::InvalidCellType { + column: copy_text(column, "Origin column name")?, + row, + expected, + actual: cell_kind(cell), + }) +} + +#[allow(clippy::too_many_arguments)] +fn source_metadata( + version: &str, + parameters: &[OriginMetadataEntry], + notes: &[OriginNote], + diagnostics: &[OriginDiagnostic], + unsupported: &[OriginUnsupportedObjectSummary], + usage: &OriginResourceUsage, + workbook: &str, + worksheet: &OriginWorksheet, + imported_names: &[String], +) -> Result, OriginImportError> { + let mut metadata = BTreeMap::new(); + insert_text(&mut metadata, FORMAT_KEY, "opj")?; + insert_text(&mut metadata, VERSION_KEY, version)?; + insert_text(&mut metadata, WORKBOOK_KEY, workbook)?; + insert_text(&mut metadata, WORKSHEET_KEY, &worksheet.name)?; + metadata.insert(PARAMETERS_KEY.to_owned(), entries_json(parameters)?); + metadata.insert(NOTES_KEY.to_owned(), notes_json(notes)?); + metadata.insert(DIAGNOSTICS_KEY.to_owned(), diagnostics_json(diagnostics)?); + metadata.insert(UNSUPPORTED_KEY.to_owned(), unsupported_json(unsupported)?); + metadata.insert(USAGE_KEY.to_owned(), usage_json(usage)); + metadata.insert( + COLUMNS_KEY.to_owned(), + columns_json(&worksheet.columns, imported_names)?, + ); + metadata.insert( + WORKSHEET_METADATA_KEY.to_owned(), + entries_json(&worksheet.metadata)?, + ); + Ok(metadata) +} + +fn snapshot_metadata( + source: &BTreeMap, +) -> Result, OriginImportError> { + let mut metadata = BTreeMap::new(); + for key in [ + FORMAT_KEY, + VERSION_KEY, + WORKBOOK_KEY, + WORKSHEET_KEY, + DIAGNOSTICS_KEY, + ] { + let value = source + .get(key) + .ok_or_else(|| OriginImportError::InvalidModel { + detail: format!("source metadata is missing {key}"), + })?; + metadata.insert(key.to_owned(), value.clone()); + } + Ok(metadata) +} + +fn entries_json(entries: &[OriginMetadataEntry]) -> Result { + let mut values = Vec::new(); + try_reserve(&mut values, entries.len(), "Origin metadata JSON")?; + for entry in entries { + values.push(Value::Object(Map::from_iter([ + ( + "key".to_owned(), + Value::String(copy_text(&entry.key, "metadata key")?), + ), + ( + "value".to_owned(), + Value::String(copy_text(&entry.value, "metadata value")?), + ), + ]))); + } + Ok(Value::Array(values)) +} + +fn notes_json(notes: &[OriginNote]) -> Result { + let mut values = Vec::new(); + try_reserve(&mut values, notes.len(), "Origin notes JSON")?; + for note in notes { + values.push(Value::Object(Map::from_iter([ + ( + "name".to_owned(), + Value::String(copy_text(¬e.name, "note name")?), + ), + ( + "content".to_owned(), + Value::String(copy_text(¬e.content, "note content")?), + ), + ]))); + } + Ok(Value::Array(values)) +} + +fn diagnostics_json(diagnostics: &[OriginDiagnostic]) -> Result { + let mut values = Vec::new(); + try_reserve(&mut values, diagnostics.len(), "Origin diagnostics JSON")?; + for diagnostic in diagnostics { + let location = diagnostic.location.as_ref().map(|location| { + Value::Object(Map::from_iter([ + ( + "workbook".to_owned(), + option_text(location.workbook.as_deref()), + ), + ( + "worksheet".to_owned(), + option_text(location.worksheet.as_deref()), + ), + ("column".to_owned(), option_text(location.column.as_deref())), + ("byte_offset".to_owned(), usize_value(location.byte_offset)), + ])) + }); + values.push(Value::Object(Map::from_iter([ + ( + "code".to_owned(), + Value::String(diagnostic_code(diagnostic.code).to_owned()), + ), + ( + "severity".to_owned(), + Value::String(diagnostic_severity(diagnostic.severity).to_owned()), + ), + ("location".to_owned(), location.unwrap_or(Value::Null)), + ( + "message".to_owned(), + Value::String(copy_text(&diagnostic.message, "diagnostic message")?), + ), + ]))); + } + Ok(Value::Array(values)) +} + +fn unsupported_json( + unsupported: &[OriginUnsupportedObjectSummary], +) -> Result { + let mut values = Vec::new(); + try_reserve( + &mut values, + unsupported.len(), + "unsupported Origin object JSON", + )?; + for summary in unsupported { + values.push(Value::Object(Map::from_iter([ + ( + "kind".to_owned(), + Value::String(copy_text(&summary.kind, "object kind")?), + ), + ("count".to_owned(), Value::from(summary.count)), + ]))); + } + Ok(Value::Array(values)) +} + +fn columns_json( + columns: &[OriginColumn], + imported_names: &[String], +) -> Result { + let mut values = Vec::new(); + try_reserve(&mut values, columns.len(), "Origin column metadata JSON")?; + for (index, (column, imported_name)) in columns.iter().zip(imported_names).enumerate() { + let mut value = Map::new(); + value.insert("index".to_owned(), Value::from(index)); + value.insert( + "source_name".to_owned(), + Value::String(copy_text(&column.name, "column source name")?), + ); + value.insert( + "imported_name".to_owned(), + Value::String(copy_text(imported_name, "column imported name")?), + ); + value.insert( + "long_name".to_owned(), + option_text(column.long_name.as_deref()), + ); + value.insert("role".to_owned(), option_text(column.role.as_deref())); + value.insert("units".to_owned(), option_text(column.units.as_deref())); + value.insert( + "comments".to_owned(), + option_text(column.comments.as_deref()), + ); + values.push(Value::Object(value)); + } + Ok(Value::Array(values)) +} + +fn usage_json(usage: &OriginResourceUsage) -> Value { + Value::Object(Map::from_iter([ + ("input_bytes".to_owned(), Value::from(usage.input_bytes)), + ("parser_bytes".to_owned(), Value::from(usage.parser_bytes)), + ( + "decoded_text_bytes".to_owned(), + Value::from(usage.decoded_text_bytes), + ), + ( + "total_owned_bytes".to_owned(), + Value::from(usage.total_owned_bytes), + ), + ("workbooks".to_owned(), Value::from(usage.workbooks)), + ("worksheets".to_owned(), Value::from(usage.worksheets)), + ("columns".to_owned(), Value::from(usage.columns)), + ("cells".to_owned(), Value::from(usage.cells)), + ( + "metadata_records".to_owned(), + Value::from(usage.metadata_records), + ), + ])) +} + +fn candidate_name(workbook: &str, worksheet: &str) -> Result { + let capacity = checked_add( + checked_add(workbook.len(), worksheet.len(), "candidate name")?, + 3, + "candidate name", + )?; + let mut name = String::new(); + name.try_reserve_exact(capacity) + .map_err(|_| OriginImportError::AllocationFailed { + resource: "Origin candidate name", + requested: capacity, + })?; + name.push_str(workbook); + name.push_str(" / "); + name.push_str(worksheet); + Ok(name) +} + +fn insert_text( + metadata: &mut BTreeMap, + key: &str, + value: &str, +) -> Result<(), OriginImportError> { + metadata.insert( + key.to_owned(), + Value::String(copy_text(value, "Origin metadata text")?), + ); + Ok(()) +} + +fn option_text(value: Option<&str>) -> Value { + value.map_or(Value::Null, |value| Value::String(value.to_owned())) +} + +fn usize_value(value: Option) -> Value { + value.map_or(Value::Null, Value::from) +} + +fn diagnostic_code(code: OriginDiagnosticCode) -> &'static str { + match code { + OriginDiagnosticCode::UnsupportedObjectSkipped => "unsupported_object_skipped", + OriginDiagnosticCode::UnsupportedColumnSkipped => "unsupported_column_skipped", + OriginDiagnosticCode::MetadataSkipped => "metadata_skipped", + OriginDiagnosticCode::DecodingWarning => "decoding_warning", + } +} + +fn diagnostic_severity(severity: OriginDiagnosticSeverity) -> &'static str { + match severity { + OriginDiagnosticSeverity::Info => "info", + OriginDiagnosticSeverity::Warning => "warning", + } +} + +fn cell_matches(column_type: OriginColumnType, cell: &OriginCell) -> bool { + matches!( + (column_type, cell), + (_, OriginCell::Null) + | (OriginColumnType::Float, OriginCell::Float(_)) + | (OriginColumnType::Integer, OriginCell::Integer(_)) + | (OriginColumnType::Text, OriginCell::Text(_)) + | ( + OriginColumnType::Mixed, + OriginCell::Float(_) | OriginCell::Integer(_) | OriginCell::Text(_), + ) + ) +} + +fn cell_kind(cell: &OriginCell) -> &'static str { + match cell { + OriginCell::Null => "null", + OriginCell::Float(_) => "a floating-point value", + OriginCell::Integer(_) => "an integer", + OriginCell::Text(_) => "text", + } +} + +fn copy_text(text: &str, resource: &'static str) -> Result { + let mut copy = String::new(); + copy.try_reserve_exact(text.len()) + .map_err(|_| OriginImportError::AllocationFailed { + resource, + requested: text.len(), + })?; + copy.push_str(text); + Ok(copy) +} + +fn try_reserve( + values: &mut Vec, + additional: usize, + resource: &'static str, +) -> Result<(), OriginImportError> { + let requested = checked_mul(additional, size_of::(), resource)?; + values + .try_reserve_exact(additional) + .map_err(|_| OriginImportError::AllocationFailed { + resource, + requested, + }) +} + +fn enforce(resource: &'static str, actual: usize, limit: usize) -> Result<(), OriginImportError> { + if actual > limit { + return Err(OriginImportError::LimitExceeded { + resource, + limit, + actual, + }); + } + Ok(()) +} + +fn checked_add( + left: usize, + right: usize, + resource: &'static str, +) -> Result { + left.checked_add(right) + .ok_or(OriginImportError::ArithmeticOverflow { resource }) +} + +fn checked_mul( + left: usize, + right: usize, + resource: &'static str, +) -> Result { + left.checked_mul(right) + .ok_or(OriginImportError::ArithmeticOverflow { resource }) +} diff --git a/crates/core/src/origin/names.rs b/crates/core/src/origin/names.rs new file mode 100644 index 0000000..aa8ed9c --- /dev/null +++ b/crates/core/src/origin/names.rs @@ -0,0 +1,100 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use plotx_io::origin::{OriginColumn, OriginLimits}; + +use super::{OriginImportError, checked_add, copy_text, enforce, try_reserve}; + +pub(super) fn normalize( + columns: &[OriginColumn], + limits: &OriginLimits, +) -> Result, OriginImportError> { + let mut reserved = BTreeSet::new(); + for column in columns { + if !column.name.trim().is_empty() { + reserved.insert(copy_text(&column.name, "Origin column name")?); + } + } + + let mut names = Vec::new(); + try_reserve(&mut names, columns.len(), "Origin column names")?; + let mut used = BTreeSet::new(); + let mut next_suffix = BTreeMap::new(); + for (index, column) in columns.iter().enumerate() { + let source_name = !column.name.trim().is_empty(); + let base = if source_name { + copy_text(&column.name, "Origin column name")? + } else { + generated_column_name(index, limits)? + }; + enforce("string bytes", base.len(), limits.max_string_bytes)?; + let can_use_base = !used.contains(&base) && (source_name || !reserved.contains(&base)); + let candidate = if can_use_base { + base + } else { + unique_suffixed_name(&base, &reserved, &used, &mut next_suffix, limits)? + }; + if !used.insert(copy_text(&candidate, "Origin column name")?) { + return Err(OriginImportError::InvalidModel { + detail: "column name normalization produced a duplicate".to_owned(), + }); + } + names.push(candidate); + } + Ok(names) +} + +fn unique_suffixed_name( + base: &str, + reserved: &BTreeSet, + used: &BTreeSet, + next_suffix: &mut BTreeMap, + limits: &OriginLimits, +) -> Result { + let mut suffix = next_suffix.get(base).copied().unwrap_or(2); + let candidate = loop { + let candidate = suffixed_name(base, suffix, limits)?; + suffix = checked_add(suffix, 1, "column name suffix")?; + if !reserved.contains(&candidate) && !used.contains(&candidate) { + break candidate; + } + }; + next_suffix.insert(copy_text(base, "Origin column name")?, suffix); + Ok(candidate) +} + +fn generated_column_name(index: usize, limits: &OriginLimits) -> Result { + let number = checked_add(index, 1, "generated column number")?.to_string(); + joined_name("Column ", "", &number, limits) +} + +fn suffixed_name( + base: &str, + suffix: usize, + limits: &OriginLimits, +) -> Result { + joined_name(base, " (", &format!("{suffix})"), limits) +} + +fn joined_name( + left: &str, + separator: &str, + right: &str, + limits: &OriginLimits, +) -> Result { + let length = checked_add( + checked_add(left.len(), separator.len(), "column name")?, + right.len(), + "column name", + )?; + enforce("string bytes", length, limits.max_string_bytes)?; + let mut name = String::new(); + name.try_reserve_exact(length) + .map_err(|_| OriginImportError::AllocationFailed { + resource: "Origin column name", + requested: length, + })?; + name.push_str(left); + name.push_str(separator); + name.push_str(right); + Ok(name) +} diff --git a/crates/core/src/origin/preflight.rs b/crates/core/src/origin/preflight.rs new file mode 100644 index 0000000..4578c65 --- /dev/null +++ b/crates/core/src/origin/preflight.rs @@ -0,0 +1,682 @@ +use std::mem::size_of; + +use plotx_data::{ + ChunkDescriptor, ColumnChunk, ColumnManifest, ColumnSchema, RowId, TableSnapshot, +}; +use plotx_io::origin::{ + OriginByteOrder, OriginCell, OriginColumn, OriginColumnType, OriginDiagnostic, OriginFormat, + OriginHeaderVersion, OriginLimits, OriginMetadataEntry, OriginProfile, OriginProject, + OriginResourceUsage, OriginSupport, OriginWorksheet, +}; + +use super::{ + CHUNK_ROWS, ImportedOriginWorksheet, OriginImportError, PreparedOriginWorksheet, cell_kind, + cell_matches, checked_add, checked_mul, copy_text, enforce, names, try_reserve, +}; + +const ARROW_BLOCK_OVERHEAD: usize = 4_096; +const BTREE_ENTRY_OVERHEAD: usize = 64; +const IPC_OFFSET_BYTES: usize = 4; +const FINGERPRINT_LENGTH_BYTES: usize = 8; +const UUID_TEXT_BYTES: usize = 36; +const JSON_ENTRY_OVERHEAD: usize = 256; +const MIXED_FLOAT_TEXT_MAX: usize = 32; +const MIXED_INTEGER_TEXT_MAX: usize = 20; + +mod model; + +pub(super) struct Preflight { + pub(super) worksheets: Vec, + pub(super) total_owned_bytes: usize, +} + +pub(super) struct WorksheetPreflight { + pub(super) imported_names: Vec, +} + +pub(super) fn validate( + project: &OriginProject, + limits: &OriginLimits, +) -> Result { + validate_probe(project)?; + validate_reported_usage(&project.resource_usage, limits)?; + enforce("workbooks", project.workbooks.len(), limits.max_workbooks)?; + let retained_model = checked_add( + project.resource_usage.input_bytes, + model::owned_lower_bound(project)?, + "retained Origin model", + )?; + let mut estimated_total = project.resource_usage.total_owned_bytes.max(retained_model); + enforce( + "total owned bytes", + estimated_total, + limits.max_total_owned_bytes, + )?; + + let mut text_bytes = 0_usize; + charge_text(&mut text_bytes, &project.probe.raw_version, limits)?; + let mut metadata_records = 0_usize; + validate_entries( + &project.parameters, + &mut text_bytes, + &mut metadata_records, + limits, + )?; + for note in &project.notes { + add_record(&mut metadata_records, limits)?; + charge_text(&mut text_bytes, ¬e.name, limits)?; + charge_text(&mut text_bytes, ¬e.content, limits)?; + } + validate_diagnostics( + &project.diagnostics, + &mut text_bytes, + &mut metadata_records, + limits, + )?; + for summary in &project.unsupported_objects { + add_record(&mut metadata_records, limits)?; + charge_text(&mut text_bytes, &summary.kind, limits)?; + } + + let mut total_columns = 0_usize; + let mut total_cells = 0_usize; + let mut total_worksheets = 0_usize; + let mut retained_metadata_records = checked_add( + project.parameters.len(), + project.notes.len(), + "retained metadata records", + )?; + let mut worksheets = Vec::new(); + for workbook in &project.workbooks { + charge_text(&mut text_bytes, &workbook.name, limits)?; + enforce( + "worksheets per workbook", + workbook.worksheets.len(), + limits.max_worksheets_per_workbook, + )?; + for worksheet in &workbook.worksheets { + total_worksheets = checked_add(total_worksheets, 1, "worksheets")?; + retained_metadata_records = checked_add( + retained_metadata_records, + worksheet.metadata.len(), + "retained metadata records", + )?; + charge_text(&mut text_bytes, &worksheet.name, limits)?; + validate_entries( + &worksheet.metadata, + &mut text_bytes, + &mut metadata_records, + limits, + )?; + total_columns = checked_add(total_columns, worksheet.columns.len(), "columns")?; + enforce("columns", total_columns, limits.max_columns)?; + enforce( + "rows per column", + worksheet.row_count, + limits.max_rows_per_column, + )?; + for column in &worksheet.columns { + validate_column( + column, + worksheet.row_count, + &mut text_bytes, + &mut total_cells, + limits, + )?; + } + if worksheet.row_count > 0 && !worksheet.columns.is_empty() { + let imported_names = names::normalize(&worksheet.columns, limits)?; + estimate_worksheet( + &mut estimated_total, + project, + &workbook.name, + worksheet, + &imported_names, + limits, + )?; + try_reserve(&mut worksheets, 1, "Origin worksheet preflight")?; + worksheets.push(WorksheetPreflight { imported_names }); + } + } + } + enforce( + "decoded text bytes", + text_bytes, + limits.max_decoded_text_bytes, + )?; + enforce( + "metadata records", + metadata_records, + limits.max_metadata_records, + )?; + enforce("cells", total_cells, limits.max_cells)?; + validate_retained_counts( + &project.resource_usage, + project.workbooks.len(), + total_worksheets, + total_columns, + total_cells, + retained_metadata_records, + )?; + enforce( + "total owned bytes", + estimated_total, + limits.max_total_owned_bytes, + )?; + Ok(Preflight { + worksheets, + total_owned_bytes: estimated_total, + }) +} + +fn validate_probe(project: &OriginProject) -> Result<(), OriginImportError> { + const ORIGIN_7_VERSION: OriginHeaderVersion = OriginHeaderVersion { + major: 4, + minor: 2673, + build: 552, + }; + const ORIGIN_9_VERSION: OriginHeaderVersion = OriginHeaderVersion { + major: 4, + minor: 3268, + build: 195, + }; + let exact_profile = match project.probe.profile { + Some(OriginProfile::Origin7V552) => { + project.probe.raw_version == "4.2673 552" && project.probe.version == ORIGIN_7_VERSION + } + Some(OriginProfile::Origin9V951) => { + project.probe.raw_version == "4.3268 195 W64" + && project.probe.version == ORIGIN_9_VERSION + } + None => false, + }; + if project.probe.format != OriginFormat::Opj + || project.probe.support != OriginSupport::Supported + || project.probe.byte_order != OriginByteOrder::LittleEndian + || !exact_profile + { + return Err(OriginImportError::InvalidModel { + detail: + "only the exact verified little-endian Origin7V552 or Origin9V951 OPJ profile can be converted" + .to_owned(), + }); + } + Ok(()) +} + +fn validate_reported_usage( + usage: &OriginResourceUsage, + limits: &OriginLimits, +) -> Result<(), OriginImportError> { + enforce("input bytes", usage.input_bytes, limits.max_input_bytes)?; + enforce("parser bytes", usage.parser_bytes, limits.max_parser_bytes)?; + enforce( + "decoded text bytes", + usage.decoded_text_bytes, + limits.max_decoded_text_bytes, + )?; + enforce( + "total owned bytes", + usage.total_owned_bytes, + limits.max_total_owned_bytes, + )?; + enforce("workbooks", usage.workbooks, limits.max_workbooks)?; + enforce("columns", usage.columns, limits.max_columns)?; + enforce("cells", usage.cells, limits.max_cells)?; + enforce( + "metadata records", + usage.metadata_records, + limits.max_metadata_records, + )?; + let parser_minimum = checked_add(usage.input_bytes, usage.parser_bytes, "parser ownership")?; + if usage.total_owned_bytes < parser_minimum { + return Err(OriginImportError::InvalidModel { + detail: "resource usage total is smaller than input plus parser ownership".to_owned(), + }); + } + if usage.decoded_text_bytes > usage.parser_bytes { + return Err(OriginImportError::InvalidModel { + detail: "decoded text accounting exceeds parser ownership".to_owned(), + }); + } + Ok(()) +} + +fn validate_retained_counts( + usage: &OriginResourceUsage, + workbooks: usize, + worksheets: usize, + columns: usize, + cells: usize, + metadata_records: usize, +) -> Result<(), OriginImportError> { + for (resource, reported, retained, exact) in [ + ("workbooks", usage.workbooks, workbooks, true), + ("worksheets", usage.worksheets, worksheets, true), + ("columns", usage.columns, columns, false), + ("cells", usage.cells, cells, false), + ( + "metadata records", + usage.metadata_records, + metadata_records, + false, + ), + ] { + if (exact && reported != retained) || (!exact && reported < retained) { + let relation = if exact { "equal" } else { "cover" }; + return Err(OriginImportError::InvalidModel { + detail: format!( + "reported {resource} count {reported} does not {relation} the retained count {retained}" + ), + }); + } + } + Ok(()) +} + +fn validate_entries( + entries: &[OriginMetadataEntry], + text_bytes: &mut usize, + records: &mut usize, + limits: &OriginLimits, +) -> Result<(), OriginImportError> { + for entry in entries { + add_record(records, limits)?; + charge_text(text_bytes, &entry.key, limits)?; + charge_text(text_bytes, &entry.value, limits)?; + } + Ok(()) +} + +fn validate_diagnostics( + diagnostics: &[OriginDiagnostic], + text_bytes: &mut usize, + records: &mut usize, + limits: &OriginLimits, +) -> Result<(), OriginImportError> { + for diagnostic in diagnostics { + add_record(records, limits)?; + charge_text(text_bytes, &diagnostic.message, limits)?; + if let Some(location) = &diagnostic.location { + for value in [ + location.workbook.as_deref(), + location.worksheet.as_deref(), + location.column.as_deref(), + ] + .into_iter() + .flatten() + { + charge_text(text_bytes, value, limits)?; + } + } + } + Ok(()) +} + +fn validate_column( + column: &OriginColumn, + row_count: usize, + text_bytes: &mut usize, + total_cells: &mut usize, + limits: &OriginLimits, +) -> Result<(), OriginImportError> { + for text in [ + Some(column.name.as_str()), + column.long_name.as_deref(), + column.role.as_deref(), + column.units.as_deref(), + column.comments.as_deref(), + ] + .into_iter() + .flatten() + { + charge_text(text_bytes, text, limits)?; + } + enforce( + "rows per column", + column.cells.len(), + limits.max_rows_per_column, + )?; + if column.cells.len() > row_count { + return Err(OriginImportError::InvalidModel { + detail: format!( + "column {:?} has {} cells but worksheet row_count is {}", + column.name, + column.cells.len(), + row_count + ), + }); + } + *total_cells = checked_add(*total_cells, column.cells.len(), "cells")?; + enforce("cells", *total_cells, limits.max_cells)?; + for (row, cell) in column.cells.iter().enumerate() { + if let OriginCell::Text(text) = cell { + charge_text(text_bytes, text, limits)?; + } + if !cell_matches(column.column_type, cell) { + return Err(OriginImportError::InvalidCellType { + column: copy_text(&column.name, "Origin column name")?, + row, + expected: column.column_type, + actual: cell_kind(cell), + }); + } + } + Ok(()) +} + +fn estimate_worksheet( + total: &mut usize, + project: &OriginProject, + workbook_name: &str, + worksheet: &OriginWorksheet, + imported_names: &[String], + limits: &OriginLimits, +) -> Result<(), OriginImportError> { + let rows = worksheet.row_count; + let columns = worksheet.columns.len(); + let batches = checked_add((rows - 1) / CHUNK_ROWS, 1, "snapshot batches")?; + estimate_add(total, size_of::(), limits)?; + estimate_add(total, size_of::(), limits)?; + estimate_add(total, size_of::(), limits)?; + estimate_add(total, size_of::(), limits)?; + estimate_mul(total, columns, size_of::(), limits)?; + estimate_mul(total, columns, size_of::(), limits)?; + estimate_mul(total, batches, size_of::(), limits)?; + estimate_mul( + total, + checked_mul(columns, batches, "column descriptors")?, + size_of::(), + limits, + )?; + estimate_mul(total, rows, size_of::(), limits)?; + // SnapshotBuilder validates every batch in a BTreeSet before formatting + // UUID row ids as strings and sending them through the UTF-8 codec path. + estimate_mul( + total, + rows, + checked_add(size_of::(), BTREE_ENTRY_OVERHEAD, "row identity set")?, + limits, + )?; + let row_identity_text = checked_mul(rows, UUID_TEXT_BYTES, "row identity text")?; + estimate_utf8_conversion(total, rows, row_identity_text, row_identity_text, limits)?; + estimate_mul( + total, + checked_mul(columns, batches, "column chunks")?, + size_of::(), + limits, + )?; + estimate_mul(total, columns, size_of::(), limits)?; + let imported_name_bytes = imported_names.iter().try_fold(0_usize, |total, name| { + checked_add(total, name.len(), "imported column names") + })?; + // Normalization retains the final names and temporarily owns reserved and + // used-name set copies while protecting genuine source names. + estimate_mul(total, imported_name_bytes, 3, limits)?; + estimate_mul(total, columns, BTREE_ENTRY_OVERHEAD * 2, limits)?; + + for column in &worksheet.columns { + match column.column_type { + OriginColumnType::Float | OriginColumnType::Integer => { + estimate_numeric_conversion(total, rows, limits)?; + } + OriginColumnType::Text | OriginColumnType::Mixed => { + let text = estimated_column_text(column)?; + estimate_utf8_conversion( + total, + rows, + text.encoded_bytes, + text.new_target_bytes, + limits, + )?; + } + } + estimate_add( + total, + checked_add( + column.name.len(), + checked_mul(5, JSON_ENTRY_OVERHEAD, "column metadata")?, + "column metadata", + )?, + limits, + )?; + } + estimate_mul( + total, + source_metadata_text_bytes(project, workbook_name, worksheet)?, + 4, + limits, + )?; + let records = checked_add( + checked_add( + project.parameters.len(), + project.notes.len(), + "metadata estimate", + )?, + checked_add( + project.diagnostics.len(), + project.unsupported_objects.len(), + "metadata estimate", + )?, + "metadata estimate", + )?; + let records = checked_add(records, worksheet.metadata.len(), "metadata estimate")?; + let records = checked_add(records, worksheet.columns.len(), "metadata estimate")?; + estimate_mul( + total, + checked_add(records, 12, "metadata estimate")?, + JSON_ENTRY_OVERHEAD, + limits, + )?; + estimate_mul( + total, + checked_mul( + batches, + checked_add(columns, 1, "encoded blocks")?, + "encoded blocks", + )?, + ARROW_BLOCK_OVERHEAD, + limits, + ) +} + +fn estimate_numeric_conversion( + total: &mut usize, + rows: usize, + limits: &OriginLimits, +) -> Result<(), OriginImportError> { + let validity = bitmap_bytes(rows); + // Conversion target plus the byte-per-row validity input. + estimate_mul(total, rows, size_of::(), limits)?; + estimate_add(total, rows, limits)?; + estimate_add(total, validity, limits)?; + // The Arrow codec first materializes Vec>, then Arrow value and + // validity buffers. IPC retains another value representation in the block + // store, and logical_fingerprint builds a separate canonical byte vector. + estimate_mul(total, rows, size_of::>(), limits)?; + estimate_numeric_buffers(total, rows, validity, limits)?; + estimate_numeric_buffers(total, rows, validity, limits)?; + estimate_numeric_buffers(total, rows, validity, limits) +} + +fn estimate_numeric_buffers( + total: &mut usize, + rows: usize, + validity: usize, + limits: &OriginLimits, +) -> Result<(), OriginImportError> { + estimate_mul(total, rows, size_of::(), limits)?; + estimate_add(total, validity, limits) +} + +fn estimate_utf8_conversion( + total: &mut usize, + rows: usize, + text_bytes: usize, + new_target_text_bytes: usize, + limits: &OriginLimits, +) -> Result<(), OriginImportError> { + let validity = bitmap_bytes(rows); + // Conversion target plus the byte-per-row validity input and the retained + // ColumnChunk bitmap. + estimate_mul(total, rows, size_of::(), limits)?; + estimate_add(total, new_target_text_bytes, limits)?; + estimate_add(total, rows, limits)?; + estimate_add(total, validity, limits)?; + // StringArray::from first collects borrowed values into Vec>. + estimate_mul(total, rows, size_of::>(), limits)?; + // Arrow, retained IPC, and the canonical fingerprint each own a separate + // row-scaled representation. The fingerprint uses u64 lengths rather than + // Arrow's i32 offsets. + estimate_utf8_buffers(total, rows, text_bytes, validity, IPC_OFFSET_BYTES, limits)?; + estimate_utf8_buffers(total, rows, text_bytes, validity, IPC_OFFSET_BYTES, limits)?; + estimate_utf8_buffers( + total, + rows, + text_bytes, + validity, + FINGERPRINT_LENGTH_BYTES, + limits, + ) +} + +fn estimate_utf8_buffers( + total: &mut usize, + rows: usize, + text_bytes: usize, + validity: usize, + offset_width: usize, + limits: &OriginLimits, +) -> Result<(), OriginImportError> { + estimate_mul( + total, + checked_add(rows, 1, "UTF-8 offsets")?, + offset_width, + limits, + )?; + estimate_add(total, text_bytes, limits)?; + estimate_add(total, validity, limits) +} + +struct EstimatedColumnText { + encoded_bytes: usize, + new_target_bytes: usize, +} + +fn estimated_column_text(column: &OriginColumn) -> Result { + let mut encoded_bytes = 0_usize; + let mut new_target_bytes = 0_usize; + for cell in &column.cells { + let (encoded, newly_allocated) = match cell { + OriginCell::Text(value) => (value.len(), 0), + OriginCell::Float(_) => (MIXED_FLOAT_TEXT_MAX, MIXED_FLOAT_TEXT_MAX), + OriginCell::Integer(_) => (MIXED_INTEGER_TEXT_MAX, MIXED_INTEGER_TEXT_MAX), + OriginCell::Null => (0, 0), + }; + encoded_bytes = checked_add(encoded_bytes, encoded, "UTF-8 cell data")?; + new_target_bytes = checked_add( + new_target_bytes, + newly_allocated, + "converted UTF-8 cell data", + )?; + } + Ok(EstimatedColumnText { + encoded_bytes, + new_target_bytes, + }) +} + +fn source_metadata_text_bytes( + project: &OriginProject, + workbook_name: &str, + worksheet: &OriginWorksheet, +) -> Result { + let mut bytes = checked_add( + project.probe.raw_version.len(), + workbook_name.len(), + "metadata", + )?; + bytes = checked_add(bytes, worksheet.name.len(), "metadata")?; + for entry in project.parameters.iter().chain(&worksheet.metadata) { + bytes = checked_add(bytes, entry.key.len(), "metadata")?; + bytes = checked_add(bytes, entry.value.len(), "metadata")?; + } + for note in &project.notes { + bytes = checked_add(bytes, note.name.len(), "metadata")?; + bytes = checked_add(bytes, note.content.len(), "metadata")?; + } + for diagnostic in &project.diagnostics { + bytes = checked_add(bytes, diagnostic.message.len(), "metadata")?; + if let Some(location) = &diagnostic.location { + for text in [ + location.workbook.as_deref(), + location.worksheet.as_deref(), + location.column.as_deref(), + ] + .into_iter() + .flatten() + { + bytes = checked_add(bytes, text.len(), "metadata")?; + } + } + } + for summary in &project.unsupported_objects { + bytes = checked_add(bytes, summary.kind.len(), "metadata")?; + } + for column in &worksheet.columns { + for text in [ + Some(column.name.as_str()), + column.long_name.as_deref(), + column.role.as_deref(), + column.units.as_deref(), + column.comments.as_deref(), + ] + .into_iter() + .flatten() + { + bytes = checked_add(bytes, text.len(), "metadata")?; + } + } + Ok(bytes) +} + +fn add_record(records: &mut usize, limits: &OriginLimits) -> Result<(), OriginImportError> { + *records = checked_add(*records, 1, "metadata records")?; + enforce("metadata records", *records, limits.max_metadata_records) +} + +fn charge_text( + total: &mut usize, + text: &str, + limits: &OriginLimits, +) -> Result<(), OriginImportError> { + enforce("string bytes", text.len(), limits.max_string_bytes)?; + *total = checked_add(*total, text.len(), "decoded text bytes")?; + enforce("decoded text bytes", *total, limits.max_decoded_text_bytes) +} + +fn estimate_add( + total: &mut usize, + bytes: usize, + limits: &OriginLimits, +) -> Result<(), OriginImportError> { + *total = checked_add(*total, bytes, "total owned bytes")?; + enforce("total owned bytes", *total, limits.max_total_owned_bytes) +} + +fn estimate_mul( + total: &mut usize, + count: usize, + width: usize, + limits: &OriginLimits, +) -> Result<(), OriginImportError> { + estimate_add( + total, + checked_mul(count, width, "total owned bytes")?, + limits, + ) +} + +fn bitmap_bytes(rows: usize) -> usize { + rows / 8 + usize::from(!rows.is_multiple_of(8)) +} diff --git a/crates/core/src/origin/preflight/model.rs b/crates/core/src/origin/preflight/model.rs new file mode 100644 index 0000000..b02bf55 --- /dev/null +++ b/crates/core/src/origin/preflight/model.rs @@ -0,0 +1,109 @@ +use std::mem::size_of; + +use plotx_io::origin::{ + OriginCell, OriginColumn, OriginDiagnostic, OriginMetadataEntry, OriginNote, OriginProject, + OriginUnsupportedObjectSummary, OriginWorkbook, OriginWorksheet, +}; + +use super::super::{OriginImportError, checked_add, checked_mul}; + +pub(super) fn owned_lower_bound(project: &OriginProject) -> Result { + let mut bytes = size_of::(); + add_text_storage(&mut bytes, &project.probe.raw_version)?; + add_vec_storage::(&mut bytes, project.parameters.len())?; + for entry in &project.parameters { + add_entry_storage(&mut bytes, entry)?; + } + add_vec_storage::(&mut bytes, project.notes.len())?; + for note in &project.notes { + add_text_storage(&mut bytes, ¬e.name)?; + add_text_storage(&mut bytes, ¬e.content)?; + } + add_vec_storage::(&mut bytes, project.workbooks.len())?; + for workbook in &project.workbooks { + add_text_storage(&mut bytes, &workbook.name)?; + add_vec_storage::(&mut bytes, workbook.worksheets.len())?; + for worksheet in &workbook.worksheets { + add_worksheet_storage(&mut bytes, worksheet)?; + } + } + add_vec_storage::(&mut bytes, project.diagnostics.len())?; + for diagnostic in &project.diagnostics { + add_text_storage(&mut bytes, &diagnostic.message)?; + if let Some(location) = &diagnostic.location { + for text in [ + location.workbook.as_ref(), + location.worksheet.as_ref(), + location.column.as_ref(), + ] + .into_iter() + .flatten() + { + add_text_storage(&mut bytes, text)?; + } + } + } + add_vec_storage::( + &mut bytes, + project.unsupported_objects.len(), + )?; + for summary in &project.unsupported_objects { + add_text_storage(&mut bytes, &summary.kind)?; + } + Ok(bytes) +} + +fn add_worksheet_storage( + bytes: &mut usize, + worksheet: &OriginWorksheet, +) -> Result<(), OriginImportError> { + add_text_storage(bytes, &worksheet.name)?; + add_vec_storage::(bytes, worksheet.metadata.len())?; + for entry in &worksheet.metadata { + add_entry_storage(bytes, entry)?; + } + add_vec_storage::(bytes, worksheet.columns.len())?; + for column in &worksheet.columns { + for text in [ + Some(&column.name), + column.long_name.as_ref(), + column.role.as_ref(), + column.units.as_ref(), + column.comments.as_ref(), + ] + .into_iter() + .flatten() + { + add_text_storage(bytes, text)?; + } + add_vec_storage::(bytes, column.cells.len())?; + for cell in &column.cells { + if let OriginCell::Text(text) = cell { + add_text_storage(bytes, text)?; + } + } + } + Ok(()) +} + +fn add_entry_storage( + bytes: &mut usize, + entry: &OriginMetadataEntry, +) -> Result<(), OriginImportError> { + add_text_storage(bytes, &entry.key)?; + add_text_storage(bytes, &entry.value) +} + +fn add_text_storage(bytes: &mut usize, text: &str) -> Result<(), OriginImportError> { + *bytes = checked_add(*bytes, text.len(), "retained Origin model")?; + Ok(()) +} + +fn add_vec_storage(bytes: &mut usize, len: usize) -> Result<(), OriginImportError> { + *bytes = checked_add( + *bytes, + checked_mul(len, size_of::(), "retained Origin model")?, + "retained Origin model", + )?; + Ok(()) +} diff --git a/crates/core/src/origin_hardening_tests.rs b/crates/core/src/origin_hardening_tests.rs new file mode 100644 index 0000000..c86d141 --- /dev/null +++ b/crates/core/src/origin_hardening_tests.rs @@ -0,0 +1,286 @@ +use plotx_data::{CodecRegistry, MemoryBlockStore, ScalarValue, SnapshotReader}; +use plotx_io::origin::{ + OriginByteOrder, OriginCell, OriginColumn, OriginColumnType, OriginFormat, OriginHeaderVersion, + OriginLimits, OriginProbe, OriginProfile, OriginProject, OriginResourceUsage, OriginSupport, + OriginWorkbook, OriginWorksheet, +}; + +use crate::origin::{OriginImportError, import_origin_project}; + +const ORIGINAL_NAME_KEY: &str = "space.nmrtist.plotx.import.origin.original_name"; + +fn project(worksheets: Vec) -> OriginProject { + let mut project = OriginProject { + probe: OriginProbe { + format: OriginFormat::Opj, + raw_version: "4.2673 552".to_owned(), + version: OriginHeaderVersion { + major: 4, + minor: 2673, + build: 552, + }, + byte_order: OriginByteOrder::LittleEndian, + profile: Some(OriginProfile::Origin7V552), + support: OriginSupport::Supported, + }, + parameters: Vec::new(), + notes: Vec::new(), + workbooks: vec![OriginWorkbook { + name: "Book".to_owned(), + worksheets, + }], + diagnostics: Vec::new(), + unsupported_objects: Vec::new(), + resource_usage: OriginResourceUsage::default(), + }; + sync_resource_counts(&mut project); + project +} + +fn sync_resource_counts(project: &mut OriginProject) { + project.resource_usage.workbooks = project.workbooks.len(); + project.resource_usage.worksheets = project + .workbooks + .iter() + .map(|workbook| workbook.worksheets.len()) + .sum(); + project.resource_usage.columns = project + .workbooks + .iter() + .flat_map(|workbook| &workbook.worksheets) + .map(|worksheet| worksheet.columns.len()) + .sum(); + project.resource_usage.cells = project + .workbooks + .iter() + .flat_map(|workbook| &workbook.worksheets) + .flat_map(|worksheet| &worksheet.columns) + .map(|column| column.cells.len()) + .sum(); +} + +fn worksheet(name: &str, columns: Vec) -> OriginWorksheet { + OriginWorksheet { + name: name.to_owned(), + columns, + row_count: 1, + metadata: Vec::new(), + } +} + +fn columns(names: &[&str]) -> Vec { + names + .iter() + .map(|name| OriginColumn { + name: (*name).to_owned(), + long_name: None, + role: None, + units: None, + comments: None, + column_type: OriginColumnType::Integer, + cells: vec![OriginCell::Integer(1)], + }) + .collect() +} + +fn imported_names(names: &[&str]) -> Vec { + let imported = import_origin_project( + project(vec![worksheet("Sheet", columns(names))]), + &MemoryBlockStore::default(), + &CodecRegistry::with_arrow_ipc(), + OriginLimits::default(), + ) + .expect("test project should convert"); + imported[0] + .snapshot + .schema + .columns + .iter() + .map(|column| column.name.clone()) + .collect() +} + +#[test] +fn converts_the_exact_verified_origin_9_profile() { + let mut project = project(vec![worksheet("Sheet", columns(&["value"]))]); + project.probe = OriginProbe { + format: OriginFormat::Opj, + raw_version: "4.3268 195 W64".to_owned(), + version: OriginHeaderVersion { + major: 4, + minor: 3268, + build: 195, + }, + byte_order: OriginByteOrder::LittleEndian, + profile: Some(OriginProfile::Origin9V951), + support: OriginSupport::Supported, + }; + + let imported = import_origin_project( + project, + &MemoryBlockStore::default(), + &CodecRegistry::with_arrow_ipc(), + OriginLimits::default(), + ) + .expect("the verified Origin 9.51 profile should convert"); + + assert_eq!(imported[0].snapshot.schema.columns[0].name, "value"); +} + +#[test] +fn generated_duplicate_names_do_not_steal_later_source_names() { + assert_eq!( + imported_names(&["name", "name", "name (2)"]), + ["name", "name (3)", "name (2)"] + ); +} + +#[test] +fn generated_blank_names_do_not_steal_real_position_names() { + assert_eq!( + imported_names(&["", "Column 1"]), + ["Column 1 (2)", "Column 1"] + ); + assert_eq!(imported_names(&["Column 1", ""]), ["Column 1", "Column 2"]); +} + +#[test] +fn every_changed_name_retains_its_source_name() { + let imported = import_origin_project( + project(vec![worksheet( + "Sheet", + columns(&["name", "name", "name (2)", ""]), + )]), + &MemoryBlockStore::default(), + &CodecRegistry::with_arrow_ipc(), + OriginLimits::default(), + ) + .expect("test project should convert"); + let schemas = &imported[0].snapshot.schema.columns; + + assert_eq!(schemas[1].metadata[ORIGINAL_NAME_KEY], "name"); + assert_eq!(schemas[3].metadata[ORIGINAL_NAME_KEY], ""); + assert!(!schemas[0].metadata.contains_key(ORIGINAL_NAME_KEY)); + assert!(!schemas[2].metadata.contains_key(ORIGINAL_NAME_KEY)); +} + +#[test] +fn prepares_every_candidate_before_writing_any_blocks() { + let project = project(vec![ + worksheet("First", columns(&["x"])), + worksheet("Second", columns(&["1234567890123456", "1234567890123456"])), + ]); + let store = MemoryBlockStore::default(); + let limits = OriginLimits { + max_string_bytes: 16, + ..OriginLimits::default() + }; + + let error = import_origin_project(project, &store, &CodecRegistry::with_arrow_ipc(), limits) + .expect_err("all candidate names must be prepared before block writes"); + + assert!(matches!( + error, + OriginImportError::LimitExceeded { + resource: "string bytes", + .. + } + )); + assert_eq!(store.block_count(), 0); +} + +#[test] +fn pads_short_columns_across_the_first_batch_and_into_the_second() { + const ROWS: usize = 65_537; + let mut sheet = worksheet( + "Sheet", + vec![ + OriginColumn { + name: "anchor".to_owned(), + long_name: None, + role: None, + units: None, + comments: None, + column_type: OriginColumnType::Integer, + cells: (0..ROWS) + .map(|value| OriginCell::Integer(value as i64)) + .collect(), + }, + OriginColumn { + name: "short".to_owned(), + long_name: None, + role: None, + units: None, + comments: None, + column_type: OriginColumnType::Integer, + cells: vec![OriginCell::Integer(10), OriginCell::Integer(20)], + }, + ], + ); + sheet.row_count = ROWS; + let store = MemoryBlockStore::default(); + let codecs = CodecRegistry::with_arrow_ipc(); + let imported = import_origin_project( + project(vec![sheet]), + &store, + &codecs, + OriginLimits::default(), + ) + .expect("short columns should be padded with nulls"); + let reader = SnapshotReader::new(&imported[0].snapshot, &store, &codecs).unwrap(); + let first = reader.read_batch(0, &[]).unwrap(); + let second = reader.read_batch(1, &[]).unwrap(); + + assert_eq!(first.columns[1].1.value(1), Some(ScalarValue::Int64(20))); + assert_eq!(first.columns[1].1.value(2), Some(ScalarValue::Null)); + assert_eq!(first.columns[1].1.value(65_535), Some(ScalarValue::Null)); + assert_eq!(second.columns[1].1.value(0), Some(ScalarValue::Null)); +} + +#[test] +fn pads_a_short_column_that_ends_exactly_on_a_batch_boundary() { + const ROWS: usize = 65_537; + let mut sheet = worksheet( + "Sheet", + vec![ + OriginColumn { + name: "anchor".to_owned(), + long_name: None, + role: None, + units: None, + comments: None, + column_type: OriginColumnType::Integer, + cells: (0..ROWS) + .map(|value| OriginCell::Integer(value as i64)) + .collect(), + }, + OriginColumn { + name: "short".to_owned(), + long_name: None, + role: None, + units: None, + comments: None, + column_type: OriginColumnType::Integer, + cells: (0..65_536) + .map(|value| OriginCell::Integer(value as i64)) + .collect(), + }, + ], + ); + sheet.row_count = ROWS; + let store = MemoryBlockStore::default(); + let codecs = CodecRegistry::with_arrow_ipc(); + let imported = import_origin_project( + project(vec![sheet]), + &store, + &codecs, + OriginLimits::default(), + ) + .expect("a batch-boundary short column should be padded with nulls"); + let second = SnapshotReader::new(&imported[0].snapshot, &store, &codecs) + .unwrap() + .read_batch(1, &[]) + .unwrap(); + + assert_eq!(second.columns[1].1.value(0), Some(ScalarValue::Null)); +} diff --git a/crates/core/src/origin_tests.rs b/crates/core/src/origin_tests.rs new file mode 100644 index 0000000..a992508 --- /dev/null +++ b/crates/core/src/origin_tests.rs @@ -0,0 +1,772 @@ +use std::collections::BTreeMap; + +use plotx_data::{CodecRegistry, LogicalType, MemoryBlockStore, ScalarValue, SnapshotReader}; +use plotx_io::origin::{ + OriginByteOrder, OriginCell, OriginColumn, OriginColumnType, OriginDiagnostic, + OriginDiagnosticCode, OriginDiagnosticSeverity, OriginFormat, OriginHeaderVersion, + OriginLimits, OriginMetadataEntry, OriginNote, OriginObjectLocation, OriginProbe, + OriginProfile, OriginProject, OriginResourceUsage, OriginSupport, + OriginUnsupportedObjectSummary, OriginWorkbook, OriginWorksheet, +}; + +use crate::origin::{ORIGIN_IMPORT_OPERATION, OriginImportError, import_origin_project}; + +fn probe() -> OriginProbe { + OriginProbe { + format: OriginFormat::Opj, + raw_version: "4.2673 552".to_owned(), + version: OriginHeaderVersion { + major: 4, + minor: 2673, + build: 552, + }, + byte_order: OriginByteOrder::LittleEndian, + profile: Some(OriginProfile::Origin7V552), + support: OriginSupport::Supported, + } +} + +fn column(name: &str, column_type: OriginColumnType, cells: Vec) -> OriginColumn { + OriginColumn { + name: name.to_owned(), + long_name: None, + role: None, + units: None, + comments: None, + column_type, + cells, + } +} + +fn worksheet(name: &str, row_count: usize, columns: Vec) -> OriginWorksheet { + OriginWorksheet { + name: name.to_owned(), + columns, + row_count, + metadata: Vec::new(), + } +} + +fn project(workbooks: Vec) -> OriginProject { + let mut project = OriginProject { + probe: probe(), + parameters: Vec::new(), + notes: Vec::new(), + workbooks, + diagnostics: Vec::new(), + unsupported_objects: Vec::new(), + resource_usage: OriginResourceUsage::default(), + }; + sync_resource_counts(&mut project); + project +} + +fn sync_resource_counts(project: &mut OriginProject) { + project.resource_usage.workbooks = project.workbooks.len(); + project.resource_usage.worksheets = project + .workbooks + .iter() + .map(|workbook| workbook.worksheets.len()) + .sum(); + project.resource_usage.columns = project + .workbooks + .iter() + .flat_map(|workbook| &workbook.worksheets) + .map(|worksheet| worksheet.columns.len()) + .sum(); + project.resource_usage.cells = project + .workbooks + .iter() + .flat_map(|workbook| &workbook.worksheets) + .flat_map(|worksheet| &worksheet.columns) + .map(|column| column.cells.len()) + .sum(); + project.resource_usage.metadata_records = project.parameters.len() + + project.notes.len() + + project + .workbooks + .iter() + .flat_map(|workbook| &workbook.worksheets) + .map(|worksheet| worksheet.metadata.len()) + .sum::(); +} + +fn workbook(name: &str, worksheets: Vec) -> OriginWorkbook { + OriginWorkbook { + name: name.to_owned(), + worksheets, + } +} + +fn imported( + mut project: OriginProject, +) -> ( + Vec, + MemoryBlockStore, + CodecRegistry, +) { + sync_resource_counts(&mut project); + let store = MemoryBlockStore::default(); + let codecs = CodecRegistry::with_arrow_ipc(); + let imported = import_origin_project(project, &store, &codecs, OriginLimits::default()) + .expect("test project should convert"); + (imported, store, codecs) +} + +#[test] +fn exposes_the_stable_origin_import_operation() { + assert_eq!(ORIGIN_IMPORT_OPERATION, "plotx.import.origin.v1"); +} + +#[test] +fn converts_supported_types_and_pads_short_columns_with_nulls() { + let columns = vec![ + column( + "double", + OriginColumnType::Float, + vec![OriginCell::Float(1.25), OriginCell::Null], + ), + column( + "float", + OriginColumnType::Float, + vec![OriginCell::Float(f32::from_bits(0x43ac_cccd) as f64)], + ), + column( + "integer", + OriginColumnType::Integer, + vec![OriginCell::Integer(-1000), OriginCell::Integer(34)], + ), + column( + "text", + OriginColumnType::Text, + vec![OriginCell::Text("alpha".to_owned()), OriginCell::Null], + ), + ]; + let (imported, store, codecs) = imported(project(vec![workbook( + "Book1", + vec![worksheet("Sheet1", 2, columns)], + )])); + + assert_eq!(imported.len(), 1); + assert_eq!(imported[0].name, "Book1 / Sheet1"); + assert_eq!(imported[0].snapshot.row_count, 2); + assert_eq!( + imported[0] + .snapshot + .schema + .columns + .iter() + .map(|column| column.logical_type.clone()) + .collect::>(), + vec![ + LogicalType::Float64, + LogicalType::Float64, + LogicalType::Int64, + LogicalType::Utf8, + ] + ); + assert!( + imported[0].snapshot.schema.columns[3].unit.is_none(), + "text columns must never acquire a numeric unit" + ); + + let batch = SnapshotReader::new(&imported[0].snapshot, &store, &codecs) + .unwrap() + .read_batch(0, &[]) + .unwrap(); + assert_eq!( + batch.columns[0].1.value(0), + Some(ScalarValue::Float64(1.25)) + ); + assert_eq!(batch.columns[0].1.value(1), Some(ScalarValue::Null)); + assert_eq!( + batch.columns[1].1.value(0), + Some(ScalarValue::Float64(f32::from_bits(0x43ac_cccd) as f64)) + ); + assert_eq!(batch.columns[1].1.value(1), Some(ScalarValue::Null)); + assert_eq!(batch.columns[2].1.value(0), Some(ScalarValue::Int64(-1000))); + assert_eq!( + batch.columns[3].1.value(0), + Some(ScalarValue::Utf8("alpha".to_owned())) + ); + assert_eq!(batch.columns[3].1.value(1), Some(ScalarValue::Null)); +} + +#[test] +fn converts_mixed_cells_without_dropping_numbers() { + let mixed_number = "3.14".parse::().unwrap(); + let mixed = column( + "mixed", + OriginColumnType::Mixed, + vec![ + OriginCell::Text("text".to_owned()), + OriginCell::Float(mixed_number), + OriginCell::Integer(-7), + OriginCell::Null, + ], + ); + let (imported, store, codecs) = imported(project(vec![workbook( + "Book", + vec![worksheet("Sheet", 4, vec![mixed])], + )])); + + assert_eq!( + imported[0].snapshot.schema.columns[0].logical_type, + LogicalType::Utf8 + ); + let batch = SnapshotReader::new(&imported[0].snapshot, &store, &codecs) + .unwrap() + .read_batch(0, &[]) + .unwrap(); + assert_eq!( + (0..4) + .map(|row| batch.columns[0].1.value(row)) + .collect::>(), + vec![ + Some(ScalarValue::Utf8("text".to_owned())), + Some(ScalarValue::Utf8("3.14".to_owned())), + Some(ScalarValue::Utf8("-7".to_owned())), + Some(ScalarValue::Null), + ] + ); +} + +#[test] +fn generates_empty_names_and_disambiguates_exact_duplicates_case_sensitively() { + let names = ["", "", "name", "name", "name", "Name"]; + let columns = names + .into_iter() + .map(|name| { + column( + name, + OriginColumnType::Integer, + vec![OriginCell::Integer(1)], + ) + }) + .collect(); + let (imported, _, _) = imported(project(vec![workbook( + "Book", + vec![worksheet("Sheet", 1, columns)], + )])); + + assert_eq!( + imported[0] + .snapshot + .schema + .columns + .iter() + .map(|column| column.name.as_str()) + .collect::>(), + [ + "Column 1", "Column 2", "name", "name (2)", "name (3)", "Name" + ] + ); + let source_columns = imported[0].source_metadata["space.nmrtist.plotx.import.origin.columns"] + .as_array() + .unwrap(); + assert_eq!(source_columns[0]["source_name"], ""); + assert_eq!(source_columns[0]["imported_name"], "Column 1"); + assert_eq!(source_columns[3]["source_name"], "name"); + assert_eq!(source_columns[3]["imported_name"], "name (2)"); + assert_eq!( + imported[0].snapshot.schema.columns[3].metadata["space.nmrtist.plotx.import.origin.original_name"], + "name" + ); +} + +#[test] +fn creates_one_candidate_for_each_nonempty_worksheet() { + let value = || column("x", OriginColumnType::Float, vec![OriginCell::Float(1.0)]); + let project = project(vec![ + workbook( + "Book1", + vec![ + worksheet("Empty", 0, Vec::new()), + worksheet("Data", 1, vec![value()]), + ], + ), + workbook("Book2", vec![worksheet("More", 1, vec![value()])]), + ]); + let (imported, _, _) = imported(project); + + assert_eq!( + imported + .iter() + .map(|item| item.name.as_str()) + .collect::>(), + ["Book1 / Data", "Book2 / More"] + ); + assert_eq!( + imported[0].resource_usage.total_owned_bytes, + imported[1].resource_usage.total_owned_bytes + ); + assert_eq!( + imported[0].source_metadata["space.nmrtist.plotx.import.origin.resource_usage"]["total_owned_bytes"], + imported[1].resource_usage.total_owned_bytes + ); +} + +#[test] +fn preserves_project_column_and_parser_metadata() { + let mut data = column( + "signal", + OriginColumnType::Float, + vec![OriginCell::Float(2.0)], + ); + data.long_name = Some("Detector signal".to_owned()); + data.role = Some("Y".to_owned()); + data.units = Some("mV".to_owned()); + data.comments = Some("calibrated".to_owned()); + let mut sheet = worksheet("Sheet1", 1, vec![data]); + sheet.metadata = vec![OriginMetadataEntry { + key: "layer".to_owned(), + value: "raw".to_owned(), + }]; + let mut project = project(vec![workbook("Book1", vec![sheet])]); + project.parameters = vec![OriginMetadataEntry { + key: "temperature".to_owned(), + value: "298".to_owned(), + }]; + project.notes = vec![OriginNote { + name: "Methods".to_owned(), + content: "Prepared under nitrogen.".to_owned(), + }]; + project.diagnostics = vec![OriginDiagnostic { + code: OriginDiagnosticCode::UnsupportedObjectSkipped, + severity: OriginDiagnosticSeverity::Warning, + location: Some(OriginObjectLocation { + workbook: Some("Book1".to_owned()), + worksheet: Some("Sheet1".to_owned()), + column: None, + byte_offset: Some(42), + }), + message: "PlotX skipped a graph.".to_owned(), + }]; + project.unsupported_objects = vec![OriginUnsupportedObjectSummary { + kind: "graphs".to_owned(), + count: 1, + }]; + project.resource_usage = OriginResourceUsage { + input_bytes: 100, + parser_bytes: 50, + decoded_text_bytes: 20, + total_owned_bytes: 150, + workbooks: 1, + worksheets: 1, + columns: 1, + cells: 1, + metadata_records: 3, + }; + let (imported, _, _) = imported(project); + let item = &imported[0]; + + assert_eq!(item.diagnostics.len(), 1); + assert!(item.resource_usage.total_owned_bytes > 150); + assert_eq!( + item.source_metadata["space.nmrtist.plotx.import.format"], + "opj" + ); + assert_eq!( + item.source_metadata["space.nmrtist.plotx.import.origin.producer_version"], + "4.2673 552" + ); + assert_eq!( + item.source_metadata["space.nmrtist.plotx.import.origin.parameters"][0]["key"], + "temperature" + ); + assert_eq!( + item.source_metadata["space.nmrtist.plotx.import.origin.notes"][0]["content"], + "Prepared under nitrogen." + ); + assert_eq!( + item.source_metadata["space.nmrtist.plotx.import.origin.unsupported_objects"][0]["kind"], + "graphs" + ); + assert_eq!( + item.source_metadata["space.nmrtist.plotx.import.origin.resource_usage"]["total_owned_bytes"], + item.resource_usage.total_owned_bytes + ); + assert_eq!( + item.source_metadata["space.nmrtist.plotx.import.origin.diagnostics"][0]["message"], + "PlotX skipped a graph." + ); + assert_eq!( + item.snapshot.metadata["space.nmrtist.plotx.import.origin.workbook"], + "Book1" + ); + assert_eq!( + item.snapshot.metadata["space.nmrtist.plotx.import.origin.worksheet"], + "Sheet1" + ); + assert_eq!( + item.snapshot.metadata["space.nmrtist.plotx.import.origin.diagnostics"][0]["message"], + "PlotX skipped a graph." + ); + let schema_metadata = &item.snapshot.schema.columns[0].metadata; + assert_eq!( + schema_metadata["space.nmrtist.plotx.import.origin.long_name"], + "Detector signal" + ); + assert_eq!( + schema_metadata["space.nmrtist.plotx.import.origin.role"], + "Y" + ); + assert_eq!( + schema_metadata["space.nmrtist.plotx.import.origin.units"], + "mV" + ); + assert_eq!( + schema_metadata["space.nmrtist.plotx.import.origin.comments"], + "calibrated" + ); +} + +#[test] +fn rejects_empty_projects_and_projects_with_only_empty_worksheets() { + let store = MemoryBlockStore::default(); + let codecs = CodecRegistry::with_arrow_ipc(); + for project in [ + project(Vec::new()), + project(vec![workbook( + "Book", + vec![worksheet("Empty", 0, Vec::new())], + )]), + ] { + assert!(matches!( + import_origin_project(project, &store, &codecs, OriginLimits::default()), + Err(OriginImportError::NoSupportedWorksheet) + )); + } +} + +#[test] +fn rejects_snapshot_capacity_before_writing_any_blocks() { + let mut project = project(vec![workbook( + "Book", + vec![worksheet( + "Sheet", + 1, + vec![column( + "value", + OriginColumnType::Float, + vec![OriginCell::Float(1.0)], + )], + )], + )]); + project.resource_usage.input_bytes = 1; + project.resource_usage.total_owned_bytes = 1; + let limits = OriginLimits { + max_total_owned_bytes: 1, + ..OriginLimits::default() + }; + let store = MemoryBlockStore::default(); + let codecs = CodecRegistry::with_arrow_ipc(); + + let error = import_origin_project(project, &store, &codecs, limits).unwrap_err(); + + assert!(matches!( + error, + OriginImportError::LimitExceeded { + resource: "total owned bytes", + limit: 1, + actual: _, + } + )); + assert_eq!(store.block_count(), 0); +} + +#[test] +fn rejects_nonempty_models_with_zero_reported_resource_usage() { + let mut project = project(vec![workbook( + "Book", + vec![worksheet( + "Sheet", + 1, + vec![column( + "value", + OriginColumnType::Integer, + vec![OriginCell::Integer(1)], + )], + )], + )]); + project.resource_usage = OriginResourceUsage::default(); + let store = MemoryBlockStore::default(); + + let error = import_origin_project( + project, + &store, + &CodecRegistry::with_arrow_ipc(), + OriginLimits::default(), + ) + .expect_err("a nonempty model cannot report zero decoded objects"); + + assert!(matches!(error, OriginImportError::InvalidModel { .. })); + assert_eq!(store.block_count(), 0); +} + +#[test] +fn rejects_resource_counts_smaller_than_the_retained_model() { + let mut project = project(vec![workbook( + "Book", + vec![worksheet( + "Sheet", + 1, + vec![column( + "value", + OriginColumnType::Integer, + vec![OriginCell::Integer(1)], + )], + )], + )]); + project.resource_usage.columns = 0; + project.resource_usage.cells = 0; + let store = MemoryBlockStore::default(); + + let error = import_origin_project( + project, + &store, + &CodecRegistry::with_arrow_ipc(), + OriginLimits::default(), + ) + .expect_err("reported counts must cover retained objects"); + + assert!(matches!(error, OriginImportError::InvalidModel { .. })); + assert_eq!(store.block_count(), 0); +} + +#[test] +fn rejects_supported_profile_claims_with_inconsistent_version_fields() { + let base = project(vec![workbook( + "Book", + vec![worksheet( + "Sheet", + 1, + vec![column( + "value", + OriginColumnType::Integer, + vec![OriginCell::Integer(1)], + )], + )], + )]); + let mut projects = vec![base.clone(), base]; + projects[0].probe.raw_version = "4.2673 553".to_owned(); + projects[1].probe.version.build = 553; + + for project in projects { + let store = MemoryBlockStore::default(); + let error = import_origin_project( + project, + &store, + &CodecRegistry::with_arrow_ipc(), + OriginLimits::default(), + ) + .expect_err("a supported profile must match its exact verified header"); + + assert!(matches!(error, OriginImportError::InvalidModel { .. })); + assert_eq!(store.block_count(), 0); + } +} + +#[test] +fn rejects_declared_cell_type_mismatches_without_panicking() { + let invalid = project(vec![workbook( + "Book", + vec![worksheet( + "Sheet", + 1, + vec![column( + "value", + OriginColumnType::Float, + vec![OriginCell::Text("not a float".to_owned())], + )], + )], + )]); + let store = MemoryBlockStore::default(); + let codecs = CodecRegistry::with_arrow_ipc(); + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + import_origin_project(invalid, &store, &codecs, OriginLimits::default()) + })); + + assert!(result.is_ok()); + assert!(matches!( + result.unwrap(), + Err(OriginImportError::InvalidCellType { + column, + row: 0, + expected: OriginColumnType::Float, + .. + }) if column == "value" + )); +} + +#[test] +fn streams_large_worksheets_in_fixed_size_batches() { + const ROWS: usize = 65_537; + let values = (0..ROWS) + .map(|value| OriginCell::Integer(value as i64)) + .collect(); + let (imported, store, codecs) = imported(project(vec![workbook( + "Book", + vec![worksheet( + "Sheet", + ROWS, + vec![column("index", OriginColumnType::Integer, values)], + )], + )])); + + assert_eq!(imported[0].snapshot.batch_count(), 2); + let second = SnapshotReader::new(&imported[0].snapshot, &store, &codecs) + .unwrap() + .read_batch(1, &[]) + .unwrap(); + assert_eq!(second.row_ids.len(), 1); + assert_eq!( + second.columns[0].1.value(0), + Some(ScalarValue::Int64(65_536)) + ); +} + +#[test] +fn generated_names_skip_existing_generated_and_suffixed_names() { + let names = ["name", "name (2)", "name", "", "Column 4"]; + let columns = names + .into_iter() + .map(|name| { + column( + name, + OriginColumnType::Integer, + vec![OriginCell::Integer(1)], + ) + }) + .collect(); + let (imported, _, _) = imported(project(vec![workbook( + "Book", + vec![worksheet("Sheet", 1, columns)], + )])); + + assert_eq!( + imported[0] + .snapshot + .schema + .columns + .iter() + .map(|column| column.name.as_str()) + .collect::>(), + ["name", "name (2)", "name (3)", "Column 4 (2)", "Column 4"] + ); +} + +#[test] +fn all_blank_column_names_receive_position_based_names() { + let columns = ["", " ", "\t"] + .into_iter() + .map(|name| column(name, OriginColumnType::Text, vec![OriginCell::Null])) + .collect(); + let (imported, _, _) = imported(project(vec![workbook( + "Book", + vec![worksheet("Sheet", 1, columns)], + )])); + + assert_eq!( + imported[0] + .snapshot + .schema + .columns + .iter() + .map(|column| column.name.as_str()) + .collect::>(), + ["Column 1", "Column 2", "Column 3"] + ); +} + +#[test] +fn source_metadata_is_a_direct_table_import_source_map() { + let (imported, _, _) = imported(project(vec![workbook( + "Book", + vec![worksheet( + "Sheet", + 1, + vec![column( + "value", + OriginColumnType::Integer, + vec![OriginCell::Integer(1)], + )], + )], + )])); + + let _: &BTreeMap = &imported[0].source_metadata; +} + +#[test] +fn rejects_large_numeric_conversion_before_writing_blocks() { + const ROWS: usize = 65_536; + let mut project = project(vec![workbook( + "Book", + vec![worksheet( + "Sheet", + ROWS, + vec![column( + "value", + OriginColumnType::Float, + (0..ROWS) + .map(|value| OriginCell::Float(value as f64)) + .collect(), + )], + )], + )]); + sync_resource_counts(&mut project); + let store = MemoryBlockStore::default(); + let limits = OriginLimits { + max_total_owned_bytes: 20_000_000, + ..OriginLimits::default() + }; + + let error = import_origin_project(project, &store, &CodecRegistry::with_arrow_ipc(), limits) + .expect_err("all row-scaled conversion allocations must be preflighted"); + + assert!(matches!( + error, + OriginImportError::LimitExceeded { + resource: "total owned bytes", + .. + } + )); + assert_eq!(store.block_count(), 0); +} + +#[test] +fn rejects_large_utf8_conversion_before_writing_blocks() { + const ROWS: usize = 65_536; + let mut project = project(vec![workbook( + "Book", + vec![worksheet( + "Sheet", + ROWS, + vec![column( + "value", + OriginColumnType::Text, + (0..ROWS) + .map(|_| OriginCell::Text("0123456789abcdef".to_owned())) + .collect(), + )], + )], + )]); + sync_resource_counts(&mut project); + let store = MemoryBlockStore::default(); + let limits = OriginLimits { + max_total_owned_bytes: 24_000_000, + ..OriginLimits::default() + }; + + let error = import_origin_project(project, &store, &CodecRegistry::with_arrow_ipc(), limits) + .expect_err("all row-scaled conversion allocations must be preflighted"); + + assert!(matches!( + error, + OriginImportError::LimitExceeded { + resource: "total owned bytes", + .. + } + )); + assert_eq!(store.block_count(), 0); +} diff --git a/crates/io/src/lib.rs b/crates/io/src/lib.rs index 9bf1795..4947d44 100644 --- a/crates/io/src/lib.rs +++ b/crates/io/src/lib.rs @@ -8,6 +8,7 @@ pub mod jcamp_dx; pub mod jeol; mod mass_spec; pub mod nanoscope; +pub mod origin; pub mod waters; pub mod xlsx; diff --git a/crates/io/src/origin.rs b/crates/io/src/origin.rs new file mode 100644 index 0000000..b49e1fa --- /dev/null +++ b/crates/io/src/origin.rs @@ -0,0 +1,772 @@ +//! Bounded detection and engine-neutral transport types for Origin projects. +//! +//! Format detection is based only on the first line of the file. The module +//! does not use filename extensions, and OPJU input is detection-only. + +mod opj; +mod opju; +mod reader; + +const DEFAULT_MAX_HEADER_BYTES: usize = 128; +const MIB: usize = 1024 * 1024; +const OPJ_MAGIC: &[u8] = b"CPYA"; +const OPJU_MAGIC: &[u8] = b"CPYUA"; +const ORIGIN_7_V552_VERSION: &str = "4.2673 552"; +const ORIGIN_9_V951_VERSION: &str = "4.3268 195 W64"; + +/// Maximum leading-byte prefix needed to validate every supported Origin profile. +/// +/// Bounded format dispatchers should read this many bytes, or the complete file +/// when it is shorter, before calling [`probe_origin`]. +pub const MAX_PROBE_BYTES: usize = 152; + +/// Origin project container family detected from its file header. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OriginFormat { + /// Classic Origin project container. + Opj, + /// Newer Unicode Origin project container. + Opju, +} + +/// Whether the detected container can be decoded by this parser profile. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OriginSupport { + /// The exact producer profile is supported. + Supported, + /// The family is recognized but deliberately not decoded. + RecognizedUnsupported, +} + +/// Exact producer profiles with verified parsing rules. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OriginProfile { + /// Classic Origin 7 project header `CPYA 4.2673 552#`. + Origin7V552, + /// 64-bit Origin 9.51 project header `CPYA 4.3268 195 W64 #`. + Origin9V951, +} + +/// Byte order used by a verified Origin profile. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OriginByteOrder { + /// Least-significant byte first. + LittleEndian, +} + +/// Integer components parsed from the producer version header. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct OriginHeaderVersion { + /// Version component before the dot. + pub major: u16, + /// Version component after the dot. + pub minor: u16, + /// Numeric producer build. + pub build: u32, +} + +/// Result of bounded, content-based Origin format detection. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OriginProbe { + /// Detected container family. + pub format: OriginFormat, + /// Version and build text exactly as written in the header. + pub raw_version: String, + /// Parsed integer version components. + pub version: OriginHeaderVersion, + /// Byte order of the detected family. + pub byte_order: OriginByteOrder, + /// Exact supported parser profile, if one is verified. + pub profile: Option, + /// Whether full decoding is supported. + pub support: OriginSupport, +} + +/// A cell decoded from an Origin worksheet column. +#[derive(Debug, Clone, PartialEq)] +pub enum OriginCell { + /// Missing or invalid value. + Null, + /// Floating-point value. + Float(f64), + /// Signed integer value. + Integer(i64), + /// Text value. + Text(String), +} + +/// Logical storage class of an Origin worksheet column. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OriginColumnType { + /// Floating-point values and nulls. + Float, + /// Signed integer values and nulls. + Integer, + /// Text values and nulls. + Text, + /// A verified mixture of numeric and text values. + Mixed, +} + +/// One bounded metadata value retained from an Origin object. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OriginMetadataEntry { + /// Stable or source-derived metadata key. + pub key: String, + /// User-safe decoded value. + pub value: String, +} + +/// One named project note retained without concatenating independent content. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OriginNote { + /// Source note name. + pub name: String, + /// Decoded note content. + pub content: String, +} + +/// One worksheet column in the neutral import model. +#[derive(Debug, Clone, PartialEq)] +pub struct OriginColumn { + /// Source column name. + pub name: String, + /// Optional source long name. + pub long_name: Option, + /// Optional Origin column role such as X or Y. + pub role: Option, + /// Optional source units. + pub units: Option, + /// Optional source comments. + pub comments: Option, + /// Logical class of the decoded cells. + pub column_type: OriginColumnType, + /// Cells in source row order. + pub cells: Vec, +} + +/// One worksheet decoded from an Origin workbook. +#[derive(Debug, Clone, PartialEq)] +pub struct OriginWorksheet { + /// Source worksheet name. + pub name: String, + /// Worksheet columns in source order. + pub columns: Vec, + /// Logical row count, including trailing null rows. + pub row_count: usize, + /// Bounded worksheet-level metadata. + pub metadata: Vec, +} + +/// One workbook decoded from an Origin project. +#[derive(Debug, Clone, PartialEq)] +pub struct OriginWorkbook { + /// Source workbook name. + pub name: String, + /// Worksheets in source order. + pub worksheets: Vec, +} + +/// Severity of a recoverable Origin parsing diagnostic. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OriginDiagnosticSeverity { + /// Informational detail that does not alter imported values. + Info, + /// A bounded object or value was skipped or degraded. + Warning, +} + +/// Stable category for a recoverable Origin parsing diagnostic. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OriginDiagnosticCode { + /// An independently framed unsupported object was skipped. + UnsupportedObjectSkipped, + /// An independently framed unsupported column was skipped. + UnsupportedColumnSkipped, + /// Nonessential metadata could not be retained. + MetadataSkipped, + /// Text or value decoding required a documented fallback. + DecodingWarning, +} + +/// Structured location of a recoverable parsing diagnostic. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OriginObjectLocation { + /// Workbook name, when known. + pub workbook: Option, + /// Worksheet name, when known. + pub worksheet: Option, + /// Column name, when known. + pub column: Option, + /// Source byte offset, when meaningful. + pub byte_offset: Option, +} + +/// User-safe diagnostic emitted while retaining a usable project. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OriginDiagnostic { + /// Stable diagnostic category. + pub code: OriginDiagnosticCode, + /// Diagnostic severity. + pub severity: OriginDiagnosticSeverity, + /// Source object or byte location, when known. + pub location: Option, + /// Plain-language message safe to present to a user. + pub message: String, +} + +/// Count of skipped Origin objects of one source-defined kind. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OriginUnsupportedObjectSummary { + /// Source object kind, such as graph or matrix. + pub kind: String, + /// Number of skipped objects of this kind. + pub count: usize, +} + +/// Conservative resource accounting carried into later conversion stages. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct OriginResourceUsage { + /// Retained source input bytes. + pub input_bytes: usize, + /// Parser-owned decoded and structural bytes. + pub parser_bytes: usize, + /// Decoded text bytes included in parser storage. + pub decoded_text_bytes: usize, + /// Cumulative owned-byte charge across import stages. + pub total_owned_bytes: usize, + /// Decoded workbook count. + pub workbooks: usize, + /// Decoded worksheet count. + pub worksheets: usize, + /// Decoded column count. + pub columns: usize, + /// Decoded cell count. + pub cells: usize, + /// Logical metadata records traversed, excluding list terminators. + pub metadata_records: usize, +} + +/// Complete engine-neutral result of an Origin project read. +#[derive(Debug, Clone, PartialEq)] +pub struct OriginProject { + /// Probe that selected the exact parsing profile. + pub probe: OriginProbe, + /// Bounded project parameters. + pub parameters: Vec, + /// Bounded project notes. + pub notes: Vec, + /// Decoded workbooks in source order. + pub workbooks: Vec, + /// Recoverable parsing diagnostics. + pub diagnostics: Vec, + /// Counts of independently skipped unsupported objects. + pub unsupported_objects: Vec, + /// Conservative parser and allocation accounting. + pub resource_usage: OriginResourceUsage, +} + +/// Resource limits applied to untrusted Origin input. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct OriginLimits { + /// Maximum complete source size. + pub max_input_bytes: usize, + /// Maximum first-line header size, including LF. + pub max_header_bytes: usize, + /// Maximum individual framed block size. + pub max_block_bytes: usize, + /// Maximum individual decoded string size. + pub max_string_bytes: usize, + /// Maximum cumulative decoded text size. + pub max_decoded_text_bytes: usize, + /// Maximum cumulative parser-owned allocation. + pub max_parser_bytes: usize, + /// Maximum cumulative owned allocation across import stages. + pub max_total_owned_bytes: usize, + /// Maximum workbook count. + pub max_workbooks: usize, + /// Maximum source window records retained for worksheet association. + pub max_window_records: usize, + /// Maximum worksheet count per workbook. + pub max_worksheets_per_workbook: usize, + /// Maximum total worksheet data column count. + pub max_columns: usize, + /// Maximum logical metadata records traversed, excluding list terminators. + pub max_metadata_records: usize, + /// Maximum logical rows in one column. + pub max_rows_per_column: usize, + /// Maximum total decoded cell count. + pub max_cells: usize, + /// Maximum metadata nesting depth accepted by readers. + pub max_metadata_depth: usize, +} + +impl Default for OriginLimits { + fn default() -> Self { + Self { + max_input_bytes: 128 * MIB, + max_header_bytes: DEFAULT_MAX_HEADER_BYTES, + max_block_bytes: 32 * MIB, + max_string_bytes: MIB, + max_decoded_text_bytes: 32 * MIB, + max_parser_bytes: 128 * MIB, + max_total_owned_bytes: 384 * MIB, + max_workbooks: 256, + max_window_records: 1024, + max_worksheets_per_workbook: 128, + max_columns: 4096, + max_metadata_records: 65_536, + max_rows_per_column: 1_000_000, + max_cells: 2_000_000, + max_metadata_depth: 32, + } + } +} + +impl OriginLimits { + /// Rejects unusable custom limits before any input is parsed. + pub fn validate(&self) -> Result<(), OriginError> { + let values = [ + ("max_input_bytes", self.max_input_bytes), + ("max_header_bytes", self.max_header_bytes), + ("max_block_bytes", self.max_block_bytes), + ("max_string_bytes", self.max_string_bytes), + ("max_decoded_text_bytes", self.max_decoded_text_bytes), + ("max_parser_bytes", self.max_parser_bytes), + ("max_total_owned_bytes", self.max_total_owned_bytes), + ("max_workbooks", self.max_workbooks), + ("max_window_records", self.max_window_records), + ( + "max_worksheets_per_workbook", + self.max_worksheets_per_workbook, + ), + ("max_columns", self.max_columns), + ("max_metadata_records", self.max_metadata_records), + ("max_rows_per_column", self.max_rows_per_column), + ("max_cells", self.max_cells), + ("max_metadata_depth", self.max_metadata_depth), + ]; + if let Some((name, value)) = values.into_iter().find(|(_, value)| *value == 0) { + return Err(OriginError::InvalidLimit { + name, + value, + reason: "the limit must be greater than zero", + }); + } + if self.max_input_bytes.checked_add(1).is_none() { + return Err(OriginError::InvalidLimit { + name: "max_input_bytes", + value: self.max_input_bytes, + reason: "the limit must leave room for an oversize sentinel byte", + }); + } + if self.max_header_bytes.checked_add(1).is_none() { + return Err(OriginError::InvalidLimit { + name: "max_header_bytes", + value: self.max_header_bytes, + reason: "the limit is too large for bounded header probing", + }); + } + if self.max_string_bytes.checked_add(1).is_none() { + return Err(OriginError::InvalidLimit { + name: "max_string_bytes", + value: self.max_string_bytes, + reason: "the limit must leave room for a metadata-line sentinel byte", + }); + } + Ok(()) + } +} + +/// Errors returned while probing or reading untrusted Origin projects. +#[derive(Debug, thiserror::Error, PartialEq, Eq)] +pub enum OriginError { + /// The leading bytes do not match an Origin project family. + #[error("the file is not a recognized Origin project")] + UnrecognizedFormat, + + /// Input ended before a required bounded field was complete. + #[error("Origin project data is truncated at byte {offset}: need {needed} bytes, have {have}")] + Truncated { + /// Offset of the incomplete field. + offset: usize, + /// Bytes required at the offset. + needed: usize, + /// Bytes available at the offset. + have: usize, + }, + + /// The first line cannot fit within the configured header bound. + #[error("Origin project header exceeds the configured {limit}-byte limit")] + HeaderTooLong { + /// Configured maximum bytes, including LF. + limit: usize, + }, + + /// The family signature is present but the version line is invalid. + #[error("malformed Origin project header: {detail}")] + MalformedHeader { + /// User-safe description of the failed grammar rule. + detail: String, + }, + + /// A valid classic header names a producer profile without verified rules. + #[error("Origin project version {raw_version} is not supported")] + UnsupportedVersion { + /// Version and build text from the validated header. + raw_version: String, + }, + + /// OPJU was recognized, but this release intentionally does not decode it. + #[error("{message}")] + UnsupportedOpjuVariant { + /// Plain user-safe explanation that no data was imported. + message: String, + }, + + /// A caller supplied a limit that cannot be applied safely. + #[error("invalid Origin limit {name}={value}: {reason}")] + InvalidLimit { + /// Public field name of the invalid limit. + name: &'static str, + /// Invalid value. + value: usize, + /// Stable reason the value cannot be used. + reason: &'static str, + }, + + /// A verified resource count exceeds its configured bound. + #[error("Origin import {resource} is {actual}, exceeding the configured limit of {limit}")] + LimitExceeded { + /// Resource being bounded. + resource: &'static str, + /// Configured maximum. + limit: usize, + /// Observed or requested amount. + actual: usize, + }, + + /// Checked arithmetic could not represent a derived size or offset. + #[error("Origin import size calculation overflowed for {resource}")] + ArithmeticOverflow { + /// Resource whose calculation overflowed. + resource: &'static str, + }, + + /// A bounded allocation could not be reserved. + #[error("Origin import could not reserve {requested} bytes for {resource}")] + AllocationFailed { + /// Allocation purpose. + resource: &'static str, + /// Requested capacity in bytes. + requested: usize, + }, + + /// Validated framing contains an impossible or unsupported structure. + #[error("corrupt Origin project structure at byte {offset}: {detail}")] + CorruptStructure { + /// Source byte offset. + offset: usize, + /// User-safe structural description. + detail: String, + }, + + /// A value uses an encoding that cannot be decoded without guessing. + #[error("unsupported Origin text encoding at byte {offset}: {encoding}")] + UnsupportedEncoding { + /// Source byte offset. + offset: usize, + /// Source encoding description. + encoding: String, + }, + + /// A structurally valid project uses an unimplemented independent feature. + #[error("unsupported Origin project feature: {feature}")] + UnsupportedFeature { + /// User-safe feature description. + feature: String, + }, + + /// Declared and decoded row geometry disagree. + #[error("Origin column {column} declares {declared} rows but contains {decoded} decoded rows")] + InconsistentRowCount { + /// Source column name. + column: String, + /// Declared logical row count. + declared: usize, + /// Decoded row count. + decoded: usize, + }, + + /// Parsing completed without any worksheet that can be imported. + #[error("the Origin project contains no supported worksheet data")] + NoSupportedWorksheet, +} + +struct AccountedOriginProbe { + probe: OriginProbe, + retained_parser_bytes: usize, +} + +/// Detects an Origin family and exact supported profile from bounded content. +pub fn probe_origin(bytes: &[u8]) -> Result { + let limits = OriginLimits::default(); + Ok(probe_origin_with_limits(bytes, &limits, 0)?.probe) +} + +/// Reads a complete Origin project under explicit resource limits. +pub fn read_origin(bytes: &[u8], limits: OriginLimits) -> Result { + limits.validate()?; + enforce_limit("input bytes", bytes.len(), limits.max_input_bytes)?; + enforce_limit( + "total owned bytes", + bytes.len(), + limits.max_total_owned_bytes, + )?; + + let accounted = probe_origin_with_limits(bytes, &limits, bytes.len())?; + let probe = accounted.probe; + match probe.format { + OriginFormat::Opju => opju::read(probe), + OriginFormat::Opj => opj::read(bytes, &limits, probe, accounted.retained_parser_bytes), + } +} + +fn enforce_limit(resource: &'static str, actual: usize, limit: usize) -> Result<(), OriginError> { + if actual > limit { + return Err(OriginError::LimitExceeded { + resource, + limit, + actual, + }); + } + Ok(()) +} + +fn probe_origin_with_limits( + bytes: &[u8], + limits: &OriginLimits, + initial_total_owned_bytes: usize, +) -> Result { + limits.validate()?; + let format = identify_family(bytes)?; + let line = bounded_first_line(bytes, limits.max_header_bytes)?; + if !line.iter().all(|byte| matches!(byte, b' '..=b'~')) { + return malformed("the version line must contain printable ASCII only"); + } + + let line = std::str::from_utf8(line) + .map_err(|_| malformed_error("the version line must contain printable ASCII only"))?; + let raw_version = match format { + OriginFormat::Opj => { + let value = line + .strip_prefix("CPYA ") + .and_then(|value| value.strip_suffix('#')) + .ok_or_else(|| { + malformed_error("classic OPJ headers must end with exactly one # before LF") + })?; + value + .strip_suffix(' ') + .filter(|value| value.ends_with(" W64")) + .unwrap_or(value) + } + OriginFormat::Opju => line.strip_prefix("CPYUA ").ok_or_else(|| { + malformed_error("OPJU headers must contain one space after the CPYUA signature") + })?, + }; + let version = parse_version(raw_version, format)?; + let (raw_version, retained_parser_bytes) = + copy_header_version(raw_version, limits, initial_total_owned_bytes)?; + + match format { + OriginFormat::Opj => { + let profile = match raw_version.as_str() { + ORIGIN_7_V552_VERSION => OriginProfile::Origin7V552, + ORIGIN_9_V951_VERSION => OriginProfile::Origin9V951, + _ => return Err(OriginError::UnsupportedVersion { raw_version }), + }; + opj::validate_initial_structure(bytes, profile, limits.max_block_bytes)?; + Ok(AccountedOriginProbe { + probe: OriginProbe { + format, + raw_version, + version, + byte_order: OriginByteOrder::LittleEndian, + profile: Some(profile), + support: OriginSupport::Supported, + }, + retained_parser_bytes, + }) + } + OriginFormat::Opju => Ok(AccountedOriginProbe { + probe: OriginProbe { + format, + raw_version, + version, + byte_order: OriginByteOrder::LittleEndian, + profile: None, + support: OriginSupport::RecognizedUnsupported, + }, + retained_parser_bytes, + }), + } +} + +fn identify_family(bytes: &[u8]) -> Result { + if bytes.starts_with(OPJU_MAGIC) { + return Ok(OriginFormat::Opju); + } + if bytes.starts_with(OPJ_MAGIC) { + return Ok(OriginFormat::Opj); + } + + let needed = if OPJ_MAGIC.starts_with(bytes) { + Some(OPJ_MAGIC.len()) + } else if OPJU_MAGIC.starts_with(bytes) { + Some(OPJU_MAGIC.len()) + } else { + None + }; + match needed { + Some(needed) => Err(OriginError::Truncated { + offset: 0, + needed, + have: bytes.len(), + }), + None => Err(OriginError::UnrecognizedFormat), + } +} + +fn bounded_first_line(bytes: &[u8], max_header_bytes: usize) -> Result<&[u8], OriginError> { + let scan_len = bytes.len().min(max_header_bytes); + let scan = bytes.get(..scan_len).ok_or(OriginError::Truncated { + offset: 0, + needed: scan_len, + have: bytes.len(), + })?; + if let Some(newline) = scan.iter().position(|byte| *byte == b'\n') { + return scan.get(..newline).ok_or(OriginError::Truncated { + offset: 0, + needed: newline, + have: scan.len(), + }); + } + if bytes.len() >= max_header_bytes { + return Err(OriginError::HeaderTooLong { + limit: max_header_bytes, + }); + } + Err(OriginError::Truncated { + offset: bytes.len(), + needed: 1, + have: 0, + }) +} + +fn parse_version( + raw_version: &str, + format: OriginFormat, +) -> Result { + let mut fields = raw_version.split(' '); + let version = fields.next().unwrap_or_default(); + let build = fields.next().unwrap_or_default(); + let platform = fields.next(); + let platform_is_valid = + platform.is_none() || matches!((format, platform), (OriginFormat::Opj, Some("W64"))); + if version.is_empty() || build.is_empty() || !platform_is_valid || fields.next().is_some() { + return malformed( + "the header must contain version and numeric build fields followed only by the optional classic OPJ W64 marker", + ); + } + + let (major, minor) = version + .split_once('.') + .ok_or_else(|| malformed_error("the version token must contain major.minor integers"))?; + if !is_ascii_digits(major) + || !is_ascii_digits(minor) + || minor.contains('.') + || !is_ascii_digits(build) + { + return malformed("the version and build fields must contain decimal digits only"); + } + + let major = major + .parse::() + .map_err(|_| malformed_error("the major version is outside the supported integer range"))?; + let minor = minor + .parse::() + .map_err(|_| malformed_error("the minor version is outside the supported integer range"))?; + let build = build + .parse::() + .map_err(|_| malformed_error("the build is outside the supported integer range"))?; + Ok(OriginHeaderVersion { + major, + minor, + build, + }) +} + +fn is_ascii_digits(value: &str) -> bool { + !value.is_empty() && value.bytes().all(|byte| byte.is_ascii_digit()) +} + +fn copy_header_version( + raw_version: &str, + limits: &OriginLimits, + initial_total_owned_bytes: usize, +) -> Result<(String, usize), OriginError> { + let requested = raw_version.len(); + enforce_limit("string bytes", requested, limits.max_string_bytes)?; + enforce_limit("parser bytes", requested, limits.max_parser_bytes)?; + let preflight_total = + reader::checked_add(initial_total_owned_bytes, requested, "total owned bytes")?; + enforce_limit( + "total owned bytes", + preflight_total, + limits.max_total_owned_bytes, + )?; + + let mut owned = String::new(); + owned + .try_reserve_exact(requested) + .map_err(|_| OriginError::AllocationFailed { + resource: "header version text", + requested, + })?; + let retained_parser_bytes = owned.capacity(); + enforce_limit( + "parser bytes", + retained_parser_bytes, + limits.max_parser_bytes, + )?; + let actual_total = reader::checked_add( + initial_total_owned_bytes, + retained_parser_bytes, + "total owned bytes", + )?; + enforce_limit( + "total owned bytes", + actual_total, + limits.max_total_owned_bytes, + )?; + owned.push_str(raw_version); + Ok((owned, retained_parser_bytes)) +} + +fn malformed(detail: &str) -> Result { + Err(malformed_error(detail)) +} + +fn malformed_error(detail: &str) -> OriginError { + OriginError::MalformedHeader { + detail: detail.to_owned(), + } +} + +#[cfg(test)] +#[path = "origin/reader_tests.rs"] +mod reader_tests; + +#[cfg(test)] +#[path = "origin/tests.rs"] +mod tests; diff --git a/crates/io/src/origin/opj.rs b/crates/io/src/origin/opj.rs new file mode 100644 index 0000000..e998eca --- /dev/null +++ b/crates/io/src/origin/opj.rs @@ -0,0 +1,621 @@ +use std::mem::size_of; + +use super::reader::{FramedBlock, Reader, checked_add}; +use super::{ + OriginColumn, OriginDiagnosticCode, OriginError, OriginLimits, OriginProbe, OriginProfile, + OriginProject, OriginResourceUsage, OriginWorkbook, OriginWorksheet, +}; + +mod metadata; +mod records; + +const ORIGIN_VERSION_OFFSET: usize = 0x1b; +const BLOCK_PREFIX_LEN: usize = 5; + +#[derive(Debug, Clone, Copy)] +enum MetadataPolicy { + Full, + WindowsOnly, +} + +#[derive(Debug, Clone, Copy)] +struct ProfileLayout { + name: &'static str, + raw_version: &'static str, + signature: &'static [u8], + global_header_len: usize, + embedded_version: f64, + data_header_len: usize, + metadata_policy: MetadataPolicy, +} + +const ORIGIN7_LAYOUT: ProfileLayout = ProfileLayout { + name: "Origin7V552", + raw_version: "4.2673 552", + signature: b"CPYA 4.2673 552#\n", + global_header_len: 39, + embedded_version: 7.0552, + data_header_len: 123, + metadata_policy: MetadataPolicy::Full, +}; + +const ORIGIN9_LAYOUT: ProfileLayout = ProfileLayout { + name: "Origin9V951", + raw_version: "4.3268 195 W64", + signature: b"CPYA 4.3268 195 W64 #\n", + global_header_len: 115, + embedded_version: 9.510195, + data_header_len: 147, + metadata_policy: MetadataPolicy::WindowsOnly, +}; + +fn profile_layout(profile: OriginProfile) -> &'static ProfileLayout { + match profile { + OriginProfile::Origin7V552 => &ORIGIN7_LAYOUT, + OriginProfile::Origin9V951 => &ORIGIN9_LAYOUT, + } +} + +#[derive(Debug)] +pub(super) struct RawOpjDataSection<'a> { + pub(super) header: &'a [u8], + pub(super) content: Option<&'a [u8]>, +} + +#[derive(Debug)] +pub(super) struct RawOpjProject<'a> { + pub(super) origin_header: &'a [u8], + pub(super) data_sections: Vec>, + pub(super) remaining: &'a [u8], + pub(super) resource_usage: OriginResourceUsage, +} + +pub(super) fn validate_initial_structure( + bytes: &[u8], + profile: OriginProfile, + max_block_bytes: usize, +) -> Result<(), OriginError> { + let layout = profile_layout(profile); + let initial_structure_len = layout + .signature + .len() + .checked_add(BLOCK_PREFIX_LEN) + .and_then(|length| length.checked_add(layout.global_header_len)) + .and_then(|length| length.checked_add(1 + BLOCK_PREFIX_LEN)) + .ok_or(OriginError::ArithmeticOverflow { + resource: "initial OPJ probe structure", + })?; + let initial_len = bytes.len().min(initial_structure_len); + let initial = bytes + .get(..initial_len) + .ok_or(OriginError::ArithmeticOverflow { + resource: "initial OPJ probe structure", + })?; + let limits = OriginLimits { + max_input_bytes: initial_structure_len, + max_block_bytes, + max_total_owned_bytes: initial_structure_len, + ..OriginLimits::default() + }; + let mut reader = Reader::new(initial, &limits)?; + let signature = reader.read_slice(layout.signature.len())?; + if signature != layout.signature { + return Err(OriginError::CorruptStructure { + offset: 0, + detail: "the classic OPJ signature changed inside its initial framing".to_owned(), + }); + } + let header_block = reader.read_block()?; + let (header_offset, header) = + require_data_block(header_block, "the Origin header must be a data block")?; + require_exact_length( + header_offset, + header, + layout.global_header_len, + "Origin header payload", + )?; + validate_embedded_origin_version(header_offset, header, layout)?; + let terminator_offset = reader.offset(); + let terminator = reader.read_slice(BLOCK_PREFIX_LEN)?; + if terminator != [0, 0, 0, 0, b'\n'] { + return Err(OriginError::CorruptStructure { + offset: terminator_offset, + detail: "the Origin header must end with a null block".to_owned(), + }); + } + Ok(()) +} + +pub(super) fn read( + bytes: &[u8], + limits: &OriginLimits, + probe: OriginProbe, + retained_probe_bytes: usize, +) -> Result { + let profile = probe + .profile + .ok_or_else(|| OriginError::UnsupportedVersion { + raw_version: probe.raw_version.clone(), + })?; + let raw = parse_raw(bytes, limits, profile, retained_probe_bytes)?; + if raw.data_sections.is_empty() && raw.remaining.is_empty() { + return Err(OriginError::NoSupportedWorksheet); + } + + let _validated_header_len = raw.origin_header.len(); + let mut usage = raw.resource_usage; + let metadata_offset = + bytes + .len() + .checked_sub(raw.remaining.len()) + .ok_or(OriginError::ArithmeticOverflow { + resource: "OPJ metadata offset", + })?; + let mut parsed = match profile_layout(profile).metadata_policy { + MetadataPolicy::Full => { + metadata::parse(raw.remaining, metadata_offset, limits, &mut usage)? + } + MetadataPolicy::WindowsOnly => { + metadata::parse_windows_only(raw.remaining, metadata_offset, limits, &mut usage)? + } + }; + let mut unsupported_columns = 0_usize; + + for section in &raw.data_sections { + match records::decode_column_record( + profile, + section.header, + section.content, + limits, + &mut usage, + ) { + Ok(decoded) => { + match associate_dataset(&decoded.dataset_name, &parsed.windows) { + DatasetAssociation::Window { + index, + prefix_bytes, + } => { + let column = make_column(decoded, Some(prefix_bytes))?; + let window = parsed.windows.get_mut(index).ok_or( + OriginError::ArithmeticOverflow { + resource: "associated Origin window", + }, + )?; + metadata::try_reserve( + &mut window.columns, + 1, + "Origin worksheet columns", + limits, + &mut usage, + )?; + window.columns.push(column); + } + DatasetAssociation::ExactWindow => { + unsupported_columns = + checked_add(unsupported_columns, 1, "unsupported Origin columns")?; + } + DatasetAssociation::Fallback => { + // A decoded column is still unsafe to expose when no + // verified window record proves its table grouping. + // Combining unrelated unmatched columns would invent + // row alignment that is not present in the source. + unsupported_columns = + checked_add(unsupported_columns, 1, "unsupported Origin columns")?; + } + } + } + Err(OriginError::UnsupportedFeature { .. }) => { + // The Task 4 decoder emits UnsupportedFeature only while + // classifying a fully framed column header, before it reads or + // interprets that column's payload. The outer data-section + // bounds therefore make this individual skip deterministic. + unsupported_columns = + checked_add(unsupported_columns, 1, "unsupported Origin columns")?; + } + Err(error) => return Err(error), + } + } + + if unsupported_columns > 0 { + metadata::push_summary( + &mut parsed.unsupported_objects, + "worksheet columns", + unsupported_columns, + limits, + &mut usage, + )?; + metadata::push_diagnostic( + &mut parsed.diagnostics, + OriginDiagnosticCode::UnsupportedColumnSkipped, + "PlotX skipped independently framed Origin columns whose value layouts are not verified.", + None, + limits, + &mut usage, + )?; + } + + let unused_windows = parsed + .windows + .iter() + .filter(|window| window.columns.is_empty()) + .count(); + if unused_windows > 0 { + metadata::push_summary( + &mut parsed.unsupported_objects, + "unsupported window records", + unused_windows, + limits, + &mut usage, + )?; + metadata::push_diagnostic( + &mut parsed.diagnostics, + OriginDiagnosticCode::UnsupportedObjectSkipped, + "PlotX skipped Origin window records that had no supported worksheet columns.", + None, + limits, + &mut usage, + )?; + } + + let workbooks = assemble_workbooks(parsed.windows, limits, &mut usage)?; + Ok(OriginProject { + probe, + parameters: parsed.parameters, + notes: parsed.notes, + workbooks, + diagnostics: parsed.diagnostics, + unsupported_objects: parsed.unsupported_objects, + resource_usage: usage, + }) +} + +enum DatasetAssociation { + Window { index: usize, prefix_bytes: usize }, + ExactWindow, + Fallback, +} + +fn associate_dataset(dataset_name: &str, windows: &[metadata::WindowInfo]) -> DatasetAssociation { + let mut best = None; + let mut best_len = 0_usize; + let mut ambiguous = false; + let mut exact = false; + + for (index, window) in windows.iter().enumerate() { + let Some(window_name) = window.name.as_deref() else { + continue; + }; + if dataset_name == window_name { + exact = true; + continue; + } + let Some(suffix) = dataset_name + .strip_prefix(window_name) + .and_then(|rest| rest.strip_prefix('_')) + else { + continue; + }; + if suffix.is_empty() { + continue; + } + + match window_name.len().cmp(&best_len) { + std::cmp::Ordering::Greater => { + best = Some(index); + best_len = window_name.len(); + ambiguous = false; + } + std::cmp::Ordering::Equal => ambiguous = true, + std::cmp::Ordering::Less => {} + } + } + + if exact { + return DatasetAssociation::ExactWindow; + } + match (best, ambiguous) { + (Some(index), false) => DatasetAssociation::Window { + index, + prefix_bytes: best_len, + }, + _ => DatasetAssociation::Fallback, + } +} + +fn make_column( + decoded: records::DecodedColumnRecord, + prefix_bytes: Option, +) -> Result { + let mut name = decoded.dataset_name; + if let Some(prefix_bytes) = prefix_bytes { + if !name.is_char_boundary(prefix_bytes) || name.get(prefix_bytes..).is_none() { + return Err(OriginError::CorruptStructure { + offset: 0, + detail: "an associated Origin dataset prefix is not a text boundary".to_owned(), + }); + } + name.replace_range(..prefix_bytes, ""); + if name.as_bytes().first() == Some(&b'_') { + name.remove(0); + } + } + Ok(OriginColumn { + name, + long_name: None, + role: None, + units: None, + comments: None, + column_type: decoded.column_type, + cells: decoded.cells, + }) +} + +fn assemble_workbooks( + windows: Vec, + limits: &OriginLimits, + usage: &mut OriginResourceUsage, +) -> Result, OriginError> { + let workbook_count = windows + .iter() + .filter(|window| !window.columns.is_empty()) + .count(); + if workbook_count == 0 { + return Err(OriginError::NoSupportedWorksheet); + } + enforce_count("workbooks", workbook_count, limits.max_workbooks)?; + enforce_count( + "worksheets per workbook", + 1, + limits.max_worksheets_per_workbook, + )?; + let has_supported_rows = windows + .iter() + .any(|window| window.columns.iter().any(|column| !column.cells.is_empty())); + if !has_supported_rows { + return Err(OriginError::NoSupportedWorksheet); + } + + let mut workbooks = Vec::new(); + metadata::try_reserve( + &mut workbooks, + workbook_count, + "Origin workbooks", + limits, + usage, + )?; + for window in windows { + if window.columns.is_empty() { + continue; + } + let name = window.name.ok_or(OriginError::CorruptStructure { + offset: 0, + detail: "a supported Origin worksheet lost its validated window name".to_owned(), + })?; + push_workbook(&mut workbooks, name, window.columns, limits, usage)?; + } + usage.workbooks = workbook_count; + usage.worksheets = workbook_count; + Ok(workbooks) +} + +fn push_workbook( + workbooks: &mut Vec, + name: String, + columns: Vec, + limits: &OriginLimits, + usage: &mut OriginResourceUsage, +) -> Result<(), OriginError> { + let row_count = columns + .iter() + .map(|column| column.cells.len()) + .max() + .unwrap_or(0); + let worksheet_name = metadata::copy_generated_text("Sheet1", limits, usage)?; + let mut worksheets = Vec::new(); + metadata::try_reserve(&mut worksheets, 1, "Origin worksheets", limits, usage)?; + worksheets.push(OriginWorksheet { + name: worksheet_name, + columns, + row_count, + metadata: Vec::new(), + }); + workbooks.push(OriginWorkbook { name, worksheets }); + Ok(()) +} + +fn enforce_count(resource: &'static str, actual: usize, limit: usize) -> Result<(), OriginError> { + if actual > limit { + return Err(OriginError::LimitExceeded { + resource, + limit, + actual, + }); + } + Ok(()) +} + +pub(super) fn parse_raw<'a>( + bytes: &'a [u8], + limits: &OriginLimits, + profile: OriginProfile, + initial_parser_bytes: usize, +) -> Result, OriginError> { + let layout = profile_layout(profile); + let mut reader = Reader::new_with_parser_bytes(bytes, limits, initial_parser_bytes)?; + let signature = reader.read_slice(layout.signature.len())?; + if signature != layout.signature { + return Err(OriginError::UnsupportedVersion { + raw_version: format!("classic OPJ signature does not match {}", layout.name), + }); + } + + // OpenOPJ documents the Origin header as one data block followed by a null + // block, with the Origin version f64 at payload offset 0x1b: + // https://github.com/jgonera/openopj/blob/42ddcf1eb3a490744c54fca0a4ed6fe7a5e723ca/docs/opj_format.markdown + // https://github.com/jgonera/openopj/blob/42ddcf1eb3a490744c54fca0a4ed6fe7a5e723ca/lib/OpenOPJ/OPJFile.php + let origin_header_block = reader.read_block()?; + let (origin_header_offset, origin_header) = require_data_block( + origin_header_block, + "the Origin header must be a data block", + )?; + require_exact_length( + origin_header_offset, + origin_header, + layout.global_header_len, + "Origin header payload", + )?; + validate_embedded_origin_version(origin_header_offset, origin_header, layout)?; + + let header_terminator = reader.read_block()?; + require_null_block( + header_terminator, + "the Origin header must end with a null block", + )?; + + let mut data_sections = Vec::new(); + loop { + // Each verified profile's data list is a sequence of + // , + // followed by a consumed list null. Parsing stops at that exact + // bounded boundary. + // https://github.com/jgonera/openopj/blob/42ddcf1eb3a490744c54fca0a4ed6fe7a5e723ca/docs/opj_format.markdown + // https://github.com/jgonera/openopj/blob/42ddcf1eb3a490744c54fca0a4ed6fe7a5e723ca/lib/OpenOPJ/OPJFile.php + let header_block = reader.read_block()?; + let (header_offset, header) = match header_block { + FramedBlock::Null { .. } => break, + FramedBlock::Data { offset, payload } => (offset, payload), + }; + require_exact_length( + header_offset, + header, + layout.data_header_len, + match profile { + OriginProfile::Origin7V552 => "Origin7V552 data-header payload", + OriginProfile::Origin9V951 => "Origin9V951 data-header payload", + }, + )?; + + let content = match reader.read_block()? { + FramedBlock::Null { .. } => None, + FramedBlock::Data { payload, .. } => Some(payload), + }; + let section_terminator = reader.read_block()?; + require_null_block( + section_terminator, + "an Origin data section must end with a null block", + )?; + + let section_count = checked_add(data_sections.len(), 1, "raw OPJ data sections")?; + if section_count > limits.max_columns { + return Err(OriginError::LimitExceeded { + resource: "data sections", + limit: limits.max_columns, + actual: section_count, + }); + } + reader.try_reserve(&mut data_sections, 1, "raw OPJ data sections")?; + data_sections.push(RawOpjDataSection { header, content }); + } + + let remaining = bytes + .get(reader.offset()..) + .ok_or(OriginError::ArithmeticOverflow { + resource: "remaining OPJ structure", + })?; + let resource_usage = reader.into_usage(); + Ok(RawOpjProject { + origin_header, + data_sections, + remaining, + resource_usage, + }) +} + +fn validate_embedded_origin_version( + header_block_offset: usize, + header: &[u8], + layout: &ProfileLayout, +) -> Result<(), OriginError> { + let version_payload_delta = checked_add( + BLOCK_PREFIX_LEN, + ORIGIN_VERSION_OFFSET, + "embedded Origin version offset", + )?; + let version_offset_in_file = checked_add( + header_block_offset, + version_payload_delta, + "embedded Origin version offset", + )?; + let version_end = checked_add( + ORIGIN_VERSION_OFFSET, + size_of::(), + "embedded Origin version range", + )?; + let available_version_bytes = + header + .len() + .checked_sub(ORIGIN_VERSION_OFFSET) + .ok_or(OriginError::ArithmeticOverflow { + resource: "embedded Origin version bytes", + })?; + let version_bytes = + header + .get(ORIGIN_VERSION_OFFSET..version_end) + .ok_or(OriginError::Truncated { + offset: version_offset_in_file, + needed: size_of::(), + have: available_version_bytes, + })?; + let version_array: [u8; 8] = version_bytes + .try_into() + .map_err(|_| OriginError::Truncated { + offset: version_offset_in_file, + needed: size_of::(), + have: version_bytes.len(), + })?; + let version = f64::from_le_bytes(version_array); + if version.to_bits() != layout.embedded_version.to_bits() { + return Err(OriginError::UnsupportedVersion { + raw_version: format!("{} header embeds Origin {version}", layout.raw_version), + }); + } + Ok(()) +} + +fn require_data_block<'a>( + block: FramedBlock<'a>, + detail: &'static str, +) -> Result<(usize, &'a [u8]), OriginError> { + match block { + FramedBlock::Data { offset, payload } => Ok((offset, payload)), + FramedBlock::Null { offset } => Err(OriginError::CorruptStructure { + offset, + detail: detail.to_owned(), + }), + } +} + +fn require_null_block(block: FramedBlock<'_>, detail: &'static str) -> Result<(), OriginError> { + match block { + FramedBlock::Null { .. } => Ok(()), + FramedBlock::Data { .. } => Err(OriginError::CorruptStructure { + offset: block.offset(), + detail: detail.to_owned(), + }), + } +} + +fn require_exact_length( + block_offset: usize, + payload: &[u8], + expected: usize, + field: &'static str, +) -> Result<(), OriginError> { + if payload.len() != expected { + return Err(OriginError::CorruptStructure { + offset: block_offset, + detail: format!("{field} must be exactly {expected} bytes"), + }); + } + Ok(()) +} diff --git a/crates/io/src/origin/opj/metadata.rs b/crates/io/src/origin/opj/metadata.rs new file mode 100644 index 0000000..483a815 --- /dev/null +++ b/crates/io/src/origin/opj/metadata.rs @@ -0,0 +1,775 @@ +use std::fmt::{self, Write as _}; +use std::mem::size_of; + +use crate::origin::{ + OriginColumn, OriginDiagnostic, OriginDiagnosticCode, OriginDiagnosticSeverity, OriginError, + OriginLimits, OriginMetadataEntry, OriginNote, OriginObjectLocation, OriginResourceUsage, + OriginUnsupportedObjectSummary, +}; + +use super::super::reader::{checked_add, checked_mul}; +use cursor::{MetadataBlock, MetadataCursor}; + +mod cursor; +mod tail; + +const BLOCK_PREFIX_LEN: usize = 5; +const WINDOW_NAME_OFFSET: usize = 2; +const WINDOW_NAME_WIDTH: usize = 25; +const WINDOW_HEADER_MIN_LEN: usize = WINDOW_NAME_OFFSET + WINDOW_NAME_WIDTH; +const AXIS_PARAMETER_LISTS: usize = 3; +const FORMATTED_F64_CAPACITY: usize = 32; + +pub(super) struct WindowInfo { + pub(super) name: Option, + pub(super) columns: Vec, +} + +pub(super) struct ParsedMetadata { + pub(super) windows: Vec, + pub(super) parameters: Vec, + pub(super) notes: Vec, + pub(super) diagnostics: Vec, + pub(super) unsupported_objects: Vec, +} + +pub(super) fn parse( + bytes: &[u8], + base_offset: usize, + limits: &OriginLimits, + usage: &mut OriginResourceUsage, +) -> Result { + let mut cursor = MetadataCursor::new(bytes, base_offset, limits); + let mut diagnostics = Vec::new(); + let mut unsupported_objects = Vec::new(); + + let (windows, presentation_records) = parse_windows(&mut cursor, usage, &mut diagnostics)?; + if presentation_records > 0 { + push_summary( + &mut unsupported_objects, + "window presentation records", + presentation_records, + limits, + usage, + )?; + push_diagnostic( + &mut diagnostics, + OriginDiagnosticCode::UnsupportedObjectSkipped, + "PlotX imported worksheet values but skipped bounded Origin window presentation records.", + None, + limits, + usage, + )?; + } + let parameters = parse_parameters(&mut cursor, usage, &mut diagnostics)?; + let (notes, note_properties) = parse_notes(&mut cursor, usage, &mut diagnostics)?; + if note_properties > 0 { + push_summary( + &mut unsupported_objects, + "note properties", + note_properties, + limits, + usage, + )?; + push_diagnostic( + &mut diagnostics, + OriginDiagnosticCode::MetadataSkipped, + "PlotX imported note names and text but skipped bounded Origin note properties.", + None, + limits, + usage, + )?; + } + + tail::parse( + &mut cursor, + &mut diagnostics, + &mut unsupported_objects, + usage, + )?; + usage.metadata_records = cursor.metadata_records(); + + Ok(ParsedMetadata { + windows, + parameters, + notes, + diagnostics, + unsupported_objects, + }) +} + +pub(super) fn parse_windows_only( + bytes: &[u8], + base_offset: usize, + limits: &OriginLimits, + usage: &mut OriginResourceUsage, +) -> Result { + let mut cursor = MetadataCursor::new(bytes, base_offset, limits); + let mut diagnostics = Vec::new(); + let mut unsupported_objects = Vec::new(); + let (windows, presentation_records) = parse_windows(&mut cursor, usage, &mut diagnostics)?; + + if presentation_records > 0 { + push_summary( + &mut unsupported_objects, + "window presentation records", + presentation_records, + limits, + usage, + )?; + push_diagnostic( + &mut diagnostics, + OriginDiagnosticCode::UnsupportedObjectSkipped, + "PlotX imported worksheet values but skipped bounded Origin window presentation records.", + None, + limits, + usage, + )?; + } + + // The Origin9V951 evidence establishes the complete window-list boundary, + // but not the object grammar that follows it. Stop at the consumed null + // terminator and report one opaque tail instead of scanning for markers. + if cursor.remaining() > 0 { + push_summary( + &mut unsupported_objects, + "remaining Origin 9.51 project objects", + 1, + limits, + usage, + )?; + push_diagnostic( + &mut diagnostics, + OriginDiagnosticCode::UnsupportedObjectSkipped, + "PlotX imported worksheet values but did not import the remaining Origin 9.51 project objects.", + Some(cursor.absolute_offset()?), + limits, + usage, + )?; + } + usage.metadata_records = cursor.metadata_records(); + + Ok(ParsedMetadata { + windows, + parameters: Vec::new(), + notes: Vec::new(), + diagnostics, + unsupported_objects, + }) +} + +fn parse_windows( + cursor: &mut MetadataCursor<'_>, + usage: &mut OriginResourceUsage, + diagnostics: &mut Vec, +) -> Result<(Vec, usize), OriginError> { + enforce_depth(1, cursor.limits)?; + let mut windows = Vec::new(); + let mut presentation_records = 0_usize; + + loop { + let header = match cursor.read_block()? { + MetadataBlock::Null { .. } => break, + MetadataBlock::Data { offset, payload } => (offset, payload), + }; + cursor.charge_record()?; + let window_count = checked_add(windows.len(), 1, "window records")?; + enforce_limit( + "window records", + window_count, + cursor.limits.max_window_records, + )?; + + // This exact header-plus-layer-list traversal is reimplemented from + // the pinned MIT OpenOPJ Origin 7.0552 WindowList description: + // https://github.com/jgonera/openopj/blob/42ddcf1eb3a490744c54fca0a4ed6fe7a5e723ca/lib/OpenOPJ/WindowList.php + let name = match decode_window_name(header.1, header.0, cursor.limits, usage) { + Ok(name) if !name.is_empty() => Some(name), + Ok(_) => { + push_diagnostic( + diagnostics, + OriginDiagnosticCode::MetadataSkipped, + "PlotX skipped an empty Origin window name after validating its boundaries.", + Some(header.0), + cursor.limits, + usage, + )?; + None + } + Err(OriginError::UnsupportedEncoding { .. }) => { + push_diagnostic( + diagnostics, + OriginDiagnosticCode::MetadataSkipped, + "PlotX skipped a non-ASCII Origin window name after validating its boundaries.", + Some(header.0), + cursor.limits, + usage, + )?; + None + } + Err(error) => return Err(error), + }; + + presentation_records = checked_add( + presentation_records, + walk_layer_list(cursor, 2)?, + "Origin window presentation records", + )?; + try_reserve( + &mut windows, + 1, + "Origin window records", + cursor.limits, + usage, + )?; + windows.push(WindowInfo { + name, + columns: Vec::new(), + }); + } + Ok((windows, presentation_records)) +} + +fn walk_layer_list(cursor: &mut MetadataCursor<'_>, depth: usize) -> Result { + enforce_depth(depth, cursor.limits)?; + let mut records = 0_usize; + loop { + match cursor.read_block()? { + MetadataBlock::Null { .. } => break, + MetadataBlock::Data { .. } => cursor.charge_record()?, + } + records = checked_add(records, 1, "Origin window presentation records")?; + + records = checked_add( + records, + walk_fixed_block_list(cursor, checked_add(depth, 1, "metadata nesting depth")?, 4)?, + "Origin window presentation records", + )?; + records = checked_add( + records, + walk_fixed_block_list(cursor, checked_add(depth, 1, "metadata nesting depth")?, 2)?, + "Origin window presentation records", + )?; + records = checked_add( + records, + walk_fixed_block_list(cursor, checked_add(depth, 1, "metadata nesting depth")?, 1)?, + "Origin window presentation records", + )?; + for _ in 0..AXIS_PARAMETER_LISTS { + records = checked_add( + records, + walk_fixed_block_list(cursor, checked_add(depth, 1, "metadata nesting depth")?, 1)?, + "Origin window presentation records", + )?; + } + } + Ok(records) +} + +fn walk_fixed_block_list( + cursor: &mut MetadataCursor<'_>, + depth: usize, + blocks_per_item: usize, +) -> Result { + enforce_depth(depth, cursor.limits)?; + let mut items = 0_usize; + loop { + match cursor.read_block()? { + MetadataBlock::Null { .. } => break, + MetadataBlock::Data { .. } => cursor.charge_record()?, + } + items = checked_add(items, 1, "Origin nested records")?; + for _ in 1..blocks_per_item { + let _ = cursor.read_block()?; + } + } + Ok(items) +} + +fn parse_parameters( + cursor: &mut MetadataCursor<'_>, + usage: &mut OriginResourceUsage, + diagnostics: &mut Vec, +) -> Result, OriginError> { + // Origin7V552 parameters use an LF-terminated name, one little-endian f64 + // plus LF, and a NUL-name terminator as described by pinned MIT OpenOPJ: + // https://github.com/jgonera/openopj/blob/42ddcf1eb3a490744c54fca0a4ed6fe7a5e723ca/lib/OpenOPJ/ParametersSection.php + let mut parameters = Vec::new(); + loop { + let (name_offset, name_bytes) = cursor.read_line()?; + if name_bytes == [0] { + break; + } + cursor.charge_record()?; + if name_bytes.is_empty() { + return Err(OriginError::CorruptStructure { + offset: name_offset, + detail: "an Origin parameter name cannot be empty".to_owned(), + }); + } + + let value_offset = cursor.absolute_offset()?; + let value_bytes = cursor.read_parameter_value()?; + let value = f64::from_le_bytes(value_bytes); + + let name = match validate_ascii(name_bytes, name_offset, "Origin parameter name") { + Ok(name) => name, + Err(OriginError::UnsupportedEncoding { .. }) => { + push_diagnostic( + diagnostics, + OriginDiagnosticCode::MetadataSkipped, + "PlotX skipped a non-ASCII Origin parameter after validating its value boundary.", + Some(name_offset), + cursor.limits, + usage, + )?; + continue; + } + Err(error) => return Err(error), + }; + if !value.is_finite() { + push_diagnostic( + diagnostics, + OriginDiagnosticCode::MetadataSkipped, + "PlotX skipped a non-finite Origin parameter value.", + Some(value_offset), + cursor.limits, + usage, + )?; + continue; + } + + let key = copy_decoded_text(name, cursor.limits, usage)?; + let formatted = format_f64(value, value_offset)?; + let value = copy_parser_text(formatted, cursor.limits, usage)?; + try_reserve( + &mut parameters, + 1, + "Origin project parameters", + cursor.limits, + usage, + )?; + parameters.push(OriginMetadataEntry { key, value }); + } + + require_null_block( + cursor.read_block()?, + "the Origin parameter section must end with a null block", + )?; + Ok(parameters) +} + +fn parse_notes( + cursor: &mut MetadataCursor<'_>, + usage: &mut OriginResourceUsage, + diagnostics: &mut Vec, +) -> Result<(Vec, usize), OriginError> { + // Each note is exactly header/name/content framed blocks. PlotX retains + // only bounded ASCII name/content and reports the opaque header properties: + // https://github.com/jgonera/openopj/blob/42ddcf1eb3a490744c54fca0a4ed6fe7a5e723ca/lib/OpenOPJ/NoteSection.php + let mut notes = Vec::new(); + let mut note_properties = 0_usize; + loop { + let header_offset = match cursor.read_block()? { + MetadataBlock::Null { .. } => break, + MetadataBlock::Data { offset, .. } => offset, + }; + cursor.charge_record()?; + note_properties = checked_add(note_properties, 1, "Origin note properties")?; + let (name_offset, name_block) = require_data_block( + cursor.read_block()?, + "an Origin note name must be a data block", + )?; + let (content_offset, content_block) = require_data_block( + cursor.read_block()?, + "an Origin note content must be a data block", + )?; + + let name = validate_nul_terminated_ascii( + name_block, + name_offset, + cursor.limits, + "Origin note name", + ); + let content = validate_nul_terminated_ascii( + content_block, + content_offset, + cursor.limits, + "Origin note content", + ); + let (name, content) = match (name, content) { + (Ok(name), Ok(content)) => (name, content), + (Err(OriginError::UnsupportedEncoding { .. }), _) + | (_, Err(OriginError::UnsupportedEncoding { .. })) => { + push_diagnostic( + diagnostics, + OriginDiagnosticCode::MetadataSkipped, + "PlotX skipped a non-ASCII Origin note after validating its block boundaries.", + Some(header_offset), + cursor.limits, + usage, + )?; + continue; + } + (Err(error), _) | (_, Err(error)) => return Err(error), + }; + + let name = copy_decoded_text(name, cursor.limits, usage)?; + let content = copy_decoded_text(content, cursor.limits, usage)?; + try_reserve(&mut notes, 1, "Origin project notes", cursor.limits, usage)?; + notes.push(OriginNote { name, content }); + } + Ok((notes, note_properties)) +} + +fn decode_window_name( + header: &[u8], + block_offset: usize, + limits: &OriginLimits, + usage: &mut OriginResourceUsage, +) -> Result { + if header.len() < WINDOW_HEADER_MIN_LEN { + return Err(OriginError::CorruptStructure { + offset: block_offset, + detail: format!( + "an Origin7V552 window header must contain at least {WINDOW_HEADER_MIN_LEN} bytes" + ), + }); + } + let end = checked_add( + WINDOW_NAME_OFFSET, + WINDOW_NAME_WIDTH, + "Origin window name range", + )?; + let field = header + .get(WINDOW_NAME_OFFSET..end) + .ok_or(OriginError::Truncated { + offset: block_offset, + needed: WINDOW_HEADER_MIN_LEN, + have: header.len(), + })?; + let length = field + .iter() + .position(|byte| *byte == 0) + .unwrap_or(field.len()); + let text = field.get(..length).ok_or(OriginError::ArithmeticOverflow { + resource: "Origin window name", + })?; + let text_offset = checked_add( + block_offset, + checked_add( + BLOCK_PREFIX_LEN, + WINDOW_NAME_OFFSET, + "Origin window name offset", + )?, + "Origin window name offset", + )?; + let text = validate_ascii(text, text_offset, "Origin window name")?; + copy_decoded_text(text, limits, usage) +} + +fn validate_nul_terminated_ascii<'a>( + bytes: &'a [u8], + offset: usize, + limits: &OriginLimits, + field: &'static str, +) -> Result<&'a str, OriginError> { + let Some(text) = bytes.strip_suffix(&[0]) else { + return Err(OriginError::CorruptStructure { + offset, + detail: format!("{field} must end with a NUL byte inside its framed block"), + }); + }; + if let Some(relative) = text.iter().position(|byte| *byte == 0) { + return Err(OriginError::CorruptStructure { + offset: checked_add(offset, relative, "embedded metadata NUL offset")?, + detail: format!("{field} contains an embedded NUL byte"), + }); + } + enforce_limit("string bytes", text.len(), limits.max_string_bytes)?; + validate_ascii(text, offset, field) +} + +fn validate_ascii<'a>( + bytes: &'a [u8], + offset: usize, + field: &'static str, +) -> Result<&'a str, OriginError> { + if let Some(relative) = bytes.iter().position(|byte| !byte.is_ascii()) { + return Err(OriginError::UnsupportedEncoding { + offset: checked_add(offset, relative, "metadata ASCII offset")?, + encoding: format!("non-ASCII byte in {field}"), + }); + } + std::str::from_utf8(bytes).map_err(|_| OriginError::UnsupportedEncoding { + offset, + encoding: format!("non-ASCII byte in {field}"), + }) +} + +fn format_f64(value: f64, offset: usize) -> Result { + let mut output = FixedText::default(); + write!(&mut output, "{value}").map_err(|_| OriginError::CorruptStructure { + offset, + detail: "an Origin parameter value could not be represented as bounded text".to_owned(), + })?; + Ok(output) +} + +#[derive(Default)] +struct FixedText { + bytes: [u8; FORMATTED_F64_CAPACITY], + len: usize, +} + +impl FixedText { + fn as_str(&self) -> Result<&str, OriginError> { + std::str::from_utf8( + self.bytes + .get(..self.len) + .ok_or(OriginError::ArithmeticOverflow { + resource: "formatted Origin parameter", + })?, + ) + .map_err(|_| OriginError::CorruptStructure { + offset: 0, + detail: "formatted Origin parameter text is not ASCII".to_owned(), + }) + } +} + +impl fmt::Write for FixedText { + fn write_str(&mut self, text: &str) -> fmt::Result { + let end = self.len.checked_add(text.len()).ok_or(fmt::Error)?; + let destination = self.bytes.get_mut(self.len..end).ok_or(fmt::Error)?; + destination.copy_from_slice(text.as_bytes()); + self.len = end; + Ok(()) + } +} + +fn copy_decoded_text( + text: &str, + limits: &OriginLimits, + usage: &mut OriginResourceUsage, +) -> Result { + enforce_limit("string bytes", text.len(), limits.max_string_bytes)?; + charge_text(text.len(), limits, usage)?; + copy_after_charge(text, "decoded Origin metadata") +} + +fn copy_parser_text( + text: FixedText, + limits: &OriginLimits, + usage: &mut OriginResourceUsage, +) -> Result { + let text = text.as_str()?; + enforce_limit("string bytes", text.len(), limits.max_string_bytes)?; + charge_parser(text.len(), limits, usage)?; + copy_after_charge(text, "formatted Origin parameter") +} + +fn copy_static_text( + text: &'static str, + limits: &OriginLimits, + usage: &mut OriginResourceUsage, +) -> Result { + charge_parser(text.len(), limits, usage)?; + copy_after_charge(text, "Origin diagnostic text") +} + +pub(super) fn copy_generated_text( + text: &'static str, + limits: &OriginLimits, + usage: &mut OriginResourceUsage, +) -> Result { + copy_static_text(text, limits, usage) +} + +fn copy_after_charge(text: &str, resource: &'static str) -> Result { + let mut output = String::new(); + output + .try_reserve_exact(text.len()) + .map_err(|_| OriginError::AllocationFailed { + resource, + requested: text.len(), + })?; + output.push_str(text); + Ok(output) +} + +pub(super) fn try_reserve( + values: &mut Vec, + additional: usize, + resource: &'static str, + limits: &OriginLimits, + usage: &mut OriginResourceUsage, +) -> Result<(), OriginError> { + let requested_len = checked_add(values.len(), additional, resource)?; + let old_capacity = values.capacity(); + if requested_len <= old_capacity || size_of::() == 0 { + return Ok(()); + } + + let minimum_delta = requested_len + .checked_sub(old_capacity) + .ok_or(OriginError::ArithmeticOverflow { resource })?; + let minimum_bytes = checked_mul(minimum_delta, size_of::(), resource)?; + let mut preflight = usage.clone(); + charge_parser(minimum_bytes, limits, &mut preflight)?; + + let available_bytes = limits + .max_parser_bytes + .saturating_sub(usage.parser_bytes) + .min( + limits + .max_total_owned_bytes + .saturating_sub(usage.total_owned_bytes), + ); + let geometric_capacity = if old_capacity == 0 { + requested_len + } else { + old_capacity.checked_mul(2).unwrap_or(requested_len) + }; + let desired_capacity = requested_len.max(geometric_capacity); + let desired_delta = desired_capacity + .checked_sub(old_capacity) + .ok_or(OriginError::ArithmeticOverflow { resource })?; + let affordable_delta = (available_bytes / size_of::()).min(desired_delta); + let target_capacity = checked_add(old_capacity, affordable_delta, resource)?; + let reserve_additional = target_capacity + .checked_sub(values.len()) + .ok_or(OriginError::ArithmeticOverflow { resource })?; + let planned_delta = target_capacity + .checked_sub(old_capacity) + .ok_or(OriginError::ArithmeticOverflow { resource })?; + let planned_bytes = checked_mul(planned_delta, size_of::(), resource)?; + values + .try_reserve_exact(reserve_additional) + .map_err(|_| OriginError::AllocationFailed { + resource, + requested: planned_bytes, + })?; + + let actual_delta = values + .capacity() + .checked_sub(old_capacity) + .ok_or(OriginError::ArithmeticOverflow { resource })?; + let actual_bytes = checked_mul(actual_delta, size_of::(), resource)?; + charge_parser(actual_bytes, limits, usage) +} + +pub(super) fn push_diagnostic( + diagnostics: &mut Vec, + code: OriginDiagnosticCode, + message: &'static str, + byte_offset: Option, + limits: &OriginLimits, + usage: &mut OriginResourceUsage, +) -> Result<(), OriginError> { + let message = copy_static_text(message, limits, usage)?; + try_reserve(diagnostics, 1, "Origin diagnostics", limits, usage)?; + diagnostics.push(OriginDiagnostic { + code, + severity: OriginDiagnosticSeverity::Warning, + location: byte_offset.map(|byte_offset| OriginObjectLocation { + workbook: None, + worksheet: None, + column: None, + byte_offset: Some(byte_offset), + }), + message, + }); + Ok(()) +} + +pub(super) fn push_summary( + summaries: &mut Vec, + kind: &'static str, + count: usize, + limits: &OriginLimits, + usage: &mut OriginResourceUsage, +) -> Result<(), OriginError> { + let kind = copy_static_text(kind, limits, usage)?; + try_reserve( + summaries, + 1, + "Origin unsupported-object summaries", + limits, + usage, + )?; + summaries.push(OriginUnsupportedObjectSummary { kind, count }); + Ok(()) +} + +fn charge_text( + bytes: usize, + limits: &OriginLimits, + usage: &mut OriginResourceUsage, +) -> Result<(), OriginError> { + let decoded = checked_add(usage.decoded_text_bytes, bytes, "decoded text bytes")?; + enforce_limit("decoded text bytes", decoded, limits.max_decoded_text_bytes)?; + charge_parser(bytes, limits, usage)?; + usage.decoded_text_bytes = decoded; + Ok(()) +} + +fn charge_parser( + bytes: usize, + limits: &OriginLimits, + usage: &mut OriginResourceUsage, +) -> Result<(), OriginError> { + let parser = checked_add(usage.parser_bytes, bytes, "parser bytes")?; + enforce_limit("parser bytes", parser, limits.max_parser_bytes)?; + let total = checked_add(usage.total_owned_bytes, bytes, "total owned bytes")?; + enforce_limit("total owned bytes", total, limits.max_total_owned_bytes)?; + usage.parser_bytes = parser; + usage.total_owned_bytes = total; + Ok(()) +} + +fn enforce_depth(depth: usize, limits: &OriginLimits) -> Result<(), OriginError> { + enforce_limit("metadata nesting depth", depth, limits.max_metadata_depth) +} + +fn enforce_limit(resource: &'static str, actual: usize, limit: usize) -> Result<(), OriginError> { + if actual > limit { + return Err(OriginError::LimitExceeded { + resource, + limit, + actual, + }); + } + Ok(()) +} + +fn require_data_block<'a>( + block: MetadataBlock<'a>, + detail: &'static str, +) -> Result<(usize, &'a [u8]), OriginError> { + match block { + MetadataBlock::Data { offset, payload } => Ok((offset, payload)), + MetadataBlock::Null { offset } => Err(OriginError::CorruptStructure { + offset, + detail: detail.to_owned(), + }), + } +} + +fn require_null_block(block: MetadataBlock<'_>, detail: &'static str) -> Result<(), OriginError> { + match block { + MetadataBlock::Null { .. } => Ok(()), + MetadataBlock::Data { offset, .. } => Err(OriginError::CorruptStructure { + offset, + detail: detail.to_owned(), + }), + } +} + +#[cfg(test)] +#[path = "metadata_tests.rs"] +mod metadata_tests; + +#[cfg(test)] +#[path = "metadata_modern_tests.rs"] +mod metadata_modern_tests; diff --git a/crates/io/src/origin/opj/metadata/cursor.rs b/crates/io/src/origin/opj/metadata/cursor.rs new file mode 100644 index 0000000..edac826 --- /dev/null +++ b/crates/io/src/origin/opj/metadata/cursor.rs @@ -0,0 +1,226 @@ +use std::mem::size_of; + +use crate::origin::{OriginError, OriginLimits}; + +use super::super::super::reader::checked_add; + +const LF: u8 = b'\n'; +const PARAMETER_VALUE_LEN: usize = size_of::() + 1; + +pub(super) enum MetadataBlock<'a> { + Null { offset: usize }, + Data { offset: usize, payload: &'a [u8] }, +} + +pub(super) struct MetadataCursor<'a> { + bytes: &'a [u8], + offset: usize, + base_offset: usize, + metadata_records: usize, + pub(super) limits: &'a OriginLimits, +} + +impl<'a> MetadataCursor<'a> { + pub(super) fn new(bytes: &'a [u8], base_offset: usize, limits: &'a OriginLimits) -> Self { + Self { + bytes, + offset: 0, + base_offset, + metadata_records: 0, + limits, + } + } + + pub(super) fn charge_record(&mut self) -> Result<(), OriginError> { + let actual = checked_add(self.metadata_records, 1, "metadata records")?; + enforce_limit("metadata records", actual, self.limits.max_metadata_records)?; + self.metadata_records = actual; + Ok(()) + } + + pub(super) fn metadata_records(&self) -> usize { + self.metadata_records + } + + pub(super) fn read_block(&mut self) -> Result, OriginError> { + let start = self.offset; + let size_end = checked_add(start, size_of::(), "metadata block size")?; + let size_bytes = self.bytes.get(start..size_end).ok_or_else(|| { + self.truncated( + start, + size_of::(), + self.bytes.len().saturating_sub(start), + ) + })?; + let size_array: [u8; 4] = size_bytes + .try_into() + .map_err(|_| self.truncated(start, size_of::(), size_bytes.len()))?; + let size = usize::try_from(u32::from_le_bytes(size_array)).map_err(|_| { + OriginError::ArithmeticOverflow { + resource: "metadata block size", + } + })?; + enforce_limit("block bytes", size, self.limits.max_block_bytes)?; + + let size_lf = size_end; + self.require_lf(size_lf, "metadata block size delimiter")?; + let payload_start = checked_add(size_lf, 1, "metadata block payload")?; + if size == 0 { + self.offset = payload_start; + return Ok(MetadataBlock::Null { + offset: self.absolute(start)?, + }); + } + + let payload_end = checked_add(payload_start, size, "metadata block payload")?; + let payload = self.bytes.get(payload_start..payload_end).ok_or_else(|| { + self.truncated( + payload_start, + size, + self.bytes.len().saturating_sub(payload_start), + ) + })?; + self.require_lf(payload_end, "metadata block payload delimiter")?; + self.offset = checked_add(payload_end, 1, "metadata block end")?; + Ok(MetadataBlock::Data { + offset: self.absolute(start)?, + payload, + }) + } + + pub(super) fn read_line(&mut self) -> Result<(usize, &'a [u8]), OriginError> { + let start = self.offset; + let available = self + .bytes + .get(start..) + .ok_or_else(|| self.truncated(start, 1, self.bytes.len().saturating_sub(start)))?; + let oversize = checked_add(self.limits.max_string_bytes, 1, "metadata line bound")?; + let scan_len = available.len().min(oversize); + let scan = available + .get(..scan_len) + .ok_or(OriginError::ArithmeticOverflow { + resource: "metadata line scan", + })?; + let Some(relative_lf) = scan.iter().position(|byte| *byte == LF) else { + if available.len() > self.limits.max_string_bytes { + return Err(OriginError::LimitExceeded { + resource: "string bytes", + limit: self.limits.max_string_bytes, + actual: oversize, + }); + } + return Err(self.truncated( + checked_add(start, available.len(), "metadata line end")?, + 1, + 0, + )); + }; + enforce_limit("string bytes", relative_lf, self.limits.max_string_bytes)?; + let line = available + .get(..relative_lf) + .ok_or(OriginError::ArithmeticOverflow { + resource: "metadata line", + })?; + self.offset = checked_add( + start, + checked_add(relative_lf, 1, "metadata line length")?, + "metadata line end", + )?; + Ok((self.absolute(start)?, line)) + } + + pub(super) fn read_parameter_value(&mut self) -> Result<[u8; 8], OriginError> { + let start = self.offset; + let end = checked_add(start, PARAMETER_VALUE_LEN, "parameter value")?; + let bytes = self.bytes.get(start..end).ok_or_else(|| { + self.truncated( + start, + PARAMETER_VALUE_LEN, + self.bytes.len().saturating_sub(start), + ) + })?; + let value = bytes + .get(..size_of::()) + .ok_or_else(|| self.truncated(start, size_of::(), bytes.len()))?; + let delimiter = bytes + .get(size_of::()) + .copied() + .ok_or_else(|| self.truncated(start, PARAMETER_VALUE_LEN, bytes.len()))?; + if delimiter != LF { + return Err(OriginError::CorruptStructure { + offset: self.absolute(checked_add(start, size_of::(), "parameter LF")?)?, + detail: "an Origin parameter value must end with LF".to_owned(), + }); + } + let value: [u8; 8] = value + .try_into() + .map_err(|_| self.truncated(start, size_of::(), value.len()))?; + self.offset = end; + Ok(value) + } + + pub(super) fn read_exact(&mut self, length: usize) -> Result<&'a [u8], OriginError> { + let start = self.offset; + let end = checked_add(start, length, "terminal OPJ record")?; + let bytes = self + .bytes + .get(start..end) + .ok_or_else(|| self.truncated(start, length, self.bytes.len().saturating_sub(start)))?; + self.offset = end; + Ok(bytes) + } + + pub(super) fn skip_exact(&mut self, length: usize) -> Result<(), OriginError> { + let _ = self.read_exact(length)?; + Ok(()) + } + + pub(super) fn relative_offset(&self) -> usize { + self.offset + } + + pub(super) fn absolute_offset(&self) -> Result { + self.absolute(self.offset) + } + + pub(super) fn remaining(&self) -> usize { + self.bytes.len().saturating_sub(self.offset) + } + + fn require_lf(&self, offset: usize, field: &'static str) -> Result<(), OriginError> { + let byte = + self.bytes.get(offset).copied().ok_or_else(|| { + self.truncated(offset, 1, self.bytes.len().saturating_sub(offset)) + })?; + if byte != LF { + return Err(OriginError::CorruptStructure { + offset: self.absolute(offset)?, + detail: format!("{field} must be LF"), + }); + } + Ok(()) + } + + fn absolute(&self, relative: usize) -> Result { + checked_add(self.base_offset, relative, "metadata file offset") + } + + fn truncated(&self, offset: usize, needed: usize, have: usize) -> OriginError { + OriginError::Truncated { + offset: self.base_offset.saturating_add(offset), + needed, + have, + } + } +} + +fn enforce_limit(resource: &'static str, actual: usize, limit: usize) -> Result<(), OriginError> { + if actual > limit { + return Err(OriginError::LimitExceeded { + resource, + limit, + actual, + }); + } + Ok(()) +} diff --git a/crates/io/src/origin/opj/metadata/tail.rs b/crates/io/src/origin/opj/metadata/tail.rs new file mode 100644 index 0000000..0a80002 --- /dev/null +++ b/crates/io/src/origin/opj/metadata/tail.rs @@ -0,0 +1,212 @@ +use std::mem::size_of; + +use crate::origin::{ + OriginDiagnostic, OriginDiagnosticCode, OriginError, OriginResourceUsage, + OriginUnsupportedObjectSummary, +}; + +use super::cursor::{MetadataBlock, MetadataCursor}; +use super::{push_diagnostic, push_summary}; +use crate::origin::reader::checked_add; + +const TREE_SPAN_PAYLOAD_LEN: usize = size_of::(); + +// The unchanged public OpenOPJ MIT fixture has its first attachment header at +// absolute byte offset 276,350. Its bytes encode a 52-byte header, type +// 0x7fca0459, and an OLE compound signature immediately after that header. +// These Origin7V552 constants are derived directly from those fixture bytes; +// the pinned source and license are recorded in the fixture README. +const ATTACHMENT_HEADER_LEN: usize = 52; +const ATTACHMENT_TYPE_OLE: usize = 0x7fca_0459; +const OLE_COMPOUND_SIGNATURE: &[u8] = &[0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1]; + +pub(super) fn parse( + cursor: &mut MetadataCursor<'_>, + diagnostics: &mut Vec, + unsupported_objects: &mut Vec, + usage: &mut OriginResourceUsage, +) -> Result<(), OriginError> { + parse_project_tree(cursor)?; + push_summary(unsupported_objects, "project tree", 1, cursor.limits, usage)?; + push_diagnostic( + diagnostics, + OriginDiagnosticCode::UnsupportedObjectSkipped, + "PlotX preserved worksheet data but did not import the bounded Origin project tree.", + None, + cursor.limits, + usage, + )?; + + let attachment_count = parse_attachments(cursor)?; + if attachment_count > 0 { + push_summary( + unsupported_objects, + "embedded attachments", + attachment_count, + cursor.limits, + usage, + )?; + push_diagnostic( + diagnostics, + OriginDiagnosticCode::UnsupportedObjectSkipped, + "PlotX did not extract, open, or execute bounded embedded Origin attachments.", + None, + cursor.limits, + usage, + )?; + } + Ok(()) +} + +fn parse_project_tree(cursor: &mut MetadataCursor<'_>) -> Result<(), OriginError> { + let tree_start = cursor.relative_offset(); + let (block_offset, span_payload) = match cursor.read_block()? { + MetadataBlock::Data { offset, payload } => (offset, payload), + MetadataBlock::Null { offset } => { + return Err(OriginError::CorruptStructure { + offset, + detail: "the Origin project tree requires a bounded span block".to_owned(), + }); + } + }; + cursor.charge_record()?; + if span_payload.len() != TREE_SPAN_PAYLOAD_LEN { + return Err(OriginError::CorruptStructure { + offset: block_offset, + detail: "the Origin7V552 project-tree span must be a 4-byte value".to_owned(), + }); + } + let span_bytes: [u8; 4] = + span_payload + .try_into() + .map_err(|_| OriginError::CorruptStructure { + offset: block_offset, + detail: "the Origin7V552 project-tree span is incomplete".to_owned(), + })?; + let declared_span = usize::try_from(u32::from_le_bytes(span_bytes)).map_err(|_| { + OriginError::ArithmeticOverflow { + resource: "Origin project-tree span", + } + })?; + enforce_limit("block bytes", declared_span, cursor.limits.max_block_bytes)?; + let consumed = cursor.relative_offset().checked_sub(tree_start).ok_or( + OriginError::ArithmeticOverflow { + resource: "Origin project-tree span", + }, + )?; + let remaining_span = + declared_span + .checked_sub(consumed) + .ok_or_else(|| OriginError::CorruptStructure { + offset: block_offset, + detail: "the Origin project-tree span is shorter than its framing".to_owned(), + })?; + + // OpenOPJ documents the project tree as the section following notes. The + // public MIT fixture independently supplies this outer span, so PlotX can + // skip exactly to the attachment header without interpreting tree names, + // paths, or executable content. + // https://github.com/jgonera/openopj/blob/42ddcf1eb3a490744c54fca0a4ed6fe7a5e723ca/docs/opj_format.markdown + cursor.skip_exact(remaining_span) +} + +fn parse_attachments(cursor: &mut MetadataCursor<'_>) -> Result { + let mut count = 0_usize; + while cursor.remaining() > 0 { + cursor.charge_record()?; + let header_offset = cursor.absolute_offset()?; + let header = cursor.read_exact(ATTACHMENT_HEADER_LEN)?; + let header_len = read_u32(header, 0, header_offset, "attachment header size")?; + if header_len != ATTACHMENT_HEADER_LEN { + return Err(OriginError::UnsupportedFeature { + feature: "an Origin attachment header layout is not verified".to_owned(), + }); + } + let attachment_type = read_u32(header, 4, header_offset, "attachment type")?; + if attachment_type != ATTACHMENT_TYPE_OLE { + return Err(OriginError::UnsupportedFeature { + feature: "an embedded Origin attachment type is not supported".to_owned(), + }); + } + let payload_size = read_u32(header, 8, header_offset, "attachment payload size")?; + enforce_limit("block bytes", payload_size, cursor.limits.max_block_bytes)?; + let payload = cursor.read_exact(payload_size)?; + if !payload.starts_with(OLE_COMPOUND_SIGNATURE) { + return Err(OriginError::UnsupportedFeature { + feature: "an embedded Origin attachment payload is not a verified OLE object" + .to_owned(), + }); + } + count = checked_add(count, 1, "embedded Origin attachments")?; + } + Ok(count) +} + +fn read_u32( + bytes: &[u8], + offset: usize, + file_offset: usize, + resource: &'static str, +) -> Result { + let end = checked_add(offset, size_of::(), resource)?; + let value = bytes.get(offset..end).ok_or(OriginError::Truncated { + offset: checked_add(file_offset, offset, resource)?, + needed: size_of::(), + have: bytes.len().saturating_sub(offset), + })?; + let value_offset = checked_add(file_offset, offset, resource)?; + let value: [u8; 4] = value.try_into().map_err(|_| OriginError::Truncated { + offset: value_offset, + needed: size_of::(), + have: value.len(), + })?; + usize::try_from(u32::from_le_bytes(value)) + .map_err(|_| OriginError::ArithmeticOverflow { resource }) +} + +fn enforce_limit(resource: &'static str, actual: usize, limit: usize) -> Result<(), OriginError> { + if actual > limit { + return Err(OriginError::LimitExceeded { + resource, + limit, + actual, + }); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::{ATTACHMENT_HEADER_LEN, ATTACHMENT_TYPE_OLE, OLE_COMPOUND_SIGNATURE}; + + const OPENOPJ_FIXTURE: &[u8] = + include_bytes!("../../../../tests/fixtures/origin/test-origin-7.0552.opj"); + const ATTACHMENT_HEADER_OFFSET: usize = 276_350; + + #[test] + fn attachment_constants_match_the_pinned_openopj_fixture_bytes() { + let header_end = ATTACHMENT_HEADER_OFFSET + .checked_add(ATTACHMENT_HEADER_LEN) + .expect("fixture offsets are small"); + let header = OPENOPJ_FIXTURE + .get(ATTACHMENT_HEADER_OFFSET..header_end) + .expect("pinned fixture contains the documented attachment header"); + + let header_len_bytes = u32::try_from(ATTACHMENT_HEADER_LEN) + .expect("attachment header length fits u32") + .to_le_bytes(); + let attachment_type_bytes = u32::try_from(ATTACHMENT_TYPE_OLE) + .expect("attachment type fits u32") + .to_le_bytes(); + assert_eq!(header.get(0..4), Some(header_len_bytes.as_slice())); + assert_eq!(header.get(4..8), Some(attachment_type_bytes.as_slice())); + + let signature_end = header_end + .checked_add(OLE_COMPOUND_SIGNATURE.len()) + .expect("fixture offsets are small"); + assert_eq!( + OPENOPJ_FIXTURE.get(header_end..signature_end), + Some(OLE_COMPOUND_SIGNATURE) + ); + } +} diff --git a/crates/io/src/origin/opj/metadata_modern_tests.rs b/crates/io/src/origin/opj/metadata_modern_tests.rs new file mode 100644 index 0000000..7756828 --- /dev/null +++ b/crates/io/src/origin/opj/metadata_modern_tests.rs @@ -0,0 +1,191 @@ +use std::panic::catch_unwind; + +use crate::origin::{OriginCell, OriginError, OriginLimits, read_origin}; + +const SIGNATURE: &[u8] = b"CPYA 4.3268 195 W64 #\n"; +const GLOBAL_HEADER_LEN: usize = 115; +const DATA_HEADER_LEN: usize = 147; +const EMPTY_F64: f64 = -1.23456789E-300; + +fn push_block(bytes: &mut Vec, payload: Option<&[u8]>) { + let payload = payload.unwrap_or_default(); + bytes.extend_from_slice(&u32::try_from(payload.len()).unwrap().to_le_bytes()); + bytes.push(b'\n'); + if !payload.is_empty() { + bytes.extend_from_slice(payload); + bytes.push(b'\n'); + } +} + +fn modern_header(name: &str, row_count: u32) -> [u8; DATA_HEADER_LEN] { + let mut header = [0_u8; DATA_HEADER_LEN]; + header[0x16..0x18].copy_from_slice(&0x6121_u16.to_le_bytes()); + header[0x18] = 0x03; + header[0x19..0x1d].copy_from_slice(&row_count.to_le_bytes()); + header[0x1d..0x21].copy_from_slice(&0_u32.to_le_bytes()); + header[0x21..0x25].copy_from_slice(&row_count.to_le_bytes()); + header[0x3d] = 10; + header[0x3f] = 0x10; + header[0x58..0x58 + name.len()].copy_from_slice(name.as_bytes()); + header[0x71..0x73].copy_from_slice(&0x10ca_u16.to_le_bytes()); + header +} + +fn numeric_slots(values: &[f64]) -> Vec { + values + .iter() + .flat_map(|value| [0, 0].into_iter().chain(value.to_le_bytes())) + .collect() +} + +fn push_window(bytes: &mut Vec, name: &str) { + let mut header = [0_u8; 27]; + header[2..2 + name.len()].copy_from_slice(name.as_bytes()); + push_block(bytes, Some(&header)); + push_block(bytes, None); +} + +struct ModernProject { + bytes: Vec, + window_list_end: usize, + final_window_terminator_start: usize, +} + +fn modern_project(records: &[(&str, &[f64])], windows: &[&str]) -> ModernProject { + let mut bytes = SIGNATURE.to_vec(); + let mut global = [0_u8; GLOBAL_HEADER_LEN]; + global[0x1b..0x23].copy_from_slice(&9.510195_f64.to_le_bytes()); + push_block(&mut bytes, Some(&global)); + push_block(&mut bytes, None); + + for (name, values) in records { + push_block( + &mut bytes, + Some(&modern_header(name, u32::try_from(values.len()).unwrap())), + ); + push_block(&mut bytes, Some(&numeric_slots(values))); + push_block(&mut bytes, None); + } + push_block(&mut bytes, None); + + for name in windows { + push_window(&mut bytes, name); + } + let final_window_terminator_start = bytes.len(); + push_block(&mut bytes, None); + let window_list_end = bytes.len(); + + // This payload deliberately resembles a dataset name. A windows-only + // parser must report and ignore it rather than scan it for table markers. + bytes.extend_from_slice(b"opaque Book1_FAKE project tail\0\xff"); + ModernProject { + bytes, + window_list_end, + final_window_terminator_start, + } +} + +#[test] +fn modern_assembles_worksheets_and_reports_the_opaque_tail() { + let fixture = modern_project( + &[("Book1_A", &[0.05, 0.10]), ("Book1_B", &[1.5, 2.5])], + &["Book1", "Graph1"], + ); + let project = read_origin(&fixture.bytes, OriginLimits::default()).unwrap(); + + assert_eq!(project.workbooks.len(), 1); + let worksheet = &project.workbooks[0].worksheets[0]; + assert_eq!(project.workbooks[0].name, "Book1"); + assert_eq!(worksheet.row_count, 2); + assert_eq!(worksheet.columns.len(), 2); + assert_eq!(worksheet.columns[0].name, "A"); + assert_eq!(worksheet.columns[1].name, "B"); + assert_eq!( + worksheet.columns[0].cells, + [OriginCell::Float(0.05), OriginCell::Float(0.10)] + ); + assert!(project.diagnostics.iter().any(|diagnostic| { + diagnostic + .message + .contains("remaining Origin 9.51 project objects") + })); + assert!(!worksheet.columns.iter().any(|column| column.name == "FAKE")); +} + +#[test] +fn modern_accepts_an_exact_window_boundary_without_inventing_a_tail() { + let fixture = modern_project(&[("Book1_A", &[1.0])], &["Book1"]); + let project = read_origin( + &fixture.bytes[..fixture.window_list_end], + OriginLimits::default(), + ) + .unwrap(); + assert_eq!(project.workbooks.len(), 1); + assert!(!project.diagnostics.iter().any(|diagnostic| { + diagnostic + .message + .contains("remaining Origin 9.51 project objects") + })); +} + +#[test] +fn modern_rejects_every_prefix_before_the_window_list_terminator() { + let fixture = modern_project(&[("Book1_A", &[1.0])], &["Book1"]); + for end in 0..fixture.window_list_end { + let outcome = catch_unwind(|| read_origin(&fixture.bytes[..end], OriginLimits::default())); + assert!(outcome.is_ok(), "prefix {end} panicked"); + assert!(outcome.unwrap().is_err(), "prefix {end} succeeded"); + } +} + +#[test] +fn modern_rejects_a_missing_window_list_terminator() { + let fixture = modern_project(&[("Book1_A", &[1.0])], &["Book1"]); + let bytes = &fixture.bytes[..fixture.final_window_terminator_start]; + assert!(matches!( + read_origin(bytes, OriginLimits::default()), + Err(OriginError::Truncated { .. }) + )); +} + +#[test] +fn modern_enforces_the_window_record_limit_before_association() { + let fixture = modern_project(&[("Book1_A", &[1.0])], &["Book1", "Graph1"]); + let limits = OriginLimits { + max_window_records: 1, + ..OriginLimits::default() + }; + assert!(matches!( + read_origin(&fixture.bytes, limits), + Err(OriginError::LimitExceeded { + resource: "window records", + limit: 1, + actual: 2, + }) + )); +} + +#[test] +fn modern_rejects_ambiguous_or_absent_worksheet_associations() { + for fixture in [ + modern_project(&[("Book1_A", &[1.0])], &["Book1", "Book1"]), + modern_project(&[], &["Graph1"]), + ] { + assert_eq!( + read_origin(&fixture.bytes, OriginLimits::default()).unwrap_err(), + OriginError::NoSupportedWorksheet + ); + } +} + +#[test] +fn modern_empty_columns_remain_null_after_worksheet_padding() { + let fixture = modern_project( + &[("Book1_A", &[1.0, 2.0]), ("Book1_B", &[EMPTY_F64])], + &["Book1"], + ); + let project = read_origin(&fixture.bytes, OriginLimits::default()).unwrap(); + let worksheet = &project.workbooks[0].worksheets[0]; + assert_eq!(worksheet.row_count, 2); + assert_eq!(worksheet.columns[1].cells, [OriginCell::Null]); +} diff --git a/crates/io/src/origin/opj/metadata_tests.rs b/crates/io/src/origin/opj/metadata_tests.rs new file mode 100644 index 0000000..a04b487 --- /dev/null +++ b/crates/io/src/origin/opj/metadata_tests.rs @@ -0,0 +1,632 @@ +use crate::origin::{ + OriginCell, OriginDiagnosticCode, OriginError, OriginLimits, OriginMetadataEntry, OriginNote, + OriginResourceUsage, read_origin, +}; + +use std::mem::size_of; + +const OPENOPJ_FIXTURE: &[u8] = + include_bytes!("../../../tests/fixtures/origin/test-origin-7.0552.opj"); + +#[test] +fn imports_real_fixture_parameters_and_notes() { + let project = read_origin(OPENOPJ_FIXTURE, OriginLimits::default()) + .expect("the licensed OpenOPJ fixture should import"); + + assert!( + project + .parameters + .iter() + .any(|entry| entry.key == "ERR" && entry.value == "1") + ); + assert!(project.notes.iter().any(|note| { + note.name == "Results" && note.content == "Data1 Temperature:\t25.10242\r\n\r\n" + })); +} + +const SIGNATURE: &[u8] = b"CPYA 4.2673 552#\n"; +const ORIGIN_HEADER_LEN: usize = 39; +const DATA_HEADER_LEN: usize = 123; + +fn push_block(bytes: &mut Vec, payload: Option<&[u8]>) { + let payload = payload.unwrap_or_default(); + bytes.extend_from_slice(&u32::try_from(payload.len()).unwrap().to_le_bytes()); + bytes.push(b'\n'); + if !payload.is_empty() { + bytes.extend_from_slice(payload); + bytes.push(b'\n'); + } +} + +fn data_header(name: &str, supported: bool, row_count: u32) -> [u8; DATA_HEADER_LEN] { + let mut header = [0_u8; DATA_HEADER_LEN]; + header[0x16..0x18].copy_from_slice(&0x6001_u16.to_le_bytes()); + header[0x18] = 1; + header[0x19..0x1d].copy_from_slice(&row_count.to_le_bytes()); + header[0x1d..0x21].copy_from_slice(&0_u32.to_le_bytes()); + header[0x21..0x25].copy_from_slice(&row_count.to_le_bytes()); + header[0x3d] = 8; + header[0x3f] = u8::from(!supported); + let name = name.as_bytes(); + header[0x58..0x58 + name.len()].copy_from_slice(name); + header[0x71..0x73].copy_from_slice(&0x10ca_u16.to_le_bytes()); + header +} + +fn push_window(bytes: &mut Vec, name: &[u8]) { + let mut header = [0_u8; 27]; + header[2..2 + name.len()].copy_from_slice(name); + push_block(bytes, Some(&header)); + push_block(bytes, None); +} + +fn synthetic_project_with_parameters( + records: &[(&str, bool, u32)], + windows: &[&[u8]], + parameters: &[(&[u8], f64)], +) -> Vec { + let mut bytes = SIGNATURE.to_vec(); + let mut origin_header = [0_u8; ORIGIN_HEADER_LEN]; + origin_header[0x1b..0x23].copy_from_slice(&7.0552_f64.to_le_bytes()); + push_block(&mut bytes, Some(&origin_header)); + push_block(&mut bytes, None); + + for (name, supported, row_count) in records { + push_block(&mut bytes, Some(&data_header(name, *supported, *row_count))); + let mut content = Vec::new(); + for _ in 0..*row_count { + content.extend_from_slice(&1.5_f64.to_le_bytes()); + } + push_block(&mut bytes, Some(&content)); + push_block(&mut bytes, None); + } + push_block(&mut bytes, None); + + for name in windows { + push_window(&mut bytes, name); + } + push_block(&mut bytes, None); + + for (name, value) in parameters { + bytes.extend_from_slice(name); + bytes.push(b'\n'); + bytes.extend_from_slice(&value.to_le_bytes()); + bytes.push(b'\n'); + } + bytes.extend_from_slice(b"\0\n"); + push_block(&mut bytes, None); + + push_block(&mut bytes, Some(&[0_u8; 4])); + push_block(&mut bytes, Some(b"Results\0")); + push_block(&mut bytes, Some(b"ok\0")); + push_block(&mut bytes, None); + push_block(&mut bytes, Some(&10_u32.to_le_bytes())); + bytes +} + +fn synthetic_project(records: &[(&str, bool)], windows: &[&[u8]]) -> Vec { + let records = records + .iter() + .map(|(name, supported)| (*name, *supported, 1)) + .collect::>(); + synthetic_project_with_parameters(&records, windows, &[(b"ERR".as_slice(), 1.0)]) +} + +fn insert_layer_with_two_nested_lists(bytes: &mut Vec) { + let marker = [27_u32.to_le_bytes().as_slice(), b"\n\0\0Book"].concat(); + let window = bytes + .windows(marker.len()) + .position(|candidate| candidate == marker) + .expect("synthetic window framing"); + let layer_list = window + 5 + 27 + 1; + let mut layer = Vec::new(); + push_block(&mut layer, Some(&[0_u8])); + push_block(&mut layer, Some(&[1_u8])); + for _ in 1..4 { + push_block(&mut layer, None); + } + push_block(&mut layer, None); + push_block(&mut layer, Some(&[1_u8])); + push_block(&mut layer, None); + for _ in 0..5 { + push_block(&mut layer, None); + } + bytes.splice(layer_list..layer_list, layer); +} + +fn only_column(project: &crate::origin::OriginProject) -> &crate::origin::OriginColumn { + &project.workbooks[0].worksheets[0].columns[0] +} + +#[test] +fn chooses_the_longest_validated_window_prefix() { + let bytes = synthetic_project(&[("BookLong_Value", true)], &[b"Book", b"BookLong"]); + let project = read_origin(&bytes, OriginLimits::default()).unwrap(); + + assert_eq!(project.workbooks.len(), 1); + assert_eq!(project.workbooks[0].name, "BookLong"); + assert_eq!(only_column(&project).name, "Value"); + assert_eq!(only_column(&project).cells, [OriginCell::Float(1.5)]); +} + +#[test] +fn requires_an_underscore_between_window_and_column_names() { + let bytes = synthetic_project(&[("BookValue", true)], &[b"Book"]); + assert!(matches!( + read_origin(&bytes, OriginLimits::default()), + Err(OriginError::NoSupportedWorksheet) + )); +} + +#[test] +fn preserves_non_identifier_column_suffixes_after_an_exact_window_prefix() { + let bytes = synthetic_project(&[("Book_1", true), ("Book_A-B", true)], &[b"Book"]); + let project = read_origin(&bytes, OriginLimits::default()).unwrap(); + + assert_eq!(project.workbooks.len(), 1); + assert_eq!(project.workbooks[0].name, "Book"); + assert_eq!( + project.workbooks[0].worksheets[0] + .columns + .iter() + .map(|column| column.name.as_str()) + .collect::>(), + ["1", "A-B"] + ); +} + +#[test] +fn ambiguous_or_missing_window_associations_are_not_imported() { + for windows in [ + &[b"Other".as_slice()][..], + &[b"Book".as_slice(), b"Book".as_slice()][..], + ] { + let dataset = if windows.len() == 1 { + "Orphan_Value" + } else { + "Book_Value" + }; + let bytes = synthetic_project(&[(dataset, true)], windows); + assert!(matches!( + read_origin(&bytes, OriginLimits::default()), + Err(OriginError::NoSupportedWorksheet) + )); + } +} + +#[test] +fn unmatched_columns_are_skipped_without_inventing_cross_dataset_alignment() { + let bytes = synthetic_project( + &[ + ("Book_Good", true), + ("Orphan_Value", true), + ("Other_Value", true), + ], + &[b"Book"], + ); + let project = read_origin(&bytes, OriginLimits::default()).unwrap(); + + assert_eq!(project.workbooks.len(), 1); + assert_eq!(project.workbooks[0].name, "Book"); + assert_eq!(project.workbooks[0].worksheets[0].columns.len(), 1); + assert_eq!(only_column(&project).name, "Good"); + assert_eq!( + project + .unsupported_objects + .iter() + .find(|summary| summary.kind == "worksheet columns") + .map(|summary| summary.count), + Some(2) + ); +} + +#[test] +fn skips_an_independently_framed_unsupported_column() { + let bytes = synthetic_project(&[("Book_Good", true), ("Book_Unknown", false)], &[b"Book"]); + let project = read_origin(&bytes, OriginLimits::default()).unwrap(); + + assert_eq!(project.workbooks[0].worksheets[0].columns.len(), 1); + assert_eq!(only_column(&project).name, "Good"); + assert!( + project.diagnostics.iter().any(|diagnostic| { + diagnostic.code == OriginDiagnosticCode::UnsupportedColumnSkipped + }) + ); + assert!( + project + .unsupported_objects + .iter() + .any(|summary| { summary.kind == "worksheet columns" && summary.count == 1 }) + ); +} + +#[test] +fn does_not_treat_an_exact_window_name_as_a_worksheet_column() { + let bytes = synthetic_project(&[("Matrix", true)], &[b"Matrix"]); + assert!(matches!( + read_origin(&bytes, OriginLimits::default()), + Err(OriginError::NoSupportedWorksheet) + )); +} + +#[test] +fn rejects_project_with_only_zero_row_supported_columns() { + let bytes = synthetic_project_with_parameters( + &[("Book_Empty", true, 0)], + &[b"Book"], + &[(b"ERR".as_slice(), 1.0)], + ); + + assert!(matches!( + read_origin(&bytes, OriginLimits::default()), + Err(OriginError::NoSupportedWorksheet) + )); +} + +#[test] +fn keeps_workbook_with_a_nonempty_supported_column() { + let bytes = synthetic_project_with_parameters( + &[("Book_Empty", true, 0), ("Book_Value", true, 1)], + &[b"Book"], + &[(b"ERR".as_slice(), 1.0)], + ); + + let project = read_origin(&bytes, OriginLimits::default()).unwrap(); + + assert_eq!(project.workbooks.len(), 1); + assert_eq!(project.workbooks[0].worksheets[0].row_count, 1); + assert_eq!(project.workbooks[0].worksheets[0].columns.len(), 2); + assert_eq!(project.resource_usage.workbooks, 1); + assert_eq!(project.resource_usage.worksheets, 1); + assert_eq!(project.resource_usage.columns, 2); + assert_eq!(project.resource_usage.cells, 1); +} + +#[test] +fn skips_bounded_non_ascii_note_metadata_with_a_warning() { + let mut bytes = synthetic_project(&[("Book_A", true)], &[b"Book"]); + let note_content = bytes + .windows(3) + .rposition(|window| window == b"ok\0") + .expect("synthetic note content"); + bytes[note_content] = 0x80; + + let project = read_origin(&bytes, OriginLimits::default()).unwrap(); + assert!(project.notes.is_empty()); + assert!( + project + .diagnostics + .iter() + .any(|diagnostic| { diagnostic.code == OriginDiagnosticCode::MetadataSkipped }) + ); +} + +#[test] +fn enforces_workbook_and_metadata_limits() { + let bytes = synthetic_project(&[("One_A", true), ("Two_B", true)], &[b"One", b"Two"]); + let workbook_limits = OriginLimits { + max_workbooks: 1, + ..OriginLimits::default() + }; + assert!(matches!( + read_origin(&bytes, workbook_limits), + Err(OriginError::LimitExceeded { + resource: "workbooks", + limit: 1, + actual: 2, + }) + )); + + let string_limits = OriginLimits { + max_string_bytes: 2, + ..OriginLimits::default() + }; + assert!(matches!( + read_origin(&bytes, string_limits), + Err(OriginError::LimitExceeded { + resource: "string bytes", + limit: 2, + .. + }) + )); +} + +#[test] +fn repeated_single_item_reservations_grow_logarithmically_and_charge_capacity() { + let limits = OriginLimits::default(); + let mut usage = OriginResourceUsage::default(); + let mut values = Vec::::new(); + let mut capacity_changes = 0_usize; + + for value in 0..4096_u64 { + let previous_capacity = values.capacity(); + super::try_reserve(&mut values, 1, "test metadata", &limits, &mut usage).unwrap(); + if values.capacity() != previous_capacity { + capacity_changes += 1; + } + values.push(value); + } + + assert!( + capacity_changes <= 16, + "single-item appends reallocated {capacity_changes} times" + ); + assert_eq!(usage.parser_bytes, values.capacity() * size_of::()); + assert_eq!(usage.total_owned_bytes, usage.parser_bytes); +} + +#[test] +fn spare_vector_capacity_is_not_charged_as_a_new_allocation() { + let limits = OriginLimits::default(); + let mut usage = OriginResourceUsage::default(); + let mut values = Vec::::with_capacity(8); + let original_capacity = values.capacity(); + + super::try_reserve(&mut values, 1, "test metadata", &limits, &mut usage).unwrap(); + + assert_eq!(values.capacity(), original_capacity); + assert_eq!(usage.parser_bytes, 0); + assert_eq!(usage.total_owned_bytes, 0); +} + +#[test] +fn existing_capacity_does_not_overflow_an_unbounded_custom_budget() { + let limits = OriginLimits { + max_parser_bytes: usize::MAX, + max_total_owned_bytes: usize::MAX, + ..OriginLimits::default() + }; + let mut usage = OriginResourceUsage::default(); + let mut values = vec![0_u8; 8]; + let original_capacity = values.capacity(); + + super::try_reserve(&mut values, 1, "test metadata", &limits, &mut usage).unwrap(); + + assert!(values.capacity() > original_capacity); + assert_eq!(usage.parser_bytes, values.capacity() - original_capacity); + assert_eq!(usage.total_owned_bytes, usage.parser_bytes); +} + +#[test] +fn rejects_excess_window_records_before_dataset_association() { + let window_names = (0..1025) + .map(|index| format!("W{index:04}")) + .collect::>(); + let window_name_bytes = window_names + .iter() + .map(|name| name.as_bytes()) + .collect::>(); + let bytes = synthetic_project(&[("W0000_A", true)], &window_name_bytes); + + assert!(matches!( + read_origin(&bytes, OriginLimits::default()), + Err(OriginError::LimitExceeded { + resource: "window records", + limit: 1024, + actual: 1025, + }) + )); +} + +#[test] +fn metadata_records_do_not_consume_the_data_column_limit() { + let bytes = synthetic_project_with_parameters( + &[("Book_A", true, 1)], + &[b"Book"], + &[ + (b"ERR".as_slice(), 1.0), + (b"ALPHA".as_slice(), 2.0), + (b"BETA".as_slice(), 3.0), + ], + ); + let limits = OriginLimits { + max_columns: 1, + ..OriginLimits::default() + }; + let project = read_origin(&bytes, limits).unwrap(); + + assert_eq!(project.workbooks[0].worksheets[0].columns.len(), 1); + assert_eq!(project.parameters.len(), 3); +} + +#[test] +fn metadata_record_limit_is_cumulative_across_nested_lists() { + let mut bytes = synthetic_project(&[("Book_A", true)], &[b"Book"]); + insert_layer_with_two_nested_lists(&mut bytes); + let limits = OriginLimits { + // Window, layer, and the first nested item consume the full budget; + // the first item in the second independent list is record four. + max_metadata_records: 3, + ..OriginLimits::default() + }; + + assert!(matches!( + read_origin(&bytes, limits), + Err(OriginError::LimitExceeded { + resource: "metadata records", + limit: 3, + actual: 4, + }) + )); +} + +#[test] +fn skipped_parameters_still_consume_the_metadata_record_budget() { + let cases: &[(&[u8], f64)] = &[(&[0x80], 2.0), (b"NAN", f64::NAN)]; + for &(name, value) in cases { + let bytes = synthetic_project_with_parameters( + &[("Book_A", true, 1)], + &[b"Book"], + &[(b"ERR", 1.0), (name, value)], + ); + let limits = OriginLimits { + max_metadata_records: 2, + ..OriginLimits::default() + }; + assert!(matches!( + read_origin(&bytes, limits), + Err(OriginError::LimitExceeded { + resource: "metadata records", + limit: 2, + actual: 3, + }) + )); + } +} + +#[test] +fn metadata_record_count_equal_to_the_limit_succeeds() { + let bytes = synthetic_project(&[("Book_A", true)], &[b"Book"]); + let limits = OriginLimits { + max_metadata_records: 4, + ..OriginLimits::default() + }; + let project = read_origin(&bytes, limits).unwrap(); + + assert_eq!(project.resource_usage.metadata_records, 4); +} + +#[test] +fn enforces_metadata_nesting_depth_before_walking_nested_lists() { + let mut bytes = synthetic_project(&[("Book_A", true)], &[b"Book"]); + let marker = [27_u32.to_le_bytes().as_slice(), b"\n\0\0Book"].concat(); + let window = bytes + .windows(marker.len()) + .position(|candidate| candidate == marker) + .expect("synthetic window framing"); + let layer_list = window + 5 + 27 + 1; + let mut nested_layer = Vec::new(); + push_block(&mut nested_layer, Some(&[0_u8])); + for _ in 0..6 { + push_block(&mut nested_layer, None); + } + bytes.splice(layer_list..layer_list, nested_layer); + + let limits = OriginLimits { + max_metadata_depth: 2, + ..OriginLimits::default() + }; + assert!(matches!( + read_origin(&bytes, limits), + Err(OriginError::LimitExceeded { + resource: "metadata nesting depth", + limit: 2, + actual: 3, + }) + )); +} + +#[test] +fn accepts_framed_null_components_but_rejects_a_missing_component() { + let mut complete = synthetic_project(&[("Book_A", true)], &[b"Book"]); + let marker = [27_u32.to_le_bytes().as_slice(), b"\n\0\0Book"].concat(); + let window = complete + .windows(marker.len()) + .position(|candidate| candidate == marker) + .expect("synthetic window framing"); + let layer_list = window + 5 + 27 + 1; + let mut layer = Vec::new(); + push_block(&mut layer, Some(&[0_u8])); + push_block(&mut layer, Some(&[1_u8])); + for _ in 0..9 { + push_block(&mut layer, None); + } + let missing_component = layer_list + 14; + complete.splice(layer_list..layer_list, layer); + + let project = read_origin(&complete, OriginLimits::default()).unwrap(); + assert!( + project + .unsupported_objects + .iter() + .any(|summary| { summary.kind == "window presentation records" && summary.count == 2 }) + ); + + let mut truncated_item = complete; + truncated_item.drain(missing_component..missing_component + 5); + assert!(read_origin(&truncated_item, OriginLimits::default()).is_err()); +} + +#[test] +fn retains_validated_parameter_and_note_values() { + let bytes = synthetic_project(&[("Book_A", true)], &[b"Book"]); + let project = read_origin(&bytes, OriginLimits::default()).unwrap(); + + assert_eq!( + project.parameters, + [OriginMetadataEntry { + key: "ERR".to_owned(), + value: "1".to_owned(), + }] + ); + assert_eq!( + project.notes, + [OriginNote { + name: "Results".to_owned(), + content: "ok".to_owned(), + }] + ); +} + +#[test] +fn truncated_metadata_never_panics_or_returns_partial_output() { + let complete = synthetic_project(&[("Book_A", true)], &[b"Book"]); + let metadata_start = complete + .windows(27) + .position(|window| window.get(2..6) == Some(b"Book")) + .expect("synthetic window header"); + + for prefix in metadata_start..complete.len() { + let result = + std::panic::catch_unwind(|| read_origin(&complete[..prefix], OriginLimits::default())); + assert!(result.is_ok(), "metadata prefix {prefix} panicked"); + assert!( + result.unwrap().is_err(), + "metadata prefix {prefix} succeeded" + ); + } +} + +#[test] +fn rejects_corrupt_real_fixture_project_tree_and_attachment_bounds() { + for end in [ + 0x43602, + 0x43607, + 0x43700, + 0x4377f, + 0x43790, + OPENOPJ_FIXTURE.len() - 1, + ] { + let result = std::panic::catch_unwind(|| { + read_origin(&OPENOPJ_FIXTURE[..end], OriginLimits::default()) + }); + assert!(result.is_ok(), "real fixture prefix {end:#x} panicked"); + assert!( + result.unwrap().is_err(), + "real fixture prefix {end:#x} succeeded" + ); + } + + let tree_only = read_origin(&OPENOPJ_FIXTURE[..0x4377e], OriginLimits::default()) + .expect("an exact project-tree EOF is a valid no-attachment project"); + assert!( + tree_only + .unsupported_objects + .iter() + .any(|summary| { summary.kind == "project tree" && summary.count == 1 }) + ); + assert!( + !tree_only + .unsupported_objects + .iter() + .any(|summary| { summary.kind == "embedded attachments" }) + ); + + let mut tree = OPENOPJ_FIXTURE.to_vec(); + tree[0x43607..0x4360b].copy_from_slice(&381_u32.to_le_bytes()); + assert!(read_origin(&tree, OriginLimits::default()).is_err()); + + let mut attachment = OPENOPJ_FIXTURE.to_vec(); + attachment[0x43786..0x4378a].copy_from_slice(&5633_u32.to_le_bytes()); + assert!(read_origin(&attachment, OriginLimits::default()).is_err()); +} diff --git a/crates/io/src/origin/opj/records.rs b/crates/io/src/origin/opj/records.rs new file mode 100644 index 0000000..7fdb9ec --- /dev/null +++ b/crates/io/src/origin/opj/records.rs @@ -0,0 +1,548 @@ +use std::mem::size_of; + +use crate::origin::{ + OriginCell, OriginColumnType, OriginError, OriginLimits, OriginProfile, OriginResourceUsage, +}; + +use super::super::reader::{checked_add, checked_mul}; + +const HEADER_LEN: usize = 123; +const TYPE_OFFSET: usize = 0x16; +const SECONDARY_TYPE_OFFSET: usize = 0x18; +const TOTAL_ROWS_OFFSET: usize = 0x19; +const FIRST_ROW_OFFSET: usize = 0x1d; +const LAST_ROW_OFFSET: usize = 0x21; +const WIDTH_OFFSET: usize = 0x3d; +const UNSIGNED_FLAG_OFFSET: usize = 0x3f; +const NAME_OFFSET: usize = 0x58; +const NAME_WIDTH: usize = 25; +const TERTIARY_TYPE_OFFSET: usize = 0x71; + +const TYPE_F64: u16 = 0x6001; +const TYPE_F32: u16 = 0x6003; +const TYPE_I32: u16 = 0x6801; +const TYPE_I16: u16 = 0x6803; +const TYPE_TEXT: u16 = 0x6021; +const TYPE_MIXED: u16 = 0x6121; +const FIXED_TEXT_WIDTH: u8 = 25; +const SECONDARY: u8 = 0x01; +const TERTIARY_NUMERIC: u16 = 0x10ca; +const TERTIARY_FLOAT_OR_TEXT: u16 = 0x10e8; +const EMPTY_F64_BITS: u64 = 0x81aa_74fe_1c13_2c0e; + +#[derive(Debug, PartialEq)] +pub(super) struct DecodedColumnRecord { + pub(super) dataset_name: String, + pub(super) column_type: OriginColumnType, + pub(super) cells: Vec, + pub(super) first_row: usize, + pub(super) last_row_exclusive: usize, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ValueKind { + F64, + F32, + I32, + I16, + FixedText, + Mixed, + ModernF64, +} + +pub(super) fn decode_column_record( + profile: OriginProfile, + header: &[u8], + content: Option<&[u8]>, + limits: &OriginLimits, + usage: &mut OriginResourceUsage, +) -> Result { + require_header_length(profile, header)?; + + // This layout and the exact type combinations below are reimplemented + // from the pinned MIT-licensed OpenOPJ Origin 7.0552 description and are + // cross-checked against its redistributable test fixture: + // https://github.com/jgonera/openopj/blob/42ddcf1eb3a490744c54fca0a4ed6fe7a5e723ca/docs/opj_format.markdown + // https://github.com/jgonera/openopj/blob/42ddcf1eb3a490744c54fca0a4ed6fe7a5e723ca/lib/OpenOPJ/DataSection.php + let data_type = read_u16_at(header, TYPE_OFFSET, "dataset type")?; + let secondary = read_u8_at(header, SECONDARY_TYPE_OFFSET, "secondary dataset type")?; + let total_rows = read_u32_at(header, TOTAL_ROWS_OFFSET, "total rows")?; + let first_row = read_u32_at(header, FIRST_ROW_OFFSET, "first row")?; + let last_row = read_u32_at(header, LAST_ROW_OFFSET, "last row")?; + let width = read_u8_at(header, WIDTH_OFFSET, "value width")?; + let unsigned_flag = read_u8_at(header, UNSIGNED_FLAG_OFFSET, "unsigned flag")?; + let tertiary = read_u16_at(header, TERTIARY_TYPE_OFFSET, "tertiary dataset type")?; + + let (total_rows, first_row, last_row_exclusive) = + validate_geometry(total_rows, first_row, last_row, limits)?; + let kind = classify_type( + profile, + data_type, + secondary, + width, + unsigned_flag, + tertiary, + last_row_exclusive == 0, + )?; + let width = usize::from(width); + let expected_bytes = checked_mul(total_rows, width, "OPJ column content bytes")?; + let content = require_content(content, expected_bytes)?; + let dataset_name = decode_ascii( + field(header, NAME_OFFSET, NAME_WIDTH, "dataset name")?, + NAME_OFFSET, + limits, + usage, + )?; + + charge_column_and_cells(last_row_exclusive, limits, usage)?; + let cell_bytes = checked_mul( + last_row_exclusive, + size_of::(), + "decoded OPJ cells", + )?; + charge_parser(cell_bytes, limits, usage)?; + let mut cells = Vec::new(); + cells + .try_reserve_exact(last_row_exclusive) + .map_err(|_| OriginError::AllocationFailed { + resource: "decoded OPJ cells", + requested: cell_bytes, + })?; + + // The public fixture proves that lastRow is exclusive: TestW_Float stores + // lastRow=2 for two values, while TestW_firstRow stores firstRow=1 and + // lastRow=3 for [missing, 5.23, -7]. Decode existing payload slots rather + // than prepending firstRow synthetic nulls, which would shift the data. + for row in 0..last_row_exclusive { + let start = checked_mul(row, width, "OPJ cell offset")?; + let end = checked_add(start, width, "OPJ cell range")?; + let slot = content + .get(start..end) + .ok_or_else(|| truncated_range(start, width, content.len().saturating_sub(start)))?; + let cell = decode_cell(kind, slot, start, limits, usage)?; + if row < first_row && cell != OriginCell::Null { + return Err(OriginError::CorruptStructure { + offset: start, + detail: "a payload slot before firstRow is not the verified missing-value sentinel" + .to_owned(), + }); + } + cells.push(cell); + } + + Ok(DecodedColumnRecord { + dataset_name, + column_type: column_type(kind), + cells, + first_row, + last_row_exclusive, + }) +} + +fn require_header_length(profile: OriginProfile, header: &[u8]) -> Result<(), OriginError> { + let expected = match profile { + OriginProfile::Origin7V552 => HEADER_LEN, + OriginProfile::Origin9V951 => 147, + }; + if header.len() < expected { + return Err(OriginError::Truncated { + offset: header.len(), + needed: expected - header.len(), + have: 0, + }); + } + if header.len() > expected { + return Err(OriginError::CorruptStructure { + offset: expected, + detail: format!( + "{} data header must be exactly {expected} bytes, not {}", + profile_name(profile), + header.len() + ), + }); + } + Ok(()) +} + +fn classify_type( + profile: OriginProfile, + data_type: u16, + secondary: u8, + width: u8, + unsigned_flag: u8, + tertiary: u16, + is_empty: bool, +) -> Result { + if profile == OriginProfile::Origin9V951 { + return classify_modern_type( + data_type, + secondary, + width, + unsigned_flag, + tertiary, + is_empty, + ); + } + if unsigned_flag != 0 { + return unsupported("unsigned Origin integers are not verified for Origin7V552"); + } + if secondary != SECONDARY { + return unsupported("the secondary Origin dataset type is not verified for Origin7V552"); + } + + match (data_type, width, tertiary) { + (TYPE_F64, 8, TERTIARY_NUMERIC) => Ok(ValueKind::F64), + (TYPE_F32, 4, TERTIARY_FLOAT_OR_TEXT) => Ok(ValueKind::F32), + (TYPE_I32, 4, TERTIARY_NUMERIC) => Ok(ValueKind::I32), + (TYPE_I16, 2, TERTIARY_NUMERIC) => Ok(ValueKind::I16), + (TYPE_TEXT, FIXED_TEXT_WIDTH, TERTIARY_FLOAT_OR_TEXT) => Ok(ValueKind::FixedText), + (TYPE_MIXED, 10, TERTIARY_NUMERIC) => Ok(ValueKind::Mixed), + _ => unsupported("the Origin dataset type and value width combination is not verified"), + } +} + +fn classify_modern_type( + data_type: u16, + secondary: u8, + width: u8, + storage_flag: u8, + tertiary: u16, + is_empty: bool, +) -> Result { + // Match complete tuples from two Origin 9.51 projects. Checking fields + // independently would silently accept an unobserved Cartesian product. + let verified = matches!( + ( + data_type, + secondary, + width, + storage_flag, + tertiary, + is_empty + ), + (0x5121, 0x03, 10, 0x20, 0x10c8, false) + | (0x5121, 0x03, 10, 0x30, 0x10c8, false) + | (0x5121, 0x03, 10, 0x30, 0x10ca, false) + | (0x6121, 0x03, 10, 0x00, 0x11ca, false) + | (0x6121, 0x03, 10, 0x10, 0x10c8, false) + | (0x6121, 0x03, 10, 0x10, 0x10c9, false) + | (0x6121, 0x03, 10, 0x10, 0x10ca, false) + | (0x6121, 0x01, 10, 0x00, 0x10ca, true) + | (0x6121, 0x03, 10, 0x00, 0x10ca, true) + ); + if !verified { + return unsupported("the complete Origin9V951 dataset type tuple is not verified"); + } + Ok(ValueKind::ModernF64) +} + +fn profile_name(profile: OriginProfile) -> &'static str { + match profile { + OriginProfile::Origin7V552 => "Origin7V552", + OriginProfile::Origin9V951 => "Origin9V951", + } +} + +fn validate_geometry( + total_rows: u32, + first_row: u32, + last_row: u32, + limits: &OriginLimits, +) -> Result<(usize, usize, usize), OriginError> { + if [total_rows, first_row, last_row] + .into_iter() + .any(|value| value > i32::MAX as u32) + { + return Err(OriginError::CorruptStructure { + offset: TOTAL_ROWS_OFFSET, + detail: "Origin7V552 row geometry contains a negative or unverified high-bit value" + .to_owned(), + }); + } + if first_row > last_row || last_row > total_rows { + return Err(OriginError::CorruptStructure { + offset: FIRST_ROW_OFFSET, + detail: "Origin7V552 rows must satisfy firstRow <= lastRow <= totalRows".to_owned(), + }); + } + + let total_rows = usize::try_from(total_rows).map_err(|_| OriginError::ArithmeticOverflow { + resource: "OPJ total rows", + })?; + let first_row = usize::try_from(first_row).map_err(|_| OriginError::ArithmeticOverflow { + resource: "OPJ first row", + })?; + let last_row = usize::try_from(last_row).map_err(|_| OriginError::ArithmeticOverflow { + resource: "OPJ last row", + })?; + enforce_limit("rows per column", total_rows, limits.max_rows_per_column)?; + Ok((total_rows, first_row, last_row)) +} + +fn require_content(content: Option<&[u8]>, expected: usize) -> Result<&[u8], OriginError> { + let content = content.unwrap_or_default(); + if content.len() < expected { + return Err(OriginError::Truncated { + offset: 0, + needed: expected, + have: content.len(), + }); + } + if content.len() > expected { + return Err(OriginError::CorruptStructure { + offset: expected, + detail: format!( + "Origin column content has {} bytes but its geometry requires {expected}", + content.len() + ), + }); + } + Ok(content) +} + +fn decode_cell( + kind: ValueKind, + slot: &[u8], + offset: usize, + limits: &OriginLimits, + usage: &mut OriginResourceUsage, +) -> Result { + match kind { + ValueKind::F64 => decode_f64(slot, offset), + ValueKind::F32 => Ok(OriginCell::Float(f64::from(f32::from_le_bytes( + read_array(slot, offset)?, + )))), + ValueKind::I32 => Ok(OriginCell::Integer(i64::from(i32::from_le_bytes( + read_array(slot, offset)?, + )))), + ValueKind::I16 => Ok(OriginCell::Integer(i64::from(i16::from_le_bytes( + read_array(slot, offset)?, + )))), + ValueKind::FixedText => Ok(OriginCell::Text(decode_ascii(slot, offset, limits, usage)?)), + ValueKind::Mixed => decode_mixed(slot, offset, limits, usage), + ValueKind::ModernF64 => decode_modern_f64(slot, offset), + } +} + +fn decode_modern_f64(slot: &[u8], offset: usize) -> Result { + let discriminator = *slot.first().ok_or_else(|| truncated_range(offset, 1, 0))?; + let reserved = *slot.get(1).ok_or_else(|| truncated_range(offset, 2, 1))?; + if reserved != 0 { + return Err(OriginError::CorruptStructure { + offset: checked_add(offset, 1, "Origin9V951 reserved prefix offset")?, + detail: "the reserved Origin9V951 numeric prefix byte must be zero".to_owned(), + }); + } + if discriminator != 0 { + return unsupported("Origin9V951 text and unknown cell discriminators are not supported"); + } + let payload = slot + .get(2..) + .ok_or_else(|| truncated_range(offset, 2, slot.len()))?; + decode_f64( + payload, + checked_add(offset, 2, "Origin9V951 numeric value offset")?, + ) +} + +fn decode_f64(slot: &[u8], offset: usize) -> Result { + let value = f64::from_le_bytes(read_array(slot, offset)?); + if value.to_bits() == EMPTY_F64_BITS { + Ok(OriginCell::Null) + } else { + Ok(OriginCell::Float(value)) + } +} + +fn decode_mixed( + slot: &[u8], + offset: usize, + limits: &OriginLimits, + usage: &mut OriginResourceUsage, +) -> Result { + let prefix = *slot.first().ok_or_else(|| truncated_range(offset, 1, 0))?; + let reserved = *slot.get(1).ok_or_else(|| truncated_range(offset, 2, 1))?; + if reserved != 0 { + return Err(OriginError::CorruptStructure { + offset: checked_add(offset, 1, "mixed prefix offset")?, + detail: "the reserved mixed-cell prefix byte must be zero".to_owned(), + }); + } + let payload = slot + .get(2..) + .ok_or_else(|| truncated_range(offset, 2, slot.len()))?; + match prefix { + 0 => decode_f64(payload, checked_add(offset, 2, "mixed value offset")?), + 1 => Ok(OriginCell::Text(decode_ascii( + payload, + checked_add(offset, 2, "mixed text offset")?, + limits, + usage, + )?)), + _ => Err(OriginError::CorruptStructure { + offset, + detail: "mixed Origin cells require a numeric prefix 0 or text prefix 1".to_owned(), + }), + } +} + +fn decode_ascii( + field: &[u8], + offset: usize, + limits: &OriginLimits, + usage: &mut OriginResourceUsage, +) -> Result { + let length = field + .iter() + .position(|byte| *byte == 0) + .unwrap_or(field.len()); + let text = field.get(..length).ok_or(OriginError::ArithmeticOverflow { + resource: "bounded ASCII field", + })?; + if let Some(relative) = text.iter().position(|byte| !byte.is_ascii()) { + return Err(OriginError::UnsupportedEncoding { + offset: checked_add(offset, relative, "ASCII byte offset")?, + encoding: "non-ASCII byte in Origin7V552 text".to_owned(), + }); + } + enforce_limit("string bytes", length, limits.max_string_bytes)?; + charge_text(length, limits, usage)?; + + let mut decoded = String::new(); + decoded + .try_reserve_exact(length) + .map_err(|_| OriginError::AllocationFailed { + resource: "decoded Origin text", + requested: length, + })?; + let text = std::str::from_utf8(text).map_err(|_| OriginError::UnsupportedEncoding { + offset, + encoding: "non-ASCII byte in Origin7V552 text".to_owned(), + })?; + decoded.push_str(text); + Ok(decoded) +} + +fn column_type(kind: ValueKind) -> OriginColumnType { + match kind { + ValueKind::F64 | ValueKind::F32 => OriginColumnType::Float, + ValueKind::I32 | ValueKind::I16 => OriginColumnType::Integer, + ValueKind::FixedText => OriginColumnType::Text, + ValueKind::Mixed => OriginColumnType::Mixed, + ValueKind::ModernF64 => OriginColumnType::Float, + } +} + +fn field<'a>( + bytes: &'a [u8], + offset: usize, + length: usize, + resource: &'static str, +) -> Result<&'a [u8], OriginError> { + let end = checked_add(offset, length, resource)?; + bytes + .get(offset..end) + .ok_or_else(|| truncated_range(offset, length, bytes.len().saturating_sub(offset))) +} + +fn read_u8_at(bytes: &[u8], offset: usize, resource: &'static str) -> Result { + field(bytes, offset, 1, resource)? + .first() + .copied() + .ok_or_else(|| truncated_range(offset, 1, 0)) +} + +fn read_u16_at(bytes: &[u8], offset: usize, resource: &'static str) -> Result { + Ok(u16::from_le_bytes(read_array( + field(bytes, offset, size_of::(), resource)?, + offset, + )?)) +} + +fn read_u32_at(bytes: &[u8], offset: usize, resource: &'static str) -> Result { + Ok(u32::from_le_bytes(read_array( + field(bytes, offset, size_of::(), resource)?, + offset, + )?)) +} + +fn read_array(bytes: &[u8], offset: usize) -> Result<[u8; N], OriginError> { + bytes.try_into().map_err(|_| OriginError::Truncated { + offset, + needed: N, + have: bytes.len(), + }) +} + +fn charge_column_and_cells( + cells: usize, + limits: &OriginLimits, + usage: &mut OriginResourceUsage, +) -> Result<(), OriginError> { + let columns = checked_add(usage.columns, 1, "decoded OPJ columns")?; + enforce_limit("columns", columns, limits.max_columns)?; + let total_cells = checked_add(usage.cells, cells, "decoded OPJ cells")?; + enforce_limit("cells", total_cells, limits.max_cells)?; + usage.columns = columns; + usage.cells = total_cells; + Ok(()) +} + +fn charge_text( + bytes: usize, + limits: &OriginLimits, + usage: &mut OriginResourceUsage, +) -> Result<(), OriginError> { + let decoded = checked_add(usage.decoded_text_bytes, bytes, "decoded text bytes")?; + enforce_limit("decoded text bytes", decoded, limits.max_decoded_text_bytes)?; + charge_parser(bytes, limits, usage)?; + usage.decoded_text_bytes = decoded; + Ok(()) +} + +fn charge_parser( + bytes: usize, + limits: &OriginLimits, + usage: &mut OriginResourceUsage, +) -> Result<(), OriginError> { + let parser = checked_add(usage.parser_bytes, bytes, "parser bytes")?; + enforce_limit("parser bytes", parser, limits.max_parser_bytes)?; + let total = checked_add(usage.total_owned_bytes, bytes, "total owned bytes")?; + enforce_limit("total owned bytes", total, limits.max_total_owned_bytes)?; + usage.parser_bytes = parser; + usage.total_owned_bytes = total; + Ok(()) +} + +fn enforce_limit(resource: &'static str, actual: usize, limit: usize) -> Result<(), OriginError> { + if actual > limit { + return Err(OriginError::LimitExceeded { + resource, + limit, + actual, + }); + } + Ok(()) +} + +fn unsupported(feature: &'static str) -> Result { + Err(OriginError::UnsupportedFeature { + feature: feature.to_owned(), + }) +} + +fn truncated_range(offset: usize, needed: usize, have: usize) -> OriginError { + OriginError::Truncated { + offset, + needed, + have, + } +} + +#[cfg(test)] +#[path = "records_tests.rs"] +mod records_tests; + +#[cfg(test)] +#[path = "records_modern_tests.rs"] +mod records_modern_tests; diff --git a/crates/io/src/origin/opj/records_modern_tests.rs b/crates/io/src/origin/opj/records_modern_tests.rs new file mode 100644 index 0000000..950c77d --- /dev/null +++ b/crates/io/src/origin/opj/records_modern_tests.rs @@ -0,0 +1,288 @@ +use std::panic::catch_unwind; + +use super::{DecodedColumnRecord, decode_column_record}; +use crate::origin::{ + OriginCell, OriginColumnType, OriginError, OriginLimits, OriginProfile, OriginResourceUsage, +}; + +const HEADER_LEN: usize = 147; +const TYPE_OFFSET: usize = 0x16; +const SECONDARY_OFFSET: usize = 0x18; +const TOTAL_ROWS_OFFSET: usize = 0x19; +const FIRST_ROW_OFFSET: usize = 0x1d; +const LAST_ROW_OFFSET: usize = 0x21; +const WIDTH_OFFSET: usize = 0x3d; +const STORAGE_FLAG_OFFSET: usize = 0x3f; +const NAME_OFFSET: usize = 0x58; +const TERTIARY_OFFSET: usize = 0x71; +const EMPTY_F64: f64 = -1.23456789E-300; + +#[derive(Clone, Copy)] +struct ModernHeader { + data_type: u16, + secondary: u8, + total_rows: u32, + first_row: u32, + last_row: u32, + width: u8, + storage_flag: u8, + tertiary: u16, +} + +impl Default for ModernHeader { + fn default() -> Self { + Self { + data_type: 0x6121, + secondary: 0x03, + total_rows: 2, + first_row: 0, + last_row: 2, + width: 10, + storage_flag: 0x10, + tertiary: 0x10ca, + } + } +} + +fn header(spec: ModernHeader) -> Vec { + let mut bytes = vec![0_u8; HEADER_LEN]; + bytes[TYPE_OFFSET..TYPE_OFFSET + 2].copy_from_slice(&spec.data_type.to_le_bytes()); + bytes[SECONDARY_OFFSET] = spec.secondary; + bytes[TOTAL_ROWS_OFFSET..TOTAL_ROWS_OFFSET + 4].copy_from_slice(&spec.total_rows.to_le_bytes()); + bytes[FIRST_ROW_OFFSET..FIRST_ROW_OFFSET + 4].copy_from_slice(&spec.first_row.to_le_bytes()); + bytes[LAST_ROW_OFFSET..LAST_ROW_OFFSET + 4].copy_from_slice(&spec.last_row.to_le_bytes()); + bytes[WIDTH_OFFSET] = spec.width; + bytes[STORAGE_FLAG_OFFSET] = spec.storage_flag; + bytes[NAME_OFFSET..NAME_OFFSET + 7].copy_from_slice(b"Book1_A"); + bytes[TERTIARY_OFFSET..TERTIARY_OFFSET + 2].copy_from_slice(&spec.tertiary.to_le_bytes()); + bytes +} + +fn slots(values: &[f64]) -> Vec { + values + .iter() + .flat_map(|value| [0, 0].into_iter().chain(value.to_le_bytes())) + .collect() +} + +fn decode(header: &[u8], content: &[u8]) -> Result { + let mut usage = OriginResourceUsage::default(); + decode_column_record( + OriginProfile::Origin9V951, + header, + Some(content), + &OriginLimits::default(), + &mut usage, + ) +} + +#[test] +fn modern_decodes_only_the_observed_numeric_field_combinations() { + let observed = [ + (0x5121, 0x20, 0x10c8), + (0x5121, 0x30, 0x10c8), + (0x5121, 0x30, 0x10ca), + (0x6121, 0x00, 0x11ca), + (0x6121, 0x10, 0x10c8), + (0x6121, 0x10, 0x10c9), + (0x6121, 0x10, 0x10ca), + ]; + for (data_type, storage_flag, tertiary) in observed { + let decoded = decode( + &header(ModernHeader { + data_type, + storage_flag, + tertiary, + ..ModernHeader::default() + }), + &slots(&[0.05, 20.0]), + ) + .unwrap(); + assert_eq!(decoded.dataset_name, "Book1_A"); + assert_eq!(decoded.column_type, OriginColumnType::Float); + assert_eq!( + decoded.cells, + vec![OriginCell::Float(0.05), OriginCell::Float(20.0)] + ); + } +} + +#[test] +fn modern_rejects_unobserved_cross_product_of_verified_fields() { + let result = decode( + &header(ModernHeader { + data_type: 0x5121, + storage_flag: 0x00, + tertiary: 0x11ca, + ..ModernHeader::default() + }), + &slots(&[0.05, 20.0]), + ); + + assert!(matches!( + result, + Err(OriginError::UnsupportedFeature { .. }) + )); +} + +#[test] +fn modern_maps_the_verified_missing_sentinel_to_null() { + let decoded = decode(&header(ModernHeader::default()), &slots(&[EMPTY_F64, 1.0])).unwrap(); + assert_eq!( + decoded.cells, + vec![OriginCell::Null, OriginCell::Float(1.0)] + ); +} + +#[test] +fn modern_retains_an_empty_column_with_validated_storage() { + for secondary in [0x01, 0x03] { + let spec = ModernHeader { + secondary, + total_rows: 3, + last_row: 0, + storage_flag: 0x00, + tertiary: 0x10ca, + ..ModernHeader::default() + }; + let decoded = decode(&header(spec), &slots(&[EMPTY_F64; 3])).unwrap(); + assert!(decoded.cells.is_empty()); + assert_eq!(decoded.first_row, 0); + assert_eq!(decoded.last_row_exclusive, 0); + } +} + +#[test] +fn modern_rejects_text_discriminator_without_guessing() { + let mut content = slots(&[1.0, 2.0]); + content[0] = 1; + assert!(matches!( + decode(&header(ModernHeader::default()), &content), + Err(OriginError::UnsupportedFeature { .. }) + )); +} + +#[test] +fn modern_rejects_nonzero_reserved_prefix_as_corrupt() { + let mut content = slots(&[1.0, 2.0]); + content[1] = 1; + assert!(matches!( + decode(&header(ModernHeader::default()), &content), + Err(OriginError::CorruptStructure { .. }) + )); +} + +#[test] +fn modern_rejects_unobserved_header_fields() { + let cases = [ + ModernHeader { + data_type: 0x6001, + ..ModernHeader::default() + }, + ModernHeader { + secondary: 0x01, + ..ModernHeader::default() + }, + ModernHeader { + width: 8, + ..ModernHeader::default() + }, + ModernHeader { + storage_flag: 0x40, + ..ModernHeader::default() + }, + ModernHeader { + tertiary: 0x10e8, + ..ModernHeader::default() + }, + ]; + for spec in cases { + assert!(matches!( + decode(&header(spec), &slots(&[1.0, 2.0])), + Err(OriginError::UnsupportedFeature { .. }) + )); + } +} + +#[test] +fn modern_rejects_wrong_header_and_content_lengths() { + let complete_header = header(ModernHeader::default()); + let complete_content = slots(&[1.0, 2.0]); + for length in [146, 148] { + let mut wrong = complete_header.clone(); + wrong.resize(length, 0); + assert!(decode(&wrong, &complete_content).is_err()); + } + assert!(decode(&complete_header, &complete_content[..19]).is_err()); + let mut extra = complete_content.clone(); + extra.push(0); + assert!(decode(&complete_header, &extra).is_err()); +} + +#[test] +fn modern_rejects_invalid_geometry_and_configured_limits() { + let invalid = header(ModernHeader { + first_row: 2, + last_row: 1, + ..ModernHeader::default() + }); + assert!(matches!( + decode(&invalid, &slots(&[1.0, 2.0])), + Err(OriginError::CorruptStructure { .. }) + )); + + let mut usage = OriginResourceUsage::default(); + let limits = OriginLimits { + max_rows_per_column: 1, + ..OriginLimits::default() + }; + let content = slots(&[1.0, 2.0]); + assert!(matches!( + decode_column_record( + OriginProfile::Origin9V951, + &header(ModernHeader::default()), + Some(&content), + &limits, + &mut usage, + ), + Err(OriginError::LimitExceeded { + resource: "rows per column", + .. + }) + )); + + let mut usage = OriginResourceUsage::default(); + let limits = OriginLimits { + max_cells: 1, + ..OriginLimits::default() + }; + assert!(matches!( + decode_column_record( + OriginProfile::Origin9V951, + &header(ModernHeader::default()), + Some(&content), + &limits, + &mut usage, + ), + Err(OriginError::LimitExceeded { + resource: "cells", + .. + }) + )); +} + +#[test] +fn modern_every_truncated_prefix_returns_an_error_without_panicking() { + let complete_header = header(ModernHeader::default()); + let complete_content = slots(&[1.0, 2.0]); + for end in 0..complete_header.len() { + let outcome = catch_unwind(|| decode(&complete_header[..end], &complete_content)); + assert!(outcome.is_ok(), "header prefix {end} panicked"); + assert!(outcome.unwrap().is_err(), "header prefix {end} succeeded"); + } + for end in 0..complete_content.len() { + let outcome = catch_unwind(|| decode(&complete_header, &complete_content[..end])); + assert!(outcome.is_ok(), "content prefix {end} panicked"); + assert!(outcome.unwrap().is_err(), "content prefix {end} succeeded"); + } +} diff --git a/crates/io/src/origin/opj/records_tests.rs b/crates/io/src/origin/opj/records_tests.rs new file mode 100644 index 0000000..b0cac62 --- /dev/null +++ b/crates/io/src/origin/opj/records_tests.rs @@ -0,0 +1,668 @@ +use std::mem::size_of; +use std::panic::catch_unwind; + +use super::{DecodedColumnRecord, decode_column_record}; +use crate::origin::{ + OriginCell, OriginColumnType, OriginError, OriginLimits, OriginProfile, OriginResourceUsage, +}; + +const HEADER_LEN: usize = 123; +const TYPE_OFFSET: usize = 0x16; +const SECONDARY_TYPE_OFFSET: usize = 0x18; +const TOTAL_ROWS_OFFSET: usize = 0x19; +const FIRST_ROW_OFFSET: usize = 0x1d; +const LAST_ROW_OFFSET: usize = 0x21; +const WIDTH_OFFSET: usize = 0x3d; +const UNSIGNED_FLAG_OFFSET: usize = 0x3f; +const NAME_OFFSET: usize = 0x58; +const NAME_WIDTH: usize = 25; +const TERTIARY_TYPE_OFFSET: usize = 0x71; + +const TYPE_F64: u16 = 0x6001; +const TYPE_F32: u16 = 0x6003; +const TYPE_I32: u16 = 0x6801; +const TYPE_I16: u16 = 0x6803; +const TYPE_TEXT: u16 = 0x6021; +const TYPE_MIXED: u16 = 0x6121; + +const SECONDARY: u8 = 0x01; +const TERTIARY_NUMERIC: u16 = 0x10ca; +const TERTIARY_FLOAT_OR_TEXT: u16 = 0x10e8; +const EMPTY_F64: f64 = -1.23456789E-300; +const FIXTURE_F32: f32 = f32::from_bits(0x43ac_cccd); +const FIXTURE_MIXED_F64: f64 = f64::from_bits(0x4009_1eb8_51eb_851f); + +#[derive(Clone)] +struct RecordBytes { + header: Vec, + content: Vec, +} + +#[derive(Clone, Copy)] +struct HeaderSpec { + data_type: u16, + secondary: u8, + total_rows: u32, + first_row: u32, + last_row: u32, + width: u8, + unsigned_flag: u8, + tertiary: u16, +} + +impl HeaderSpec { + fn fixture_type( + data_type: u16, + total_rows: u32, + first_row: u32, + last_row: u32, + width: u8, + tertiary: u16, + ) -> Self { + Self { + data_type, + secondary: SECONDARY, + total_rows, + first_row, + last_row, + width, + unsigned_flag: 0, + tertiary, + } + } +} + +fn header(spec: HeaderSpec, name: &str) -> Vec { + let mut bytes = vec![0_u8; HEADER_LEN]; + bytes[TYPE_OFFSET..TYPE_OFFSET + 2].copy_from_slice(&spec.data_type.to_le_bytes()); + bytes[SECONDARY_TYPE_OFFSET] = spec.secondary; + bytes[TOTAL_ROWS_OFFSET..TOTAL_ROWS_OFFSET + 4].copy_from_slice(&spec.total_rows.to_le_bytes()); + bytes[FIRST_ROW_OFFSET..FIRST_ROW_OFFSET + 4].copy_from_slice(&spec.first_row.to_le_bytes()); + bytes[LAST_ROW_OFFSET..LAST_ROW_OFFSET + 4].copy_from_slice(&spec.last_row.to_le_bytes()); + bytes[WIDTH_OFFSET] = spec.width; + bytes[UNSIGNED_FLAG_OFFSET] = spec.unsigned_flag; + bytes[TERTIARY_TYPE_OFFSET..TERTIARY_TYPE_OFFSET + 2] + .copy_from_slice(&spec.tertiary.to_le_bytes()); + let name = name.as_bytes(); + assert!(name.len() < NAME_WIDTH); + bytes[NAME_OFFSET..NAME_OFFSET + name.len()].copy_from_slice(name); + bytes +} + +fn f64_record(values: &[f64]) -> RecordBytes { + let row_count = u32::try_from(values.len()).unwrap(); + RecordBytes { + header: header( + HeaderSpec::fixture_type(TYPE_F64, row_count, 0, row_count, 8, TERTIARY_NUMERIC), + "Data1_INJV", + ), + content: values + .iter() + .flat_map(|value| value.to_le_bytes()) + .collect(), + } +} + +fn f32_record(values: &[f32]) -> RecordBytes { + let row_count = u32::try_from(values.len()).unwrap(); + RecordBytes { + header: header( + HeaderSpec::fixture_type(TYPE_F32, row_count, 0, row_count, 4, TERTIARY_FLOAT_OR_TEXT), + "TestW_Float", + ), + content: values + .iter() + .flat_map(|value| value.to_le_bytes()) + .collect(), + } +} + +fn i32_record(values: &[i32]) -> RecordBytes { + let row_count = u32::try_from(values.len()).unwrap(); + RecordBytes { + header: header( + HeaderSpec::fixture_type(TYPE_I32, row_count, 0, row_count, 4, TERTIARY_NUMERIC), + "TestW_Long", + ), + content: values + .iter() + .flat_map(|value| value.to_le_bytes()) + .collect(), + } +} + +fn i16_record(values: &[i16]) -> RecordBytes { + let row_count = u32::try_from(values.len()).unwrap(); + RecordBytes { + header: header( + HeaderSpec::fixture_type(TYPE_I16, row_count, 0, row_count, 2, TERTIARY_NUMERIC), + "TestW_Integer", + ), + content: values + .iter() + .flat_map(|value| value.to_le_bytes()) + .collect(), + } +} + +fn text_record(value: &str) -> RecordBytes { + const WIDTH: usize = 25; + let mut content = vec![0_u8; WIDTH]; + content[..value.len()].copy_from_slice(value.as_bytes()); + content[value.len() + 1..].fill(0xff); + RecordBytes { + header: header( + HeaderSpec::fixture_type(TYPE_TEXT, 1, 0, 1, WIDTH as u8, TERTIARY_FLOAT_OR_TEXT), + "TestW_Text", + ), + content, + } +} + +fn mixed_record() -> RecordBytes { + let mut content = Vec::new(); + content.extend_from_slice(&[1, 0]); + content.extend_from_slice(b"text\0\xff\xfe\xfd"); + content.extend_from_slice(&[0, 0]); + content.extend_from_slice(&FIXTURE_MIXED_F64.to_le_bytes()); + RecordBytes { + header: header( + HeaderSpec::fixture_type(TYPE_MIXED, 2, 0, 2, 10, TERTIARY_NUMERIC), + "TestW_TextNumeric", + ), + content, + } +} + +fn decode(record: &RecordBytes) -> Result { + let mut usage = OriginResourceUsage::default(); + decode_with(record, OriginLimits::default(), &mut usage) +} + +fn decode_with( + record: &RecordBytes, + limits: OriginLimits, + usage: &mut OriginResourceUsage, +) -> Result { + decode_column_record( + OriginProfile::Origin7V552, + &record.header, + Some(&record.content), + &limits, + usage, + ) +} + +fn clear_dataset_name(record: &mut RecordBytes) { + record.header[NAME_OFFSET..NAME_OFFSET + NAME_WIDTH].fill(0); +} + +fn assert_limit(error: OriginError, resource: &'static str, limit: usize, actual: usize) { + assert_eq!( + error, + OriginError::LimitExceeded { + resource, + limit, + actual, + } + ); +} + +#[test] +fn decodes_fixture_backed_f64_and_missing_sentinel() { + let decoded = decode(&f64_record(&[0.4, EMPTY_F64])).unwrap(); + assert_eq!(decoded.dataset_name, "Data1_INJV"); + assert_eq!(decoded.column_type, OriginColumnType::Float); + assert_eq!( + decoded.cells, + vec![OriginCell::Float(0.4), OriginCell::Null] + ); +} + +#[test] +fn decodes_fixture_backed_f32_losslessly_into_f64() { + let decoded = decode(&f32_record(&[FIXTURE_F32])).unwrap(); + assert_eq!( + decoded.cells, + vec![OriginCell::Float(f64::from(FIXTURE_F32))] + ); +} + +#[test] +fn decodes_fixture_backed_signed_i32() { + let decoded = decode(&i32_record(&[345, -100_000])).unwrap(); + assert_eq!(decoded.column_type, OriginColumnType::Integer); + assert_eq!( + decoded.cells, + vec![OriginCell::Integer(345), OriginCell::Integer(-100_000)] + ); +} + +#[test] +fn decodes_fixture_backed_signed_i16() { + let decoded = decode(&i16_record(&[34, -1000])).unwrap(); + assert_eq!( + decoded.cells, + vec![OriginCell::Integer(34), OriginCell::Integer(-1000)] + ); +} + +#[test] +fn decodes_fixture_backed_fixed_ascii_text() { + let decoded = decode(&text_record("test string 123")).unwrap(); + assert_eq!(decoded.column_type, OriginColumnType::Text); + assert_eq!( + decoded.cells, + vec![OriginCell::Text("test string 123".to_owned())] + ); +} + +#[test] +fn decodes_fixture_backed_mixed_text_and_number() { + let decoded = decode(&mixed_record()).unwrap(); + assert_eq!(decoded.column_type, OriginColumnType::Mixed); + assert_eq!( + decoded.cells, + vec![ + OriginCell::Text("text".to_owned()), + OriginCell::Float(FIXTURE_MIXED_F64) + ] + ); +} + +#[test] +fn fixture_exclusive_last_row_preserves_verified_leading_null_slot() { + let mut content = Vec::new(); + content.extend_from_slice(&EMPTY_F64.to_le_bytes()); + content.extend_from_slice(&5.23_f64.to_le_bytes()); + content.extend_from_slice(&(-7.0_f64).to_le_bytes()); + let record = RecordBytes { + header: header( + HeaderSpec::fixture_type(TYPE_F64, 3, 1, 3, 8, TERTIARY_NUMERIC), + "TestW_firstRow", + ), + content, + }; + + let decoded = decode(&record).unwrap(); + assert_eq!(decoded.first_row, 1); + assert_eq!(decoded.last_row_exclusive, 3); + assert_eq!( + decoded.cells, + vec![ + OriginCell::Null, + OriginCell::Float(5.23), + OriginCell::Float(-7.0) + ] + ); +} + +#[test] +fn rejects_nonnull_payload_slots_before_first_row() { + let mut content = Vec::new(); + content.extend_from_slice(&999.0_f64.to_le_bytes()); + content.extend_from_slice(&5.23_f64.to_le_bytes()); + let record = RecordBytes { + header: header( + HeaderSpec::fixture_type(TYPE_F64, 2, 1, 2, 8, TERTIARY_NUMERIC), + "InvalidFirstRow", + ), + content, + }; + + assert!(matches!( + decode(&record), + Err(OriginError::CorruptStructure { .. }) + )); +} + +#[test] +fn rejects_width_and_fixture_type_mismatches() { + let cases = [ + (TYPE_F64, 4, TERTIARY_NUMERIC), + (TYPE_F32, 8, TERTIARY_FLOAT_OR_TEXT), + (TYPE_I32, 2, TERTIARY_NUMERIC), + (TYPE_I16, 4, TERTIARY_NUMERIC), + (TYPE_TEXT, 8, TERTIARY_FLOAT_OR_TEXT), + (TYPE_TEXT, 24, TERTIARY_FLOAT_OR_TEXT), + (TYPE_TEXT, 26, TERTIARY_FLOAT_OR_TEXT), + (TYPE_MIXED, 9, TERTIARY_NUMERIC), + ]; + for (data_type, width, tertiary) in cases { + let record = RecordBytes { + header: header( + HeaderSpec::fixture_type(data_type, 1, 0, 1, width, tertiary), + "Mismatch", + ), + content: vec![0; usize::from(width)], + }; + assert!(matches!( + decode(&record), + Err(OriginError::UnsupportedFeature { .. }) + )); + } +} + +#[test] +fn rejects_geometry_outside_total_rows_or_signed_range() { + let beyond_total = RecordBytes { + header: header( + HeaderSpec::fixture_type(TYPE_F64, 1, 0, 2, 8, TERTIARY_NUMERIC), + "BeyondTotal", + ), + content: vec![0; 8], + }; + assert!(matches!( + decode(&beyond_total), + Err(OriginError::CorruptStructure { .. }) + )); + + let reversed = RecordBytes { + header: header( + HeaderSpec::fixture_type(TYPE_F64, 3, 2, 1, 8, TERTIARY_NUMERIC), + "Reversed", + ), + content: vec![0; 24], + }; + assert!(matches!( + decode(&reversed), + Err(OriginError::CorruptStructure { .. }) + )); + + let signed_negative = RecordBytes { + header: header( + HeaderSpec::fixture_type(TYPE_F64, u32::MAX, 0, 1, 8, TERTIARY_NUMERIC), + "NegativeGeometry", + ), + content: Vec::new(), + }; + assert!(matches!( + decode(&signed_negative), + Err(OriginError::CorruptStructure { .. }) + )); +} + +#[test] +fn rejects_incomplete_fixed_text_and_mixed_numeric_payloads() { + let mut fixed = text_record("test string 123"); + fixed.content.pop(); + assert!(matches!(decode(&fixed), Err(OriginError::Truncated { .. }))); + + let mut mixed = mixed_record(); + mixed.header = header( + HeaderSpec::fixture_type(TYPE_MIXED, 1, 0, 1, 10, TERTIARY_NUMERIC), + "MixedNumeric", + ); + mixed.content = vec![0, 0, 1, 2, 3, 4, 5, 6, 7]; + assert!(matches!(decode(&mixed), Err(OriginError::Truncated { .. }))); +} + +#[test] +fn rejects_content_one_byte_larger_than_declared_geometry() { + let mut oversized = f64_record(&[0.4]); + oversized.content.push(0); + + assert!(matches!( + decode(&oversized), + Err(OriginError::CorruptStructure { .. }) + )); +} + +#[test] +fn enforces_row_column_and_cell_limits_with_exact_counts() { + let record = f64_record(&[0.4, EMPTY_F64]); + let mut usage = OriginResourceUsage::default(); + let row_limits = OriginLimits { + max_rows_per_column: 1, + ..OriginLimits::default() + }; + assert_limit( + decode_with(&record, row_limits, &mut usage).unwrap_err(), + "rows per column", + 1, + 2, + ); + + let mut usage = OriginResourceUsage { + columns: 1, + ..OriginResourceUsage::default() + }; + let column_limits = OriginLimits { + max_columns: 1, + ..OriginLimits::default() + }; + assert_limit( + decode_with(&record, column_limits, &mut usage).unwrap_err(), + "columns", + 1, + 2, + ); + + let mut usage = OriginResourceUsage { + cells: 1, + ..OriginResourceUsage::default() + }; + let cell_limits = OriginLimits { + max_cells: 2, + ..OriginLimits::default() + }; + assert_limit( + decode_with(&record, cell_limits, &mut usage).unwrap_err(), + "cells", + 2, + 3, + ); +} + +#[test] +fn enforces_string_and_cumulative_decoded_text_limits() { + let named_record = f64_record(&[0.4]); + let mut usage = OriginResourceUsage::default(); + let dataset_name_limits = OriginLimits { + max_string_bytes: 9, + ..OriginLimits::default() + }; + assert_limit( + decode_with(&named_record, dataset_name_limits, &mut usage).unwrap_err(), + "string bytes", + 9, + 10, + ); + + let mut record = text_record("test string 123"); + clear_dataset_name(&mut record); + + let mut usage = OriginResourceUsage::default(); + let string_limits = OriginLimits { + max_string_bytes: 14, + ..OriginLimits::default() + }; + assert_limit( + decode_with(&record, string_limits, &mut usage).unwrap_err(), + "string bytes", + 14, + 15, + ); + + let mut usage = OriginResourceUsage { + decoded_text_bytes: 1, + ..OriginResourceUsage::default() + }; + let text_limits = OriginLimits { + max_decoded_text_bytes: 15, + ..OriginLimits::default() + }; + assert_limit( + decode_with(&record, text_limits, &mut usage).unwrap_err(), + "decoded text bytes", + 15, + 16, + ); +} + +#[test] +fn enforces_parser_and_total_owned_limits_before_cell_vector_allocation() { + let mut record = f64_record(&[0.4, EMPTY_F64]); + clear_dataset_name(&mut record); + let cell_bytes = 2 * size_of::(); + + let mut usage = OriginResourceUsage { + parser_bytes: 1, + ..OriginResourceUsage::default() + }; + let parser_limits = OriginLimits { + max_parser_bytes: cell_bytes, + ..OriginLimits::default() + }; + assert_limit( + decode_with(&record, parser_limits, &mut usage).unwrap_err(), + "parser bytes", + cell_bytes, + cell_bytes + 1, + ); + + let mut usage = OriginResourceUsage { + total_owned_bytes: 1, + ..OriginResourceUsage::default() + }; + let total_limits = OriginLimits { + max_total_owned_bytes: cell_bytes, + ..OriginLimits::default() + }; + assert_limit( + decode_with(&record, total_limits, &mut usage).unwrap_err(), + "total owned bytes", + cell_bytes, + cell_bytes + 1, + ); +} + +#[test] +fn rejects_invalid_mixed_prefix_and_nonzero_reserved_prefix() { + let mut invalid = mixed_record(); + invalid.content[0] = 2; + assert!(matches!( + decode(&invalid), + Err(OriginError::CorruptStructure { .. }) + )); + + let mut reserved = mixed_record(); + reserved.content[1] = 1; + assert!(matches!( + decode(&reserved), + Err(OriginError::CorruptStructure { .. }) + )); +} + +#[test] +fn rejects_non_ascii_fixed_and_mixed_text() { + let mut fixed = text_record("ascii"); + fixed.content[0] = 0xff; + assert!(matches!( + decode(&fixed), + Err(OriginError::UnsupportedEncoding { .. }) + )); + + let mut mixed = mixed_record(); + mixed.content[2] = 0xff; + assert!(matches!( + decode(&mixed), + Err(OriginError::UnsupportedEncoding { .. }) + )); +} + +#[test] +fn rejects_unsupported_eight_bit_integer_and_unsigned_flag() { + let eight_bit = RecordBytes { + header: header( + HeaderSpec::fixture_type(TYPE_I32, 1, 0, 1, 1, TERTIARY_NUMERIC), + "EightBit", + ), + content: vec![1], + }; + assert!(matches!( + decode(&eight_bit), + Err(OriginError::UnsupportedFeature { .. }) + )); + + let mut unsigned_spec = HeaderSpec::fixture_type(TYPE_I32, 1, 0, 1, 4, TERTIARY_NUMERIC); + unsigned_spec.unsigned_flag = 8; + let unsigned = RecordBytes { + header: header(unsigned_spec, "Unsigned"), + content: 345_i32.to_le_bytes().to_vec(), + }; + assert!(matches!( + decode(&unsigned), + Err(OriginError::UnsupportedFeature { .. }) + )); +} + +#[test] +fn rejects_unverified_secondary_and_tertiary_type_fields() { + let mut secondary = HeaderSpec::fixture_type(TYPE_F64, 1, 0, 1, 8, TERTIARY_NUMERIC); + secondary.secondary = 0x91; + let secondary = RecordBytes { + header: header(secondary, "Secondary"), + content: 0.4_f64.to_le_bytes().to_vec(), + }; + assert!(matches!( + decode(&secondary), + Err(OriginError::UnsupportedFeature { .. }) + )); + + let tertiary = RecordBytes { + header: header( + HeaderSpec::fixture_type(TYPE_F64, 1, 0, 1, 8, 0x50ca), + "Tertiary", + ), + content: 0.4_f64.to_le_bytes().to_vec(), + }; + assert!(matches!( + decode(&tertiary), + Err(OriginError::UnsupportedFeature { .. }) + )); +} + +#[test] +fn every_truncated_minimal_record_returns_an_error_without_panicking() { + let records = [ + f64_record(&[0.4, EMPTY_F64]), + f32_record(&[FIXTURE_F32]), + i32_record(&[345, -100_000]), + i16_record(&[34, -1000]), + text_record("test string 123"), + mixed_record(), + ]; + + for record in records { + for end in 0..record.header.len() { + let outcome = catch_unwind(|| { + let mut usage = OriginResourceUsage::default(); + decode_column_record( + OriginProfile::Origin7V552, + &record.header[..end], + Some(&record.content), + &OriginLimits::default(), + &mut usage, + ) + }); + assert!(outcome.is_ok(), "header prefix {end} panicked"); + assert!(outcome.unwrap().is_err(), "header prefix {end} succeeded"); + } + + for end in 0..record.content.len() { + let outcome = catch_unwind(|| { + let mut usage = OriginResourceUsage::default(); + decode_column_record( + OriginProfile::Origin7V552, + &record.header, + Some(&record.content[..end]), + &OriginLimits::default(), + &mut usage, + ) + }); + assert!(outcome.is_ok(), "content prefix {end} panicked"); + assert!(outcome.unwrap().is_err(), "content prefix {end} succeeded"); + } + } +} diff --git a/crates/io/src/origin/opju.rs b/crates/io/src/origin/opju.rs new file mode 100644 index 0000000..2814ed6 --- /dev/null +++ b/crates/io/src/origin/opju.rs @@ -0,0 +1,11 @@ +use super::{OriginError, OriginProbe, OriginProject}; + +/// OPJU is detection-only until a complete, bounded container profile has +/// public evidence. In particular, this path must not scan markers or attempt +/// to decode records after the validated first line. +pub(super) fn read(_probe: OriginProbe) -> Result { + Err(OriginError::UnsupportedOpjuVariant { + message: "This OPJU file uses a record layout that PlotX does not support yet. No data was imported." + .to_owned(), + }) +} diff --git a/crates/io/src/origin/reader.rs b/crates/io/src/origin/reader.rs new file mode 100644 index 0000000..562bbc9 --- /dev/null +++ b/crates/io/src/origin/reader.rs @@ -0,0 +1,358 @@ +use std::mem::size_of; + +use super::{OriginError, OriginLimits, OriginResourceUsage}; + +const LF: u8 = b'\n'; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum FramedBlock<'a> { + Null { offset: usize }, + Data { offset: usize, payload: &'a [u8] }, +} + +impl FramedBlock<'_> { + pub(super) fn offset(&self) -> usize { + match self { + Self::Null { offset } | Self::Data { offset, .. } => *offset, + } + } +} + +pub(super) struct Reader<'bytes, 'limits> { + bytes: &'bytes [u8], + offset: usize, + limits: &'limits OriginLimits, + usage: OriginResourceUsage, +} + +impl<'bytes, 'limits> Reader<'bytes, 'limits> { + #[cfg_attr(not(test), allow(dead_code))] + pub(super) fn new( + bytes: &'bytes [u8], + limits: &'limits OriginLimits, + ) -> Result { + Self::new_with_parser_bytes(bytes, limits, 0) + } + + pub(super) fn new_with_parser_bytes( + bytes: &'bytes [u8], + limits: &'limits OriginLimits, + initial_parser_bytes: usize, + ) -> Result { + limits.validate()?; + enforce_limit("input bytes", bytes.len(), limits.max_input_bytes)?; + enforce_limit( + "parser bytes", + initial_parser_bytes, + limits.max_parser_bytes, + )?; + let total_owned_bytes = + checked_add(bytes.len(), initial_parser_bytes, "total owned bytes")?; + enforce_limit( + "total owned bytes", + total_owned_bytes, + limits.max_total_owned_bytes, + )?; + + Ok(Self { + bytes, + offset: 0, + limits, + usage: OriginResourceUsage { + input_bytes: bytes.len(), + parser_bytes: initial_parser_bytes, + total_owned_bytes, + ..OriginResourceUsage::default() + }, + }) + } + + pub(super) fn offset(&self) -> usize { + self.offset + } + + pub(super) fn into_usage(self) -> OriginResourceUsage { + self.usage + } + + pub(super) fn read_slice(&mut self, length: usize) -> Result<&'bytes [u8], OriginError> { + let start = self.offset; + let end = checked_add(start, length, "reader offset")?; + let available = + self.bytes + .len() + .checked_sub(start) + .ok_or(OriginError::ArithmeticOverflow { + resource: "remaining input bytes", + })?; + let slice = self.bytes.get(start..end).ok_or(OriginError::Truncated { + offset: start, + needed: length, + have: available, + })?; + self.offset = end; + Ok(slice) + } + + pub(super) fn read_u8(&mut self) -> Result { + let start = self.offset; + let byte = self + .bytes + .get(start) + .copied() + .ok_or(OriginError::Truncated { + offset: start, + needed: 1, + have: 0, + })?; + self.offset = checked_add(start, 1, "reader offset")?; + Ok(byte) + } + + // Task 3 establishes these checked primitives before record decoding uses + // every width in production; their focused tests keep the staged API live. + #[cfg_attr(not(test), allow(dead_code))] + pub(super) fn read_u16_le(&mut self) -> Result { + Ok(u16::from_le_bytes(self.read_array()?)) + } + + pub(super) fn read_u32_le(&mut self) -> Result { + Ok(u32::from_le_bytes(self.read_array()?)) + } + + #[cfg_attr(not(test), allow(dead_code))] + pub(super) fn read_i16_le(&mut self) -> Result { + Ok(i16::from_le_bytes(self.read_array()?)) + } + + #[cfg_attr(not(test), allow(dead_code))] + pub(super) fn read_i32_le(&mut self) -> Result { + Ok(i32::from_le_bytes(self.read_array()?)) + } + + #[cfg_attr(not(test), allow(dead_code))] + pub(super) fn read_f32_le(&mut self) -> Result { + Ok(f32::from_le_bytes(self.read_array()?)) + } + + #[cfg_attr(not(test), allow(dead_code))] + pub(super) fn read_f64_le(&mut self) -> Result { + Ok(f64::from_le_bytes(self.read_array()?)) + } + + pub(super) fn read_block(&mut self) -> Result, OriginError> { + // OpenOPJ's MIT-licensed reader defines a block as little-endian u32 + // size plus LF, followed (only for nonzero size) by payload plus LF. + // https://github.com/jgonera/openopj/blob/42ddcf1eb3a490744c54fca0a4ed6fe7a5e723ca/lib/OpenOPJ/common.php + let block_offset = self.offset; + let payload_len = self.read_u32_le()?; + self.expect_lf("block-size delimiter")?; + if payload_len == 0 { + return Ok(FramedBlock::Null { + offset: block_offset, + }); + } + + let payload_len = + usize::try_from(payload_len).map_err(|_| OriginError::ArithmeticOverflow { + resource: "block bytes", + })?; + enforce_limit("block bytes", payload_len, self.limits.max_block_bytes)?; + let payload = self.read_slice(payload_len)?; + self.expect_lf("block-payload delimiter")?; + Ok(FramedBlock::Data { + offset: block_offset, + payload, + }) + } + + #[cfg_attr(not(test), allow(dead_code))] + pub(super) fn read_fixed_ascii(&mut self, width: usize) -> Result { + let field_offset = self.offset; + let field = self.read_slice(width)?; + let text_len = field + .iter() + .position(|byte| *byte == 0) + .unwrap_or(field.len()); + let text = field + .get(..text_len) + .ok_or(OriginError::ArithmeticOverflow { + resource: "fixed ASCII field", + })?; + if let Some(relative) = text.iter().position(|byte| !byte.is_ascii()) { + return Err(OriginError::UnsupportedEncoding { + offset: checked_add(field_offset, relative, "text byte offset")?, + encoding: "non-ASCII byte in fixed-width text".to_owned(), + }); + } + enforce_limit("string bytes", text_len, self.limits.max_string_bytes)?; + self.charge_text(text_len)?; + + let mut decoded = String::new(); + decoded + .try_reserve_exact(text_len) + .map_err(|_| OriginError::AllocationFailed { + resource: "decoded text", + requested: text_len, + })?; + let text = std::str::from_utf8(text).map_err(|_| OriginError::UnsupportedEncoding { + offset: field_offset, + encoding: "non-ASCII byte in fixed-width text".to_owned(), + })?; + decoded.push_str(text); + Ok(decoded) + } + + pub(super) fn try_reserve( + &mut self, + values: &mut Vec, + additional: usize, + resource: &'static str, + ) -> Result<(), OriginError> { + let requested_len = checked_add(values.len(), additional, resource)?; + let old_capacity = values.capacity(); + let element_size = size_of::(); + if requested_len <= old_capacity || element_size == 0 { + return Ok(()); + } + + let minimum_delta = requested_len + .checked_sub(old_capacity) + .ok_or(OriginError::ArithmeticOverflow { resource })?; + let minimum_bytes = checked_mul(minimum_delta, element_size, resource)?; + checked_parser_usage(&self.usage, self.limits, minimum_bytes)?; + + // Bound geometric growth by both remaining budgets. The measured + // capacity delta is charged after reserve so allocator rounding cannot + // bypass accounting; an over-budget rounded allocation fails closed. + let available_bytes = self + .limits + .max_parser_bytes + .saturating_sub(self.usage.parser_bytes) + .min( + self.limits + .max_total_owned_bytes + .saturating_sub(self.usage.total_owned_bytes), + ); + let geometric_capacity = if old_capacity == 0 { + requested_len + } else { + old_capacity.checked_mul(2).unwrap_or(requested_len) + }; + let desired_capacity = requested_len.max(geometric_capacity); + let desired_delta = desired_capacity + .checked_sub(old_capacity) + .ok_or(OriginError::ArithmeticOverflow { resource })?; + let affordable_delta = (available_bytes / element_size).min(desired_delta); + let target_capacity = checked_add(old_capacity, affordable_delta, resource)?; + let reserve_additional = target_capacity + .checked_sub(values.len()) + .ok_or(OriginError::ArithmeticOverflow { resource })?; + let planned_delta = target_capacity + .checked_sub(old_capacity) + .ok_or(OriginError::ArithmeticOverflow { resource })?; + let planned_bytes = checked_mul(planned_delta, element_size, resource)?; + values.try_reserve_exact(reserve_additional).map_err(|_| { + OriginError::AllocationFailed { + resource, + requested: planned_bytes, + } + })?; + + let actual_delta = values + .capacity() + .checked_sub(old_capacity) + .ok_or(OriginError::ArithmeticOverflow { resource })?; + let actual_bytes = checked_mul(actual_delta, element_size, resource)?; + self.charge_parser(actual_bytes) + } + + fn read_array(&mut self) -> Result<[u8; N], OriginError> { + let start = self.offset; + let bytes = self.read_slice(N)?; + bytes.try_into().map_err(|_| OriginError::Truncated { + offset: start, + needed: N, + have: bytes.len(), + }) + } + + fn expect_lf(&mut self, field: &'static str) -> Result<(), OriginError> { + let offset = self.offset; + let delimiter = self.read_u8()?; + if delimiter != LF { + return Err(OriginError::CorruptStructure { + offset, + detail: format!("{field} must be LF"), + }); + } + Ok(()) + } + + #[cfg_attr(not(test), allow(dead_code))] + fn charge_text(&mut self, bytes: usize) -> Result<(), OriginError> { + let decoded_text_bytes = + checked_add(self.usage.decoded_text_bytes, bytes, "decoded text bytes")?; + enforce_limit( + "decoded text bytes", + decoded_text_bytes, + self.limits.max_decoded_text_bytes, + )?; + self.charge_parser(bytes)?; + self.usage.decoded_text_bytes = decoded_text_bytes; + Ok(()) + } + + fn charge_parser(&mut self, bytes: usize) -> Result<(), OriginError> { + let (parser_bytes, total_owned_bytes) = + checked_parser_usage(&self.usage, self.limits, bytes)?; + self.usage.parser_bytes = parser_bytes; + self.usage.total_owned_bytes = total_owned_bytes; + Ok(()) + } +} + +fn checked_parser_usage( + usage: &OriginResourceUsage, + limits: &OriginLimits, + bytes: usize, +) -> Result<(usize, usize), OriginError> { + let parser_bytes = checked_add(usage.parser_bytes, bytes, "parser bytes")?; + enforce_limit("parser bytes", parser_bytes, limits.max_parser_bytes)?; + let total_owned_bytes = checked_add(usage.total_owned_bytes, bytes, "total owned bytes")?; + enforce_limit( + "total owned bytes", + total_owned_bytes, + limits.max_total_owned_bytes, + )?; + Ok((parser_bytes, total_owned_bytes)) +} + +pub(super) fn checked_add( + left: usize, + right: usize, + resource: &'static str, +) -> Result { + left.checked_add(right) + .ok_or(OriginError::ArithmeticOverflow { resource }) +} + +pub(super) fn checked_mul( + left: usize, + right: usize, + resource: &'static str, +) -> Result { + left.checked_mul(right) + .ok_or(OriginError::ArithmeticOverflow { resource }) +} + +fn enforce_limit(resource: &'static str, actual: usize, limit: usize) -> Result<(), OriginError> { + if actual > limit { + return Err(OriginError::LimitExceeded { + resource, + limit, + actual, + }); + } + Ok(()) +} diff --git a/crates/io/src/origin/reader_tests.rs b/crates/io/src/origin/reader_tests.rs new file mode 100644 index 0000000..181ee5f --- /dev/null +++ b/crates/io/src/origin/reader_tests.rs @@ -0,0 +1,303 @@ +use super::reader::{FramedBlock, Reader, checked_add, checked_mul}; +use super::{OriginError, OriginLimits}; +use std::mem::size_of; + +#[test] +fn reads_checked_little_endian_primitives() { + let mut bytes = Vec::new(); + bytes.push(0xa5); + bytes.extend_from_slice(&0x1234_u16.to_le_bytes()); + bytes.extend_from_slice(&0x89ab_cdef_u32.to_le_bytes()); + bytes.extend_from_slice(&(-12_345_i16).to_le_bytes()); + bytes.extend_from_slice(&(-123_456_789_i32).to_le_bytes()); + bytes.extend_from_slice(&12.5_f32.to_le_bytes()); + bytes.extend_from_slice(&(-98.25_f64).to_le_bytes()); + let limits = OriginLimits::default(); + let mut reader = Reader::new(&bytes, &limits).unwrap(); + + assert_eq!(reader.read_u8().unwrap(), 0xa5); + assert_eq!(reader.read_u16_le().unwrap(), 0x1234); + assert_eq!(reader.read_u32_le().unwrap(), 0x89ab_cdef); + assert_eq!(reader.read_i16_le().unwrap(), -12_345); + assert_eq!(reader.read_i32_le().unwrap(), -123_456_789); + assert_eq!(reader.read_f32_le().unwrap(), 12.5); + assert_eq!(reader.read_f64_le().unwrap(), -98.25); + assert_eq!(reader.offset(), bytes.len()); +} + +#[test] +fn checked_slice_accepts_exact_end_and_rejects_one_byte_past_end() { + let limits = OriginLimits::default(); + let mut reader = Reader::new(b"abc", &limits).unwrap(); + assert_eq!(reader.read_slice(3).unwrap(), b"abc"); + + assert!(matches!( + reader.read_slice(1), + Err(OriginError::Truncated { + offset: 3, + needed: 1, + have: 0, + }) + )); +} + +#[test] +fn checked_arithmetic_reports_overflow() { + assert!(matches!( + checked_add(usize::MAX, 1, "test offset"), + Err(OriginError::ArithmeticOverflow { + resource: "test offset" + }) + )); + assert!(matches!( + checked_mul(usize::MAX, 2, "test capacity"), + Err(OriginError::ArithmeticOverflow { + resource: "test capacity" + }) + )); +} + +#[test] +fn reads_data_and_null_block_framing() { + let bytes = [ + 3, 0, 0, 0, b'\n', b'a', b'b', b'c', b'\n', 0, 0, 0, 0, b'\n', + ]; + let limits = OriginLimits::default(); + let mut reader = Reader::new(&bytes, &limits).unwrap(); + + assert!(matches!( + reader.read_block().unwrap(), + FramedBlock::Data { offset: 0, payload } if payload == b"abc" + )); + assert!(matches!( + reader.read_block().unwrap(), + FramedBlock::Null { offset: 9 } + )); + assert_eq!(reader.offset(), bytes.len()); +} + +#[test] +fn rejects_bad_block_delimiters_at_their_offsets() { + let limits = OriginLimits::default(); + let mut bad_size = Reader::new(&[1, 0, 0, 0, b'!', b'a', b'\n'], &limits).unwrap(); + assert!(matches!( + bad_size.read_block(), + Err(OriginError::CorruptStructure { offset: 4, .. }) + )); + + let mut bad_payload = Reader::new(&[1, 0, 0, 0, b'\n', b'a', b'!'], &limits).unwrap(); + assert!(matches!( + bad_payload.read_block(), + Err(OriginError::CorruptStructure { offset: 6, .. }) + )); +} + +#[test] +fn rejects_oversized_declared_block_before_slicing() { + let limits = OriginLimits { + max_block_bytes: 2, + ..OriginLimits::default() + }; + let mut reader = Reader::new(&[3, 0, 0, 0, b'\n'], &limits).unwrap(); + + assert!(matches!( + reader.read_block(), + Err(OriginError::LimitExceeded { + resource: "block bytes", + limit: 2, + actual: 3, + }) + )); +} + +#[test] +fn parser_budget_is_checked_before_vec_reserve() { + let limits = OriginLimits { + max_parser_bytes: 3, + ..OriginLimits::default() + }; + let mut reader = Reader::new(&[], &limits).unwrap(); + let mut values = Vec::::new(); + + assert!(matches!( + reader.try_reserve(&mut values, 2, "test values"), + Err(OriginError::LimitExceeded { + resource: "parser bytes", + limit: 3, + actual: 4, + }) + )); + assert_eq!(values.capacity(), 0); +} + +#[test] +fn repeated_single_item_reservations_grow_logarithmically_and_charge_capacity() { + let limits = OriginLimits::default(); + let mut reader = Reader::new(&[], &limits).unwrap(); + let mut values = Vec::::new(); + let mut capacity_changes = 0_usize; + + for value in 0..4096_u64 { + let previous_capacity = values.capacity(); + reader + .try_reserve(&mut values, 1, "test reader values") + .unwrap(); + if values.capacity() != previous_capacity { + capacity_changes += 1; + } + values.push(value); + } + + let usage = reader.into_usage(); + assert!( + capacity_changes <= 16, + "single-item appends reallocated {capacity_changes} times" + ); + assert_eq!(usage.parser_bytes, values.capacity() * size_of::()); + assert_eq!(usage.total_owned_bytes, usage.parser_bytes); +} + +#[test] +fn spare_vector_capacity_is_not_charged_as_a_new_reader_allocation() { + let limits = OriginLimits::default(); + let mut reader = Reader::new(&[], &limits).unwrap(); + let mut values = Vec::::with_capacity(8); + let original_capacity = values.capacity(); + + reader + .try_reserve(&mut values, 1, "test reader values") + .unwrap(); + + let usage = reader.into_usage(); + assert_eq!(values.capacity(), original_capacity); + assert_eq!(usage.parser_bytes, 0); + assert_eq!(usage.total_owned_bytes, 0); +} + +#[test] +fn reader_charges_the_actual_capacity_delta_with_an_unbounded_budget() { + let limits = OriginLimits { + max_parser_bytes: usize::MAX, + max_total_owned_bytes: usize::MAX, + ..OriginLimits::default() + }; + let mut reader = Reader::new(&[], &limits).unwrap(); + let mut values = vec![0_u8; 8]; + let original_capacity = values.capacity(); + + reader + .try_reserve(&mut values, 1, "test reader values") + .unwrap(); + + let usage = reader.into_usage(); + assert!(values.capacity() > original_capacity); + assert_eq!(usage.parser_bytes, values.capacity() - original_capacity); + assert_eq!(usage.total_owned_bytes, usage.parser_bytes); +} + +#[test] +fn decoded_text_budget_is_checked_before_string_reserve() { + let limits = OriginLimits { + max_decoded_text_bytes: 3, + ..OriginLimits::default() + }; + let mut reader = Reader::new(b"text", &limits).unwrap(); + + assert!(matches!( + reader.read_fixed_ascii(4), + Err(OriginError::LimitExceeded { + resource: "decoded text bytes", + limit: 3, + actual: 4, + }) + )); +} + +#[test] +fn impossible_capacity_requests_return_errors_without_panicking() { + let limits = OriginLimits { + max_parser_bytes: usize::MAX, + max_total_owned_bytes: usize::MAX, + ..OriginLimits::default() + }; + let result = std::panic::catch_unwind(|| { + let mut reader = Reader::new(&[], &limits)?; + let mut wide = Vec::::new(); + reader.try_reserve(&mut wide, usize::MAX, "wide values")?; + Ok::<(), OriginError>(()) + }); + assert!(result.is_ok()); + assert!(matches!( + result.unwrap(), + Err(OriginError::ArithmeticOverflow { .. }) + | Err(OriginError::LimitExceeded { .. }) + | Err(OriginError::AllocationFailed { .. }) + )); + + let allocation = std::panic::catch_unwind(|| { + let mut reader = Reader::new(&[], &limits)?; + let mut bytes = Vec::::new(); + reader.try_reserve(&mut bytes, isize::MAX as usize, "huge byte buffer")?; + Ok::<(), OriginError>(()) + }); + assert!(allocation.is_ok()); + assert!(matches!( + allocation.unwrap(), + Err(OriginError::AllocationFailed { .. }) | Err(OriginError::ArithmeticOverflow { .. }) + )); +} + +#[test] +fn every_truncated_block_prefix_returns_a_structured_error() { + let complete = [1, 0, 0, 0, b'\n', b'x', b'\n']; + let limits = OriginLimits::default(); + + for prefix_len in 0..complete.len() { + let result = std::panic::catch_unwind(|| { + let mut reader = Reader::new(&complete[..prefix_len], &limits)?; + reader.read_block() + }); + assert!(result.is_ok(), "prefix {prefix_len} panicked"); + assert!(matches!( + result.unwrap(), + Err(OriginError::Truncated { .. }) + )); + } +} + +#[test] +fn fixed_ascii_trims_only_in_field_nul_padding() { + let limits = OriginLimits::default(); + let mut reader = Reader::new(b"abc\0\0next", &limits).unwrap(); + + assert_eq!(reader.read_fixed_ascii(5).unwrap(), "abc"); + assert_eq!(reader.offset(), 5); +} + +#[test] +fn fixed_ascii_rejects_non_ascii_before_nul_but_ignores_bytes_after_nul() { + let limits = OriginLimits::default(); + let mut invalid = Reader::new(&[b'a', 0xff, 0, b'x'], &limits).unwrap(); + assert!(matches!( + invalid.read_fixed_ascii(4), + Err(OriginError::UnsupportedEncoding { offset: 1, .. }) + )); + + let mut padded = Reader::new(&[b'o', b'k', 0, 0xff], &limits).unwrap(); + assert_eq!(padded.read_fixed_ascii(4).unwrap(), "ok"); +} + +#[test] +fn reader_rejects_zero_custom_limits() { + let limits = OriginLimits { + max_block_bytes: 0, + ..OriginLimits::default() + }; + assert!(matches!( + Reader::new(&[], &limits), + Err(OriginError::InvalidLimit { + name: "max_block_bytes", + .. + }) + )); +} diff --git a/crates/io/src/origin/tests.rs b/crates/io/src/origin/tests.rs new file mode 100644 index 0000000..af68b45 --- /dev/null +++ b/crates/io/src/origin/tests.rs @@ -0,0 +1,716 @@ +use super::*; + +const OPENOPJ_FIXTURE: &[u8] = include_bytes!("../../tests/fixtures/origin/test-origin-7.0552.opj"); + +fn long_leading_zero_opju_header() -> Vec { + let mut bytes = b"CPYUA ".to_vec(); + bytes.extend(std::iter::repeat_n(b'0', 160)); + bytes.extend_from_slice(b"4."); + bytes.extend(std::iter::repeat_n(b'0', 160)); + bytes.extend_from_slice(b"3668 "); + bytes.extend(std::iter::repeat_n(b'0', 160)); + bytes.extend_from_slice(b"178\n"); + bytes +} + +fn supported_opj_probe_bytes() -> Vec { + const ORIGIN_HEADER_LEN: usize = 39; + let mut bytes = b"CPYA 4.2673 552#\n".to_vec(); + bytes.extend_from_slice(&(ORIGIN_HEADER_LEN as u32).to_le_bytes()); + bytes.push(b'\n'); + let mut header = [0_u8; ORIGIN_HEADER_LEN]; + header[0x1b..0x23].copy_from_slice(&7.0552_f64.to_le_bytes()); + bytes.extend_from_slice(&header); + bytes.push(b'\n'); + bytes.extend_from_slice(&[0, 0, 0, 0, b'\n']); + bytes +} + +#[test] +fn probes_opj_and_opju_by_content() { + let opj_bytes = supported_opj_probe_bytes(); + let opj = probe_origin(&opj_bytes).unwrap(); + assert_eq!(opj.format, OriginFormat::Opj); + assert_eq!(opj.profile, Some(OriginProfile::Origin7V552)); + assert_eq!(opj.support, OriginSupport::Supported); + + let opju = probe_origin(b"CPYUA 4.3668 178\n").unwrap(); + assert_eq!(opju.format, OriginFormat::Opju); + assert_eq!(opju.profile, None); + assert_eq!(opju.support, OriginSupport::RecognizedUnsupported); +} + +#[test] +fn parses_header_components_without_floating_point() { + let opj_bytes = supported_opj_probe_bytes(); + let opj = probe_origin(&opj_bytes).unwrap(); + assert_eq!(opj.raw_version, "4.2673 552"); + assert_eq!(opj.version.major, 4); + assert_eq!(opj.version.minor, 2673); + assert_eq!(opj.version.build, 552); + assert_eq!(opj.byte_order, OriginByteOrder::LittleEndian); + + let opju = probe_origin(b"CPYUA 4.3668 178\n").unwrap(); + assert_eq!(opju.raw_version, "4.3668 178"); + assert_eq!(opju.version.major, 4); + assert_eq!(opju.version.minor, 3668); + assert_eq!(opju.version.build, 178); + assert_eq!(opju.byte_order, OriginByteOrder::LittleEndian); +} + +#[test] +fn classic_version_line_without_initial_framing_is_truncated() { + assert!(matches!( + probe_origin(b"CPYA 4.2673 552#\n"), + Err(OriginError::Truncated { .. }) + )); +} + +#[test] +fn opju_is_recognized_but_not_partially_imported() { + let error = read_origin(b"CPYUA 4.3668 178\nrest", OriginLimits::default()).unwrap_err(); + assert!(matches!(error, OriginError::UnsupportedOpjuVariant { .. })); +} + +#[test] +fn rejects_unknown_or_truncated_headers() { + assert!(matches!( + probe_origin(b"CP"), + Err(OriginError::Truncated { .. }) + )); + assert!(matches!( + probe_origin(b"not an origin file"), + Err(OriginError::UnrecognizedFormat) + )); +} + +#[test] +fn rejects_malformed_classic_version_lines() { + for bytes in [ + b"CPYA 4.2673#\n".as_slice(), + b"CPYA 4.x 552#\n".as_slice(), + b"CPYA 4.2673 build#\n".as_slice(), + b"CPYA 4.2673 552\n".as_slice(), + b"CPYA 4.2673 552##\n".as_slice(), + b"CPYA 4.2673 552#\r\n".as_slice(), + ] { + assert!(matches!( + probe_origin(bytes), + Err(OriginError::MalformedHeader { .. }) + )); + } +} + +#[test] +fn rejects_headers_over_the_default_limit() { + let mut bytes = b"CPYA ".to_vec(); + bytes.resize(129, b'1'); + bytes.push(b'\n'); + + assert!(matches!( + probe_origin(&bytes), + Err(OriginError::HeaderTooLong { limit: 128 }) + )); +} + +#[test] +fn rejects_input_one_byte_over_a_custom_limit() { + let bytes = b"CPYUA 4.3668 178\n"; + let limit = bytes.len() - 1; + let limits = OriginLimits { + max_input_bytes: limit, + ..OriginLimits::default() + }; + + assert!(matches!( + read_origin(bytes, limits), + Err(OriginError::LimitExceeded { + resource: "input bytes", + limit: found_limit, + actual, + }) if found_limit == limit && actual == bytes.len() + )); +} + +#[test] +fn long_opju_version_is_bounded_by_the_string_limit_before_detection_returns() { + let bytes = long_leading_zero_opju_header(); + let limits = OriginLimits { + max_header_bytes: bytes.len(), + max_string_bytes: 64, + ..OriginLimits::default() + }; + + assert!(matches!( + read_origin(&bytes, limits), + Err(OriginError::LimitExceeded { + resource: "string bytes", + limit: 64, + actual, + }) if actual > 64 + )); +} + +#[test] +fn long_opju_version_is_bounded_by_the_parser_limit_before_detection_returns() { + let bytes = long_leading_zero_opju_header(); + let raw_version_len = bytes.len() - b"CPYUA \n".len(); + let limits = OriginLimits { + max_header_bytes: bytes.len(), + max_string_bytes: raw_version_len, + max_parser_bytes: raw_version_len - 1, + ..OriginLimits::default() + }; + + assert!(matches!( + read_origin(&bytes, limits), + Err(OriginError::LimitExceeded { + resource: "parser bytes", + limit, + actual, + }) if limit == raw_version_len - 1 && actual >= raw_version_len + )); +} + +#[test] +fn long_opju_version_is_bounded_by_source_plus_probe_total_bytes() { + let bytes = long_leading_zero_opju_header(); + let raw_version_len = bytes.len() - b"CPYUA \n".len(); + let total_limit = bytes.len() + raw_version_len - 1; + let limits = OriginLimits { + max_header_bytes: bytes.len(), + max_string_bytes: raw_version_len, + max_parser_bytes: raw_version_len, + max_total_owned_bytes: total_limit, + ..OriginLimits::default() + }; + + assert!(matches!( + read_origin(&bytes, limits), + Err(OriginError::LimitExceeded { + resource: "total owned bytes", + limit, + actual, + }) if limit == total_limit && actual > total_limit + )); +} + +#[test] +fn supported_opj_usage_covers_the_retained_probe_allocation() { + let limits = OriginLimits::default(); + let accounted = probe_origin_with_limits(OPENOPJ_FIXTURE, &limits, OPENOPJ_FIXTURE.len()) + .expect("the supported probe must be accounted under the read limits"); + let retained_probe_bytes = accounted.retained_parser_bytes; + assert_eq!(retained_probe_bytes, accounted.probe.raw_version.capacity()); + + let unseeded = opj::read(OPENOPJ_FIXTURE, &limits, accounted.probe, 0).unwrap(); + let project = read_origin(OPENOPJ_FIXTURE, limits).unwrap(); + + assert_eq!(retained_probe_bytes, project.probe.raw_version.capacity()); + assert_eq!(project.resource_usage.input_bytes, OPENOPJ_FIXTURE.len()); + assert_eq!( + project.resource_usage.parser_bytes, + unseeded.resource_usage.parser_bytes + retained_probe_bytes + ); + assert_eq!( + project.resource_usage.total_owned_bytes, + unseeded.resource_usage.total_owned_bytes + retained_probe_bytes + ); +} + +#[test] +fn opju_requires_the_exact_verified_header_grammar() { + for bytes in [ + b"CPYUA 178\n".as_slice(), + b"CPYUA 4.3668\n".as_slice(), + b"CPYUA 4.3668 178#\n".as_slice(), + b"CPYUA four.3668 178\n".as_slice(), + b"CPYUA 4.minor 178\n".as_slice(), + b"CPYUA 4.3668 build\n".as_slice(), + ] { + assert!(matches!( + probe_origin(bytes), + Err(OriginError::MalformedHeader { .. }) + )); + } +} + +#[test] +fn unsupported_classic_versions_are_not_claimed_as_supported() { + assert!(matches!( + probe_origin(b"CPYA 4.2673 551#\n"), + Err(OriginError::UnsupportedVersion { .. }) + )); +} + +#[test] +fn classic_w64_header_is_recognized_as_an_unsupported_version() { + assert_eq!( + probe_origin(b"CPYA 4.3224 220 W64 #\n").unwrap_err(), + OriginError::UnsupportedVersion { + raw_version: "4.3224 220 W64".to_owned(), + } + ); +} + +#[test] +fn default_limits_match_the_public_contract() { + let limits = OriginLimits::default(); + assert_eq!(limits.max_input_bytes, 128 * 1024 * 1024); + assert_eq!(limits.max_header_bytes, 128); + assert_eq!(limits.max_block_bytes, 32 * 1024 * 1024); + assert_eq!(limits.max_string_bytes, 1024 * 1024); + assert_eq!(limits.max_decoded_text_bytes, 32 * 1024 * 1024); + assert_eq!(limits.max_parser_bytes, 128 * 1024 * 1024); + assert_eq!(limits.max_total_owned_bytes, 384 * 1024 * 1024); + assert_eq!(limits.max_workbooks, 256); + assert_eq!(limits.max_window_records, 1024); + assert_eq!(limits.max_worksheets_per_workbook, 128); + assert_eq!(limits.max_columns, 4096); + assert_eq!(limits.max_metadata_records, 65_536); + assert_eq!(limits.max_rows_per_column, 1_000_000); + assert_eq!(limits.max_cells, 2_000_000); + assert_eq!(limits.max_metadata_depth, 32); +} + +#[test] +fn rejects_a_zero_metadata_record_limit() { + let limits = OriginLimits { + max_metadata_records: 0, + ..OriginLimits::default() + }; + assert!(matches!( + read_origin(b"CPYUA 4.3668 178\n", limits), + Err(OriginError::InvalidLimit { + name: "max_metadata_records", + value: 0, + .. + }) + )); +} + +#[test] +fn invalid_custom_limits_return_an_error_without_panicking() { + let limits = OriginLimits { + max_header_bytes: 0, + ..OriginLimits::default() + }; + let result = std::panic::catch_unwind(|| read_origin(b"CPYUA 4.3668 178\n", limits)); + + assert!(result.is_ok()); + assert!(matches!( + result.unwrap(), + Err(OriginError::InvalidLimit { + name: "max_header_bytes", + .. + }) + )); +} + +#[test] +fn rejects_max_string_limit_that_cannot_fit_the_metadata_sentinel() { + let limits = OriginLimits { + max_string_bytes: usize::MAX, + ..OriginLimits::default() + }; + + assert!(matches!( + read_origin(OPENOPJ_FIXTURE, limits), + Err(OriginError::InvalidLimit { + name: "max_string_bytes", + value: usize::MAX, + reason, + }) if reason.contains("sentinel") + )); +} + +#[test] +fn project_notes_keep_distinct_names_and_content() { + let project = OriginProject { + probe: probe_origin(&supported_opj_probe_bytes()).unwrap(), + parameters: Vec::new(), + notes: vec![ + OriginNote { + name: "Methods".to_owned(), + content: "Prepared under nitrogen.".to_owned(), + }, + OriginNote { + name: "Observations".to_owned(), + content: "The solution remained clear.".to_owned(), + }, + ], + workbooks: Vec::new(), + diagnostics: Vec::new(), + unsupported_objects: Vec::new(), + resource_usage: OriginResourceUsage::default(), + }; + + assert_eq!(project.notes.len(), 2); + assert_eq!(project.notes[0].name, "Methods"); + assert_eq!(project.notes[0].content, "Prepared under nitrogen."); + assert_eq!(project.notes[1].name, "Observations"); + assert_eq!(project.notes[1].content, "The solution remained clear."); +} + +#[test] +fn enforces_the_header_limit_at_the_lf_byte_boundary() { + let mut accepted = b"CPYUA ".to_vec(); + accepted.extend(std::iter::repeat_n(b'0', 111)); + accepted.extend_from_slice(b"4.3668 178\n"); + assert_eq!(accepted.len(), 128); + + let probe = probe_origin(&accepted).unwrap(); + assert_eq!(probe.format, OriginFormat::Opju); + assert_eq!(probe.version.major, 4); + assert_eq!(probe.version.minor, 3668); + assert_eq!(probe.version.build, 178); + + let mut too_long = b"CPYUA ".to_vec(); + too_long.extend(std::iter::repeat_n(b'0', 112)); + too_long.extend_from_slice(b"4.3668 178\n"); + assert_eq!(too_long.len(), 129); + assert!(matches!( + probe_origin(&too_long), + Err(OriginError::HeaderTooLong { limit: 128 }) + )); +} + +#[test] +fn rejects_complete_magic_with_an_invalid_following_byte() { + for bytes in [ + b"CPYAB 4.2673 552#\n".as_slice(), + b"CPYUAX 4.3668 178\n".as_slice(), + ] { + assert!(matches!( + probe_origin(bytes), + Err(OriginError::MalformedHeader { .. }) + )); + } +} + +#[test] +fn parses_numeric_maxima_and_rejects_integer_overflow() { + let opju = probe_origin(b"CPYUA 65535.65535 4294967295\n").unwrap(); + assert_eq!(opju.version.major, u16::MAX); + assert_eq!(opju.version.minor, u16::MAX); + assert_eq!(opju.version.build, u32::MAX); + + assert!(matches!( + probe_origin(b"CPYA 65535.65535 4294967295#\n"), + Err(OriginError::UnsupportedVersion { .. }) + )); + + for bytes in [ + b"CPYUA 65536.1 1\n".as_slice(), + b"CPYUA 1.65536 1\n".as_slice(), + b"CPYUA 1.1 4294967296\n".as_slice(), + ] { + let result = std::panic::catch_unwind(|| probe_origin(bytes)); + assert!(result.is_ok()); + assert!(matches!( + result.unwrap(), + Err(OriginError::MalformedHeader { .. }) + )); + } +} + +mod origin9_profile { + use super::*; + + const SIGNATURE: &[u8] = b"CPYA 4.3268 195 W64 #\n"; + const ORIGIN_HEADER_LEN: usize = 115; + + fn initial_structure(header_len: usize, embedded_version: f64) -> Vec { + let mut bytes = SIGNATURE.to_vec(); + bytes.extend_from_slice(&u32::try_from(header_len).unwrap().to_le_bytes()); + bytes.push(b'\n'); + let mut header = vec![0; header_len]; + header[0x1b..0x23].copy_from_slice(&embedded_version.to_le_bytes()); + bytes.extend_from_slice(&header); + bytes.push(b'\n'); + bytes.extend_from_slice(&[0, 0, 0, 0, b'\n']); + bytes + } + + fn assert_structured_probe_error(bytes: &[u8]) { + let result = std::panic::catch_unwind(|| probe_origin(bytes)); + let probe = result.expect("probing an invalid Origin 9 prefix must not panic"); + if let Ok(probe) = probe { + assert_ne!(probe.profile, Some(OriginProfile::Origin7V552)); + panic!("an invalid Origin 9 prefix must return a structured error"); + } + } + + #[test] + fn recognizes_exact_origin9_profile() { + let bytes = initial_structure(ORIGIN_HEADER_LEN, 9.510195); + + let probe = probe_origin(&bytes).unwrap(); + + assert_eq!(probe.profile, Some(OriginProfile::Origin9V951)); + assert_eq!(probe.support, OriginSupport::Supported); + assert_eq!(probe.raw_version, "4.3268 195 W64"); + } + + #[test] + fn rejects_114_byte_global_header() { + assert_structured_probe_error(&initial_structure(114, 9.510195)); + } + + #[test] + fn rejects_116_byte_global_header() { + assert_structured_probe_error(&initial_structure(116, 9.510195)); + } + + #[test] + fn rejects_changed_embedded_version() { + assert_structured_probe_error(&initial_structure(ORIGIN_HEADER_LEN, 9.510194)); + } + + #[test] + fn rejects_every_truncated_initial_structure_prefix() { + let bytes = initial_structure(ORIGIN_HEADER_LEN, 9.510195); + for end in 0..bytes.len() { + assert_structured_probe_error(&bytes[..end]); + } + } +} + +mod origin7_profile { + use super::*; + + const SIGNATURE: &[u8] = b"CPYA 4.2673 552#\n"; + const ORIGIN_HEADER_LEN: usize = 39; + const DATA_HEADER_LEN: usize = 123; + const SIZE_PREFIX_LEN: usize = 5; + const NULL_BLOCK_LEN: usize = 5; + + fn push_block(bytes: &mut Vec, payload: Option<&[u8]>) { + let payload = payload.unwrap_or_default(); + let length = u32::try_from(payload.len()).unwrap(); + bytes.extend_from_slice(&length.to_le_bytes()); + bytes.push(b'\n'); + if !payload.is_empty() { + bytes.extend_from_slice(payload); + bytes.push(b'\n'); + } + } + + fn origin_header() -> [u8; ORIGIN_HEADER_LEN] { + let mut header = [0; ORIGIN_HEADER_LEN]; + header[0x1b..0x23].copy_from_slice(&7.0552_f64.to_le_bytes()); + header + } + + fn data_header() -> [u8; DATA_HEADER_LEN] { + [0; DATA_HEADER_LEN] + } + + fn synthetic_project(contents: &[Option<&[u8]>]) -> Vec { + let mut bytes = SIGNATURE.to_vec(); + push_block(&mut bytes, Some(&origin_header())); + push_block(&mut bytes, None); + for content in contents { + push_block(&mut bytes, Some(&data_header())); + push_block(&mut bytes, *content); + push_block(&mut bytes, None); + } + push_block(&mut bytes, None); + bytes + } + + fn first_data_header_offset() -> usize { + SIGNATURE.len() + SIZE_PREFIX_LEN + ORIGIN_HEADER_LEN + 1 + NULL_BLOCK_LEN + } + + #[test] + fn accepts_exact_header_and_empty_data_list_framing() { + let bytes = synthetic_project(&[]); + let probe = probe_origin(&bytes).unwrap(); + assert_eq!(probe.profile, Some(OriginProfile::Origin7V552)); + assert!(matches!( + read_origin(&bytes, OriginLimits::default()), + Err(OriginError::NoSupportedWorksheet) + )); + } + + #[test] + fn framed_data_without_required_metadata_is_truncated() { + let bytes = synthetic_project(&[Some(b"values"), None]); + assert!(matches!( + read_origin(&bytes, OriginLimits::default()), + Err(OriginError::Truncated { .. }) + )); + } + + #[test] + fn rejects_other_producer_version() { + let mut bytes = synthetic_project(&[]); + bytes[14] = b'1'; + assert!(matches!( + read_origin(&bytes, OriginLimits::default()), + Err(OriginError::UnsupportedVersion { .. }) + )); + } + + #[test] + fn rejects_wrong_origin_header_length() { + let mut bytes = synthetic_project(&[]); + bytes[SIGNATURE.len()..SIGNATURE.len() + 4] + .copy_from_slice(&(ORIGIN_HEADER_LEN as u32 - 1).to_le_bytes()); + let final_header_byte = SIGNATURE.len() + SIZE_PREFIX_LEN + ORIGIN_HEADER_LEN - 1; + bytes.remove(final_header_byte); + assert!(matches!( + read_origin(&bytes, OriginLimits::default()), + Err(OriginError::CorruptStructure { + offset, + .. + }) if offset == SIGNATURE.len() + )); + } + + #[test] + fn rejects_wrong_embedded_origin_version() { + let mut bytes = synthetic_project(&[]); + let version_offset = SIGNATURE.len() + SIZE_PREFIX_LEN + 0x1b; + bytes[version_offset..version_offset + 8].copy_from_slice(&7.0551_f64.to_le_bytes()); + assert!(matches!( + read_origin(&bytes, OriginLimits::default()), + Err(OriginError::UnsupportedVersion { .. }) + )); + } + + #[test] + fn rejects_bad_origin_header_size_delimiter_with_offset() { + let mut bytes = synthetic_project(&[]); + let delimiter = SIGNATURE.len() + 4; + bytes[delimiter] = b'!'; + assert!(matches!( + read_origin(&bytes, OriginLimits::default()), + Err(OriginError::CorruptStructure { offset, .. }) if offset == delimiter + )); + } + + #[test] + fn rejects_bad_origin_header_payload_delimiter_with_offset() { + let mut bytes = synthetic_project(&[]); + let delimiter = SIGNATURE.len() + SIZE_PREFIX_LEN + ORIGIN_HEADER_LEN; + bytes[delimiter] = b'!'; + assert!(matches!( + read_origin(&bytes, OriginLimits::default()), + Err(OriginError::CorruptStructure { offset, .. }) if offset == delimiter + )); + } + + #[test] + fn rejects_declared_oversized_block_before_payload_access() { + let mut bytes = synthetic_project(&[]); + let limits = OriginLimits { + max_block_bytes: ORIGIN_HEADER_LEN, + ..OriginLimits::default() + }; + bytes[SIGNATURE.len()..SIGNATURE.len() + 4] + .copy_from_slice(&(ORIGIN_HEADER_LEN as u32 + 1).to_le_bytes()); + assert!(matches!( + read_origin(&bytes, limits), + Err(OriginError::LimitExceeded { + resource: "block bytes", + limit: ORIGIN_HEADER_LEN, + actual, + }) if actual == ORIGIN_HEADER_LEN + 1 + )); + } + + #[test] + fn rejects_missing_origin_header_null_block() { + let mut bytes = synthetic_project(&[Some(b"value")]); + let header_null = first_data_header_offset() - NULL_BLOCK_LEN; + bytes.drain(header_null..header_null + NULL_BLOCK_LEN); + assert!(matches!( + read_origin(&bytes, OriginLimits::default()), + Err(OriginError::CorruptStructure { offset, .. }) if offset == header_null + )); + } + + #[test] + fn rejects_wrong_data_header_length() { + let mut bytes = synthetic_project(&[Some(b"value")]); + let data_header = first_data_header_offset(); + bytes[data_header..data_header + 4] + .copy_from_slice(&(DATA_HEADER_LEN as u32 - 1).to_le_bytes()); + bytes.remove(data_header + SIZE_PREFIX_LEN + DATA_HEADER_LEN - 1); + assert!(matches!( + read_origin(&bytes, OriginLimits::default()), + Err(OriginError::CorruptStructure { offset, .. }) if offset == data_header + )); + } + + #[test] + fn rejects_missing_data_content_block_as_truncated() { + let mut bytes = synthetic_project(&[Some(b"value")]); + let content = first_data_header_offset() + SIZE_PREFIX_LEN + DATA_HEADER_LEN + 1; + let content_len = SIZE_PREFIX_LEN + b"value".len() + 1; + bytes.drain(content..content + content_len); + assert!(matches!( + read_origin(&bytes, OriginLimits::default()), + Err(OriginError::Truncated { .. }) + )); + } + + #[test] + fn rejects_missing_per_section_null_as_truncated() { + let mut bytes = synthetic_project(&[Some(b"value")]); + let section_null = first_data_header_offset() + + SIZE_PREFIX_LEN + + DATA_HEADER_LEN + + 1 + + SIZE_PREFIX_LEN + + b"value".len() + + 1; + bytes.drain(section_null..section_null + NULL_BLOCK_LEN); + assert!(matches!( + read_origin(&bytes, OriginLimits::default()), + Err(OriginError::Truncated { .. }) + )); + } + + #[test] + fn rejects_missing_data_list_terminator_as_truncated() { + let mut bytes = synthetic_project(&[]); + bytes.truncate(bytes.len() - NULL_BLOCK_LEN); + assert!(matches!( + read_origin(&bytes, OriginLimits::default()), + Err(OriginError::Truncated { .. }) + )); + } + + #[test] + fn rejects_the_4097th_raw_data_section() { + let contents = vec![None::<&[u8]>; 4097]; + let bytes = synthetic_project(&contents); + + assert!(matches!( + read_origin(&bytes, OriginLimits::default()), + Err(OriginError::LimitExceeded { + resource: "data sections", + limit: 4096, + actual: 4097, + }) + )); + } + + #[test] + fn every_truncated_project_prefix_returns_a_structured_error() { + let complete = synthetic_project(&[]); + for prefix_len in 0..complete.len() { + let result = std::panic::catch_unwind(|| { + read_origin(&complete[..prefix_len], OriginLimits::default()) + }); + assert!(result.is_ok(), "prefix {prefix_len} panicked"); + assert!(matches!( + result.unwrap(), + Err(OriginError::Truncated { .. }) + )); + } + } +} diff --git a/crates/io/tests/fixtures/origin/OPENOPJ-LICENSE.txt b/crates/io/tests/fixtures/origin/OPENOPJ-LICENSE.txt new file mode 100644 index 0000000..b49c2ba --- /dev/null +++ b/crates/io/tests/fixtures/origin/OPENOPJ-LICENSE.txt @@ -0,0 +1,20 @@ +Copyright (c) 2012 Juliusz Gonera, Minor Laboratory, University of Virginia + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/crates/io/tests/fixtures/origin/README.md b/crates/io/tests/fixtures/origin/README.md new file mode 100644 index 0000000..22a666a --- /dev/null +++ b/crates/io/tests/fixtures/origin/README.md @@ -0,0 +1,38 @@ +# Public Origin project fixtures + +This directory contains only publicly redistributed Origin project files used as +test fixtures. Neither fixture contains PlotX user data. + +## `test-origin-7.0552.opj` + +- Source URL: + +- Original filename: `test.opj` +- Byte length: 282,034 bytes +- SHA-256: `ac7f71c367562e85e9d4bb4ae418cbcaaa1b5dff80436180e8d3331c7e1d6308` +- License: MIT; the complete license is in + [`OPENOPJ-LICENSE.txt`](OPENOPJ-LICENSE.txt). +- Attribution: Copyright (c) 2012 Juliusz Gonera, Minor Laboratory, University + of Virginia. The fixture comes from the OpenOPJ project. + +The downloaded file content is unchanged; only its repository filename differs +from the original. + +## `RawData_Locust_Revision1_TIS_Mechanism.opju` + +- Source URL: +- Original filename: `RawData_Locust_Revision1_TIS_Mechanism.opju` +- Byte length: 64,954 bytes +- SHA-256: `13c47a6a5daaf14493da59c8f1b284d9efb08129c8320b6ad9fd0b5191faa55f` +- Figshare record and DOI: + +- License: Creative Commons Attribution 4.0 International (CC BY 4.0), + . +- Attribution: Aleksandar Opancar, Petra Ondrackova, David Rose, Jan Trajlinek, + Vedran Derek, and Eric Glowacki, "The same biophysical mechanism is involved + in both temporal interference and direct kHz stimulation of peripheral + nerves," Figshare dataset (2025), DOI 10.6084/m9.figshare.28535426.v1. + +The fixture is redistributed unchanged. CC BY 4.0 permits sharing and +adaptation provided appropriate credit is given, a link to the license is +included, and changes are indicated. diff --git a/crates/io/tests/fixtures/origin/RawData_Locust_Revision1_TIS_Mechanism.opju b/crates/io/tests/fixtures/origin/RawData_Locust_Revision1_TIS_Mechanism.opju new file mode 100644 index 0000000..9b47048 Binary files /dev/null and b/crates/io/tests/fixtures/origin/RawData_Locust_Revision1_TIS_Mechanism.opju differ diff --git a/crates/io/tests/fixtures/origin/test-origin-7.0552.opj b/crates/io/tests/fixtures/origin/test-origin-7.0552.opj new file mode 100644 index 0000000..0dbe7f1 Binary files /dev/null and b/crates/io/tests/fixtures/origin/test-origin-7.0552.opj differ diff --git a/crates/io/tests/origin_fixtures.rs b/crates/io/tests/origin_fixtures.rs new file mode 100644 index 0000000..bcb4718 --- /dev/null +++ b/crates/io/tests/origin_fixtures.rs @@ -0,0 +1,238 @@ +use plotx_io::origin::{ + OriginCell, OriginError, OriginFormat, OriginLimits, OriginProfile, OriginProject, + probe_origin, read_origin, +}; + +const MIXED_NUMERIC_VALUE: f64 = 314.0 / 100.0; + +fn cell<'a>( + project: &'a OriginProject, + workbook_name: &str, + column_name: &str, + row: usize, +) -> Option<&'a OriginCell> { + project + .workbooks + .iter() + .find(|workbook| workbook.name == workbook_name)? + .worksheets + .iter() + .flat_map(|worksheet| &worksheet.columns) + .find(|column| column.name == column_name)? + .cells + .get(row) +} + +fn assert_float_cell( + project: &OriginProject, + workbook: &str, + column: &str, + row: usize, + expected: f64, +) { + let Some(OriginCell::Float(actual)) = cell(project, workbook, column, row) else { + panic!("expected a floating-point cell at {workbook}/{column}/{row}"); + }; + assert_eq!(*actual, expected); +} + +fn parameter(project: &OriginProject, key: &str) -> f64 { + project + .parameters + .iter() + .find(|entry| entry.key == key) + .unwrap_or_else(|| panic!("missing project parameter {key}")) + .value + .parse() + .unwrap_or_else(|error| panic!("parameter {key} is not numeric: {error}")) +} + +#[test] +fn imports_openopj_origin_7_v552_fixture() { + let bytes = include_bytes!("fixtures/origin/test-origin-7.0552.opj"); + let project = read_origin(bytes, OriginLimits::default()) + .expect("the licensed OpenOPJ fixture should import"); + + assert_eq!(project.probe.profile, Some(OriginProfile::Origin7V552)); + assert_eq!( + project + .workbooks + .iter() + .map(|workbook| workbook.name.as_str()) + .collect::>(), + ["Data1", "Data1Coeff", "Data1spline", "TestW"] + ); + assert!(project.workbooks.iter().all(|workbook| { + workbook.worksheets.len() == 1 && workbook.worksheets[0].name == "Sheet1" + })); + assert_eq!( + project + .workbooks + .iter() + .map(|workbook| { + let worksheet = &workbook.worksheets[0]; + ( + workbook.name.as_str(), + worksheet.row_count, + worksheet.columns.len(), + ) + }) + .collect::>(), + [ + ("Data1", 21, 5), + ("Data1Coeff", 774, 4), + ("Data1spline", 481, 1), + ("TestW", 3, 6), + ] + ); + assert_eq!( + cell(&project, "Data1", "INJV", 0), + Some(&OriginCell::Float(0.4)) + ); + assert_eq!( + cell(&project, "Data1", "INJV", 19), + Some(&OriginCell::Float(2.0)) + ); + assert_eq!(cell(&project, "Data1", "INJV", 20), Some(&OriginCell::Null)); + assert_eq!( + cell(&project, "TestW", "TextNumeric", 0), + Some(&OriginCell::Text("text".to_owned())) + ); + assert_eq!( + cell(&project, "TestW", "TextNumeric", 1), + Some(&OriginCell::Float(MIXED_NUMERIC_VALUE)) + ); + + assert_float_cell( + &project, + "TestW", + "Float", + 0, + f64::from(f32::from_bits(0x43ac_cccd)), + ); + assert_float_cell( + &project, + "TestW", + "Float", + 1, + f64::from(f32::from_bits(0xc7c3_501a)), + ); + assert_eq!( + cell(&project, "TestW", "Long", 0), + Some(&OriginCell::Integer(345)) + ); + assert_eq!( + cell(&project, "TestW", "Long", 1), + Some(&OriginCell::Integer(-100000)) + ); + assert_eq!( + cell(&project, "TestW", "Integer", 0), + Some(&OriginCell::Integer(34)) + ); + assert_eq!( + cell(&project, "TestW", "Integer", 1), + Some(&OriginCell::Integer(-1000)) + ); + assert_eq!( + cell(&project, "TestW", "Text", 0), + Some(&OriginCell::Text("test string 123".to_owned())) + ); + assert_eq!( + cell(&project, "TestW", "Text", 1), + Some(&OriginCell::Text("only text".to_owned())) + ); + assert_eq!( + cell(&project, "TestW", "firstRow", 0), + Some(&OriginCell::Null) + ); + assert_float_cell(&project, "TestW", "firstRow", 1, 5.23); + assert_float_cell(&project, "TestW", "firstRow", 2, -7.0); + + assert_eq!(project.parameters.len(), 41); + assert_eq!(parameter(&project, "ERR"), 1.0); + assert_eq!(parameter(&project, "SYRNG_C_DATA1"), 1.25); + assert_eq!(parameter(&project, "CELL_C_DATA1"), 0.1246); + assert!((parameter(&project, "S") - 1.28889201142965).abs() < 1.0e-14); + + assert_eq!( + project + .notes + .iter() + .find(|note| note.name == "Results") + .map(|note| note.content.as_str()), + Some("Data1 Temperature:\t25.10242\r\n\r\n") + ); + assert_eq!( + project + .notes + .iter() + .find(|note| note.name == "ResultsLog") + .map(|note| note.content.as_str()), + Some( + "[3/5/2009 13:32 \"/DeltaH\" (2454895)]\r\n\ +Data: Data1_NDH\r\n\ +Model: OneSites\r\n\ +Chi^2/DoF = 3008\r\n\ +N\t0.800\t0.0346\r\n\ +K\t1.75E4\t1.86E3\r\n\ +H\t-5406\t340.5\r\n\ +S\t1.29\r\n\r\n" + ) + ); + + assert_eq!(project.resource_usage.workbooks, 4); + assert_eq!(project.resource_usage.worksheets, 4); + assert_eq!(project.resource_usage.columns, 16); + assert_eq!(project.resource_usage.cells, 1889); + assert_eq!(project.resource_usage.metadata_records, 362); + assert_eq!( + project + .unsupported_objects + .iter() + .find(|summary| summary.kind == "worksheet columns") + .map(|summary| summary.count), + Some(23) + ); + assert!(project.diagnostics.iter().any(|diagnostic| { + diagnostic.code == plotx_io::origin::OriginDiagnosticCode::UnsupportedColumnSkipped + })); + assert!( + project + .unsupported_objects + .iter() + .any(|summary| { summary.kind == "project tree" && summary.count == 1 }) + ); + assert!( + project + .unsupported_objects + .iter() + .any(|summary| { summary.kind == "embedded attachments" && summary.count == 1 }) + ); + assert!( + project + .unsupported_objects + .iter() + .any(|summary| { summary.kind == "window presentation records" && summary.count > 0 }) + ); + assert!( + project + .unsupported_objects + .iter() + .any(|summary| { summary.kind == "note properties" && summary.count == 2 }) + ); +} + +#[test] +fn recognizes_public_opju_fixture_without_partial_output() { + let bytes = include_bytes!("fixtures/origin/RawData_Locust_Revision1_TIS_Mechanism.opju"); + let probe = probe_origin(bytes).expect("the public fixture has a recognized OPJU header"); + assert_eq!(probe.format, OriginFormat::Opju); + let error = read_origin(bytes, OriginLimits::default()).unwrap_err(); + assert_eq!( + error, + OriginError::UnsupportedOpjuVariant { + message: "This OPJU file uses a record layout that PlotX does not support yet. No data was imported." + .to_owned(), + } + ); +} diff --git a/docs/src/content/docs/guides/importing-data.md b/docs/src/content/docs/guides/importing-data.md index eeef0e4..cf8b0ef 100644 --- a/docs/src/content/docs/guides/importing-data.md +++ b/docs/src/content/docs/guides/importing-data.md @@ -17,6 +17,7 @@ no conversion step is needed. | JCAMP-DX | `.dx` / `.jdx` / `.jcamp` | 1D frequency-domain NMR spectra | | Axon Binary Format 2 | `.abf` | int16/float32, multiple channels and sweeps, embedded DAC/epoch stimuli | | Tabular data | `.csv`, `.tsv`, `.txt`, `.xlsx` | Column types and empty cells preserved; one table per XLSX worksheet | +| Origin project (experimental) | `.opj`, `.opju` | Worksheets from the verified Origin 7.0552 and Origin 9.51 OPJ profiles; graphs are not imported, and `.opju` is detection-only. See [compatibility details](/reference/file-formats/). | | Zip archive | `.zip` | An archived dataset folder | | PlotX project | `.plotx` | Full project: data, processing, and layout | @@ -24,7 +25,7 @@ no conversion step is needed. Drag a file onto the PlotX window, or use the toolbar's open menu: *Open File…*, *Open Folder…* (for acquisition directories such as Bruker -TopSpin and Waters MassLynx RAW), *Open Project…*, or *Import Table / CSV…*. +TopSpin and Waters MassLynx RAW), *Open Project…*, or *Import Table…*. Each imported dataset appears in the Primary Side Bar and is placed on the board automatically. The file picker accepts several ABF files at once. Opening a folder recursively @@ -87,7 +88,7 @@ import** dialog. It shows each column's inferred type and unit, whether the column allows empty cells, a preview of the first rows, and any import diagnostics. Choose **Import table** to add it, or **Cancel** to leave your project and recent-file list untouched. An XLSX workbook with several sheets -adds a **Worksheet** selector so you can preview each one; a single **Import +adds a **Table** selector so you can preview each worksheet; a single **Import table** brings them all in as separate tables. PlotX keeps Boolean, whole-number, decimal, text, and empty cells distinct. A @@ -111,6 +112,22 @@ formula cell with no cached value imports as empty and is listed in the diagnostics. Exported XLSX files hold plain values, so they never depend on Excel recalculating them. +## Origin project import (experimental) + +Origin `.opj` and `.opju` files appear in the file picker for both *Open +File…* and *Import Table…*. Both routes identify the format from file +content and signatures rather than relying only on the extension. + +When a supported `.opj` yields worksheets, PlotX opens the existing **Review +table import** preview so you can inspect every candidate table. Confirm once +to import all candidates, or cancel to leave the current project and recent-file +list unchanged. While a preview is pending, selecting a second table path is +rejected with a clear message; finish or cancel the current preview first. + +Origin does not need to be installed or launched, and PlotX does not automate +or invoke it. See [File formats](/reference/file-formats/) for the exact, +evidence-limited compatibility boundary. + ## Pseudo-2D experiments DOSY, T1, and T2 experiments are detected automatically from the acquisition diff --git a/docs/src/content/docs/reference/file-formats.md b/docs/src/content/docs/reference/file-formats.md index 6fefd2c..b46c58b 100644 --- a/docs/src/content/docs/reference/file-formats.md +++ b/docs/src/content/docs/reference/file-formats.md @@ -1,6 +1,6 @@ --- title: File formats -description: What PlotX's own files contain and how safely they can be shared. +description: Native PlotX files, imported formats, and their compatibility boundaries. --- ## `.plotx` projects @@ -52,6 +52,57 @@ are covered in [the command line](/reference/cli/). A workflow is not a recipe: a recipe holds one processing pipeline, while a workflow describes a whole run and may reference a recipe as one of its steps. +## Origin project import (experimental) + +Origin project import is experimental. Successful import is limited to two +exact, content-detected OPJ producer profiles: + +- Origin 7.0552 (`CPYA 4.2673 build 552`) imports verified `f64`, `f32`, signed + `i32`, signed `i16`, fixed-width ASCII text, mixed numeric/text cells, nulls, + and nonzero row offsets. Project parameters and notes are retained as source + metadata. +- Origin 9.51 build 195 W64 (`CPYA 4.3268 build 195 W64`) imports worksheet + names, column names, numeric `f64` values, nulls, and validated empty columns. + This modern profile does not yet import text cells, project parameters, or + notes. + +Compatibility claims are limited to the committed Origin 7 regression fixture +and the exact Origin 9.51 profile checked against two real projects, companion +CSV exports, and an independent parser comparison. They do not extend to other +files merely because the extension or major Origin version is the same. + +PlotX preserves validated Origin window or group names and column names. Each +supported window is represented as one table under the generated worksheet +name `Sheet1`; this release does not claim to decode original worksheet labels. +Mixed Origin 7 columns are retained as text, and unequal column lengths are +padded with nulls. There is no verified-support claim for long names, units, +comments, column designations, dates, categorical values, or unverified code +pages. + +An `.opju` file is recognized from its CPYUA content signature, but `.opju` is +not importable in this release and PlotX creates no partial OPJU result. + +Unsupported content includes graphs, formulas, scripts, analysis +recomputation, saved analysis results as executable analyses, matrices, +embedded objects, modern OPJ text cells, non-ASCII text without a verified code +page, encrypted or protected projects, unverified OPJ versions or profiles, and +unverified OPJU containers. For the supported Origin 9.51 profile, PlotX stops +after the validated window-list boundary and warns that the remaining project +objects were not imported. + +PlotX never silently or heuristically guesses an import. Corrupt or truncated +files, files above the current 128 MiB input cap, extension/signature-family +mismatches, and malformed or otherwise unsupported files produce a clear error +before any table is committed. Inside an otherwise supported OPJ, an +unsupported worksheet column may be omitted, or an unsupported non-table object +skipped, only when each is independently framed and its outer boundaries are +trusted. PlotX shows warnings for every such omission; an imported worksheet +may therefore contain only the supported columns, not every source column. If +framing is ambiguous or untrusted, PlotX rejects the file rather than guessing +boundaries or silently shifting data. + +Origin need not be installed, launched, or called during import. + ## Data you import and export See [Importing data](/guides/importing-data/) for the supported instrument and diff --git a/docs/src/content/docs/zh-cn/guides/importing-data.md b/docs/src/content/docs/zh-cn/guides/importing-data.md index 280d540..4bddaf7 100644 --- a/docs/src/content/docs/zh-cn/guides/importing-data.md +++ b/docs/src/content/docs/zh-cn/guides/importing-data.md @@ -16,6 +16,7 @@ PlotX 直接读取厂商 LC–MS、NMR、AFM 与电生理格式，无需任何 | JCAMP-DX | `.dx` / `.jdx` / `.jcamp` | 1D 频域 NMR 谱 | | Axon Binary Format 2 | `.abf` | int16/float32、多通道、多 sweep，以及文件内 DAC/epoch 刺激 | | 表格数据 | `.csv`、`.tsv`、`.txt`、`.xlsx` | 保留列类型与空单元格；每个 XLSX 工作表导入为独立数据表 | +| Origin 项目（实验性） | `.opj`、`.opju` | 经验证的 Origin 7.0552 与 Origin 9.51 OPJ 配置中的工作表；不导入图形，`.opju` 仅作识别。见[兼容性详情](/zh-cn/reference/file-formats/)。 | | Zip 压缩包 | `.zip` | 打包的数据文件夹 | | PlotX 项目 | `.plotx` | 完整项目：数据、处理与排版 | @@ -23,7 +24,7 @@ PlotX 直接读取厂商 LC–MS、NMR、AFM 与电生理格式，无需任何 把文件拖到 PlotX 窗口上，或使用工具栏的打开菜单：*Open File…*、 *Open Folder…*（用于 Bruker TopSpin 与 Waters MassLynx RAW 等采集目录）、 -*Open Project…* 或 *Import Table / CSV…*。每个导入的数据集会出现在主侧栏中， +*Open Project…* 或 *Import Table…*。每个导入的数据集会出现在主侧栏中， 并自动放置到画板上。 文件选择器可以一次选择多个 ABF。打开文件夹时会递归导入其中所有 `.abf`、 `.spm`、`.pfc` 和已识别的 `.raw` 数据包。每个 `.raw` 目录会作为一次完整采集 @@ -70,8 +71,8 @@ PlotX 不提供 LC–MS 处理流程。导入的数据、活动功能、检测 无论从文件还是剪贴板导入表格，都会先打开 **Review table import** 对话框。它会 列出每列推断出的类型和单位、该列是否允许空单元格、前几行的预览，以及任何导入 诊断。选择 **Import table** 导入，或选择 **Cancel** 保持项目与最近文件列表不变。 -含多个工作表的 XLSX 会额外提供 **Worksheet** 选择器，可逐一预览各工作表；一次 -**Import table** 会把它们作为独立数据表全部导入。 +含多个工作表的 XLSX 会额外提供 **Table** 选择器，可逐一预览工作簿中的各工作表； +一次 **Import table** 会把它们作为独立数据表全部导入。 PlotX 会区分布尔、整数、小数、文本和空单元格。混合了不同类型、或取值含糊的列会 保留为文本而不会被丢弃。除非文件自带 PlotX 的类型信息（见下），只有毫不含糊的 @@ -89,6 +90,20 @@ PlotX 导出 CSV 或 TSV 时，会在旁边写入一个配套的 `.plotx-schema. 没有缓存值的公式单元格会以空导入，并列入诊断。导出的 XLSX 文件只包含确定值， 因此不依赖 Excel 重新计算。 +## Origin 项目导入（实验性） + +Origin 的 `.opj` 与 `.opju` 文件会出现在 *Open File…* 和 *Import Table…* +两个入口的文件选择器中。这两个入口均根据文件内容与签名识别格式， +而不是只看扩展名。 + +受支持的 `.opj` 成功生成工作表后，PlotX 会打开现有的 **Review table +import** 预览，可先检查每个候选数据表。确认一次会导入全部候选数据表； +取消则保持当前项目和最近文件列表不变。预览尚未处理完时，若再选择第二个 +表格路径，PlotX 会给出明确提示并拒绝该操作；请先完成或取消当前预览。 + +无需安装或启动 Origin，PlotX 也不会自动化或调用 Origin。严格且以证据为限的 +兼容范围见[文件格式](/zh-cn/reference/file-formats/)。 + ## 伪 2D 实验 DOSY、T1、T2 实验会根据采集参数自动识别，并获得专属的分析工具——参见 diff --git a/docs/src/content/docs/zh-cn/reference/file-formats.md b/docs/src/content/docs/zh-cn/reference/file-formats.md index 1fb3faa..0674892 100644 --- a/docs/src/content/docs/zh-cn/reference/file-formats.md +++ b/docs/src/content/docs/zh-cn/reference/file-formats.md @@ -1,6 +1,6 @@ --- title: 文件格式 -description: PlotX 自有文件的内容，以及它们的分享与兼容性。 +description: PlotX 自有文件、导入格式及其兼容性边界。 --- ## `.plotx` 项目 @@ -44,6 +44,46 @@ PlotX（或相反）时，文件会被拒绝并给出明确的"不支持的版 工作流不是配方：配方保存一条处理管线，而工作流描述一整次运行，可以把 配方作为其中一个步骤引用。 +## Origin 项目导入（实验性） + +Origin 项目导入仍属实验性。成功导入仅限两种通过文件内容精确识别的 OPJ +生成配置： + +- Origin 7.0552（`CPYA 4.2673 build 552`）可导入经过验证的 `f64`、`f32`、 + 有符号 `i32`、有符号 `i16`、定宽 ASCII 文本、数值与文本混合的单元格、 + 空值和非零行偏移。项目参数与备注会作为来源元数据保留。 +- Origin 9.51 build 195 W64（`CPYA 4.3268 build 195 W64`）可导入工作表名、 + 列名、`f64` 数值、空值和经过验证的空列。该现代配置目前不导入文本单元格、 + 项目参数或备注。 + +兼容性声明仅限仓库中已提交的 Origin 7 回归样本，以及通过两个真实项目、配套 +CSV 导出和独立解析器对照检查的精确 Origin 9.51 配置。不能因为扩展名相同或 +Origin 主版本相同，就推定其他文件也受支持。 + +PlotX 会保留经过验证的 Origin 窗口或分组名与列名。每个受支持的窗口以单个 +数据表表示，并使用合成的工作表名 `Sheet1`；本版本不声称能够解析原始工作表 +标签。Origin 7 的混合列会保留为文本，长度不等的列会用空值补齐。目前不声称 +已验证长名称、单位、注释、列标识、日期、分类值或未经验证的代码页。 + +PlotX 会根据 CPYUA 内容签名识别 `.opju` 文件，但本版本不能导入 `.opju`， +也不会产生任何部分 OPJU 结果。 + +不支持的内容包括图形、公式、脚本、分析重新计算、把已保存的分析结果恢复为 +可执行分析、矩阵、嵌入对象、现代 OPJ 文本单元格、没有已验证代码页的非 ASCII +文本、加密或受保护项目、未经验证的 OPJ 版本或配置，以及未经验证的 OPJU +容器。对于受支持的 Origin 9.51 配置，PlotX 会在经过验证的窗口列表边界停止 +解析，并警告剩余项目对象没有被导入。 + +PlotX 绝不会静默导入，也不会凭推测猜测导入内容。损坏或截断的文件、超过当前 +128 MiB 输入上限的文件、扩展名与签名家族不匹配的文件，以及结构异常或其他 +不受支持的文件，都会在提交任何数据表之前给出明确错误。在其他方面受支持的 +OPJ 中，只有在相应内容具有独立框架且其外层边界可信时，才可以省略不受支持的 +工作表列，或跳过不受支持的非表格对象。PlotX 会为每次此类省略或跳过显示警告； +因此，导入的工作表可能只含受支持的列，而非源文件中的全部列。若框架含糊或 +不可信，PlotX 会拒绝文件，而不会猜测边界或静默造成数据错位。 + +导入过程中无需安装、启动或调用 Origin。 + ## 你导入和导出的数据 受支持的仪器与表格格式见[导入数据](/zh-cn/guides/importing-data/)；图形 diff --git a/xtask/about.hbs b/xtask/about.hbs index 699b3b0..a87ca52 100644 --- a/xtask/about.hbs +++ b/xtask/about.hbs @@ -39,17 +39,46 @@

Third Party Licenses

-

This page lists the licenses of the projects used in cargo-about.

+

This page lists licenses for Cargo dependencies and other third-party projects whose material is distributed with or adapted by PlotX.

+ +

Non-Cargo third-party projects:

+
    +
  • +

    OpenOPJ

    +

    Origin project format documentation and parser structure adapted by PlotX; public test fixture redistributed unchanged.

    +

    OpenOPJ source repository

    + 
    Copyright (c) 2012 Juliusz Gonera, Minor Laboratory, University of Virginia
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
    +
    • +
-

Overview of licenses:

+

Overview of Cargo dependency licenses:

    {{#each overview}}
  • {{name}} ({{count}})
    • {{/each}}
-

All license text:

+

All Cargo dependency license text:

    {{#each licenses}}
  • diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 6564361..f45490e 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -424,3 +424,25 @@ fn format_duration(duration: Duration) -> String { format!("{seconds:.0}s") } } + +#[cfg(test)] +mod tests { + const ABOUT_TEMPLATE: &str = include_str!("../about.hbs"); + const OPENOPJ_LICENSE: &str = + include_str!("../../crates/io/tests/fixtures/origin/OPENOPJ-LICENSE.txt"); + + #[test] + fn license_template_includes_the_complete_openopj_notice() { + assert!(ABOUT_TEMPLATE.contains("OpenOPJ")); + assert!(ABOUT_TEMPLATE.contains( + "Copyright (c) 2012 Juliusz Gonera, Minor Laboratory, University of Virginia" + )); + assert!(ABOUT_TEMPLATE.contains(OPENOPJ_LICENSE.trim())); + } + + #[test] + fn license_template_describes_cargo_and_non_cargo_projects() { + assert!(ABOUT_TEMPLATE.contains("Cargo dependencies")); + assert!(ABOUT_TEMPLATE.contains("other third-party projects")); + } +}