Skip to content

Commit

Permalink
feat: generate martian struct field descriptions and filenames (#480)
Browse files Browse the repository at this point in the history 
* Implement struct descriptions (implemented as doc comments) and renaming
file (mro_rename attribute).

* clean up comment

* Reset test file

* Fix formatting of test file

* Fix formatting detail regarding spaces

* Update martian/src/mro.rs

Co-authored-by: Sreenath Krishnan <sreenath.krishnan@10xgenomics.com>

* Update martian/src/mro.rs

Co-authored-by: Sreenath Krishnan <sreenath.krishnan@10xgenomics.com>

* Default to empty string in the case of missing doc comment

---------

Co-authored-by: Sreenath Krishnan <sreenath.krishnan@10xgenomics.com>
  • Loading branch information
@pjedge @sreenathkrishnan
pjedge and sreenathkrishnan authored Mar 8, 2024
1 parent 345490b commit 5111114
Show file tree
Hide file tree
Showing 5 changed files with 279 additions and 98 deletions.
48 changes: 43 additions & 5 deletions martian-derive/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,9 @@ use martian::{utils, MartianBlanketType, MartianPrimaryType, StageKind, Volatile
use quote::quote;
use std::collections::{HashMap, HashSet};
use std::str::FromStr;
use syn::{Data, DeriveInput, Error, Fields, Ident, ImplItem, ItemImpl, ItemStruct, Type};
use syn::{
Data, DeriveInput, Error, Expr, Fields, Ident, ImplItem, ItemImpl, ItemStruct, Lit, Meta, Type,
};

const ATTR_NOT_ON_TRAIT_IMPL_ERROR: &str = r#"The attribute #[make_mro] should only be applied to `martian::MartianMain` or `martian::MartianStage` trait implementation of a stage struct"#;
const MARTIAN_MAIN_TRAIT: &str = "MartianMain";
Expand Down Expand Up @@ -404,7 +406,7 @@ attr_parse!(
///
/// You can optionally add a field to the "retain" section of the mro using `#[mro_retain]`.
///
#[proc_macro_derive(MartianStruct, attributes(mro_retain, mro_type))]
#[proc_macro_derive(MartianStruct, attributes(mro_retain, mro_type, mro_filename))]
pub fn martian_struct(item: proc_macro::TokenStream) -> proc_macro::TokenStream {
// ::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
// STEP 1
Expand Down Expand Up @@ -450,6 +452,9 @@ pub fn martian_struct(item: proc_macro::TokenStream) -> proc_macro::TokenStream
let name = field.ident.clone().unwrap().to_string();
let mut retain = false;
let mut mro_type = None;
let mut doc_comment = None;
let mut mro_filename = None;

for attr in &field.attrs {
if attr.path().is_ident("mro_retain") {
if !matches!(attr.meta, syn::Meta::Path(_)) {
Expand Down Expand Up @@ -513,6 +518,22 @@ pub fn martian_struct(item: proc_macro::TokenStream) -> proc_macro::TokenStream
}

mro_type = Some(val.value())
} else if attr.path().is_ident("doc") {
if let Meta::NameValue(meta) = &attr.meta {
if let Expr::Lit(elit) = &meta.value {
if let Lit::Str(lstr) = &elit.lit {
doc_comment = Some(lstr.value());
}
}
}
} else if attr.path().is_ident("mro_filename") {
if let Meta::NameValue(meta) = &attr.meta {
if let Expr::Lit(elit) = &meta.value {
if let Lit::Str(lstr) = &elit.lit {
mro_filename = Some(lstr.value());
}
}
}
}
}
if name.starts_with("__") {
Expand All @@ -534,19 +555,36 @@ pub fn martian_struct(item: proc_macro::TokenStream) -> proc_macro::TokenStream
.to_compile_error()
.into();
}
let ty = field.ty;
let ty = field.clone().ty;

let actual_type = match mro_type {
Some(t) => quote![#t.parse().unwrap()],
None => quote![<#ty as ::martian::AsMartianBlanketType>::as_martian_blanket_type()],
};

if mro_filename.is_some() && doc_comment.is_none() {
doc_comment = Some("".to_string());
}

let doc_comment_code = match doc_comment {
Some(t) => {
quote![Some(#t.trim_start().to_string())]
}
None => quote![None],
};

let mro_filename_code = match mro_filename {
Some(t) => quote![Some(#t.to_string())],
None => quote![None],
};

vec_inner.push(if retain {
quote![
<::martian::MroField>::retained(#name, #actual_type)
<::martian::MroField>::retained(#name, #actual_type, #doc_comment_code, #mro_filename_code)
]
} else {
quote![
<::martian::MroField>::new(#name, #actual_type)
<::martian::MroField>::new(#name, #actual_type, #doc_comment_code, #mro_filename_code)
]
});
}
Expand Down
14 changes: 7 additions & 7 deletions martian-derive/tests/mro/test_struct.mro
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,22 +12,22 @@ struct SampleDef(
)

struct Config(
SampleDef[] sample_def,
path reference_path,
int force_cells,
json primers,
SampleDef[] sample_def "The sample definition",
path reference_path "even more info about reference path" "the_reference_path",
int force_cells "The number of cells to force the pipeline to call",
json primers "The primer definitions as a JSON file" "renamed_primers_json.json",
)

struct ChemistryDef(
string name,
string name "The chemistry name",
string barcode_read,
int barcode_length,
int barcode_length "" "the_bc_length",
)

struct RnaChunk(
ChemistryDef chemistry_def,
int chunk_id,
fastq r1,
fastq r1 "The r1 fastq file" "read1.fastq",
)

stage SETUP_CHUNKS(
Expand Down
15 changes: 15 additions & 0 deletions martian-derive/tests/test_full_mro.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -421,23 +421,36 @@ fn test_main_only_full_name() {
fn test_with_struct() {
#[derive(Serialize, Deserialize, MartianStruct)]
struct ChemistryDef {
/// The chemistry name
name: String,
barcode_read: String,
#[mro_filename = "the_bc_length"]
barcode_length: u8,
}

// Currently only the last line is used in the case of multi-line doc comments
#[derive(Serialize, Deserialize, MartianStruct)]
struct Config {
/// The sample definition
sample_def: Vec<SampleDef>,
/// The reference path
/// more info about the reference path
/// even more info about reference path
#[mro_filename = "the_reference_path"]
reference_path: PathBuf,
/// The number of cells to force the pipeline to call
force_cells: u8,
/// The primer definitions as a JSON file
#[mro_filename = "renamed_primers_json.json"]
primers: JsonFile,
}

#[derive(Serialize, Deserialize, MartianStruct)]
struct RnaChunk {
chemistry_def: ChemistryDef,
chunk_id: u8,
/// The r1 fastq file
#[mro_filename = "read1.fastq"]
r1: FastqFile,
}

Expand Down Expand Up @@ -485,8 +498,10 @@ fn test_with_struct() {
fn test_typed_map() {
#[derive(Serialize, Deserialize, MartianStruct)]
struct ReadsStruct {
// The reads name
name: String,
reads_map: HashMap<String, FastqFile>,
// The reads multi map
multi_reads_map: Vec<HashMap<String, FastqFile>>,
}

Expand Down
42 changes: 26 additions & 16 deletions martian-derive/tests/test_martian_struct.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ fn test_simple_vec() {
#[allow(dead_code)]
values: Vec<f64>,
}
let expected = vec![MroField::new("values", Array(Float.into()))];
let expected = vec![MroField::new("values", Array(Float.into()), None, None)];
assert_eq!(expected, SimpleVec::mro_fields())
}

Expand All @@ -30,31 +30,36 @@ fn test_generic() {

assert_eq!(
Generic::<i32>::mro_fields(),
vec![MroField::new("param", Primary(Int))]
vec![MroField::new("param", Primary(Int), None, None)]
);
assert_eq!(
Generic::<f64>::mro_fields(),
vec![MroField::new("param", Primary(Float))]
vec![MroField::new("param", Primary(Float), None, None)]
);
assert_eq!(
Generic::<bool>::mro_fields(),
vec![MroField::new("param", Primary(Bool))]
vec![MroField::new("param", Primary(Bool), None, None)]
);
assert_eq!(
Generic::<TxtFile>::mro_fields(),
vec![MroField::new("param", Primary(FileType("txt".into())))]
vec![MroField::new(
"param",
Primary(FileType("txt".into())),
None,
None
)]
);
assert_eq!(
Generic::<Vec<bool>>::mro_fields(),
vec![MroField::new("param", Array(Bool.into()))]
vec![MroField::new("param", Array(Bool.into()), None, None)]
);
assert_eq!(
Generic::<Vec<String>>::mro_fields(),
vec![MroField::new("param", Array(Str.into()))]
vec![MroField::new("param", Array(Str.into()), None, None)]
);
assert_eq!(
Generic::<HashMap<String, f32>>::mro_fields(),
vec![MroField::new("param", TypedMap(Float.into()))]
vec![MroField::new("param", TypedMap(Float.into()), None, None)]
);
}

Expand All @@ -70,9 +75,9 @@ fn test_generic_two() {
assert_eq!(
GenericTwo::<i32, PathBuf>::mro_fields(),
vec![
MroField::new("foo", Primary(Int)),
MroField::new("bar", Primary(Path)),
MroField::new("far", Primary(Str)),
MroField::new("foo", Primary(Int), None, None),
MroField::new("bar", Primary(Path), None, None),
MroField::new("far", Primary(Str), None, None),
]
);
}
Expand All @@ -85,7 +90,12 @@ fn test_retain() {
#[mro_retain]
values: Vec<f64>,
}
let expected = vec![MroField::retained("values", Array(Float.into()))];
let expected = vec![MroField::retained(
"values",
Array(Float.into()),
None,
None,
)];
assert_eq!(expected, SimpleVec::mro_fields())
}

Expand All @@ -98,7 +108,7 @@ fn test_mro_type_attr() {
#[mro_type = "int"]
value: Foo,
}
let expected = vec![MroField::new("value", Primary(Int))];
let expected = vec![MroField::new("value", Primary(Int), None, None)];
assert_eq!(expected, Simple::mro_fields());

struct Bar {
Expand All @@ -109,7 +119,7 @@ fn test_mro_type_attr() {
#[mro_type = "map[]"]
values: Vec<Bar>,
}
let expected = vec![MroField::new("values", Array(Map.into()))];
let expected = vec![MroField::new("values", Array(Map.into()), None, None)];
assert_eq!(expected, SimpleVec::mro_fields());
}

Expand All @@ -123,7 +133,7 @@ fn test_mro_type_retain_attr() {
#[mro_type = "int"]
value: Foo,
}
let expected = vec![MroField::retained("value", Primary(Int))];
let expected = vec![MroField::retained("value", Primary(Int), None, None)];
assert_eq!(expected, Simple::mro_fields());

struct Bar {
Expand All @@ -135,6 +145,6 @@ fn test_mro_type_retain_attr() {
#[mro_type = "map[]"]
values: Vec<Bar>,
}
let expected = vec![MroField::retained("values", Array(Map.into()))];
let expected = vec![MroField::retained("values", Array(Map.into()), None, None)];
assert_eq!(expected, SimpleVec::mro_fields());
}
Loading

0 comments on commit 5111114

Please sign in to comment.