Merge pull request #40 from microbiomedata/issue-38-tsv-housekeeping

Delete unused sheets_and_friends TSVs

project.Makefile

@@ -231,9 +231,10 @@ local/with_shuttles_yq.yaml: local/with_shuttles.yaml
 modifications_cleanup:
 	rm -rf sheets_and_friends/yaml_out/with_modifications.yaml
 
+# sheets-for-nmdc-submission-schema_validation_converter_empty.tsv
 local/with_modifications.yaml: local/with_shuttles_yq.yaml \
 sheets_and_friends/tsv_in/sheets-for-nmdc-submission-schema_modifications_long-dont-mod-water.tsv \
-sheets_and_friends/tsv_in/sheets-for-nmdc-submission-schema_validation_converter_empty.tsv
+sheets_and_friends/tsv_in/sheets-for-nmdc-submission-schema_validation_converter.tsv
 	$(RUN) modifications_and_validation \
 		--yaml_input $< \
 		--modifications_config_tsv $(word 2,$^) \

sheets_and_friends/tsv_in/sheets-for-nmdc-submission-schema_validation_converter.tsv

@@ -1,37 +1,18 @@
-from_val	from_type	to_type	to_val	notes	tested	;	|	PMID	DOI	URL	[	time	HH	date	YYYY	temporal	float	{{	-
-date	linkml range	DH datatype	xs:date			FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	TRUE	FALSE	TRUE	FALSE	FALSE	FALSE
-timestamp value	linkml range	DH datatype	xs:date	"^([\+-]?\d{4}(?!\d{2}\b))((-?)((0[1-9]|1[0-2])(\3([12]\d|0[1-9]|3[01]))?|W([0-4]\d|5[0-2])(-?[1-7])?|(00[1-9]|0[1-9]\d|[12]\d{2}|3([0-5]\d|6[1-6])))([T\s]((([01]\d|2[0-3])((:?)[0-5]\d)?|24\:?00)([\.,]\d+(?!:))?)?(\17[0-5]\d([\.,]\d+)?)?([zZ]|([\+-])([01]\d|2[0-3]):?([0-5]\d)?)?)?)?$"		FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	TRUE	FALSE	FALSE	FALSE	TRUE	FALSE	FALSE	FALSE
-{timestamp}	MIxS string serialization	DH datatype	xs:date	"^([\+-]?\d{4}(?!\d{2}\b))((-?)((0[1-9]|1[0-2])(\3([12]\d|0[1-9]|3[01]))?|W([0-4]\d|5[0-2])(-?[1-7])?|(00[1-9]|0[1-9]\d|[12]\d{2}|3([0-5]\d|6[1-6])))([T\s]((([01]\d|2[0-3])((:?)[0-5]\d)?|24\:?00)([\.,]\d+(?!:))?)?(\17[0-5]\d([\.,]\d+)?)?([zZ]|([\+-])([01]\d|2[0-3]):?([0-5]\d)?)?)?)?$"		FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	TRUE	FALSE	FALSE	FALSE	TRUE	FALSE	FALSE	FALSE
-YYYY-MM-DD	MIxS string serialization	DH datatype	xs:date	^\d{4}\-(0[1-9]|1[012])\-(0[1-9]|[12][0-9]|3[01])$		FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	TRUE	TRUE	FALSE	FALSE	TRUE
-double	linkml range	DH datatype	xs:decimal			FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE
-{float}	MIxS string serialization	DH datatype	xs:decimal	[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?		FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	TRUE	FALSE	FALSE
-{integer}	MIxS string serialization	DH datatype	xs:nonNegativeInteger	why non-negative?		FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE
-string	linkml range	DH datatype	xs:token			FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE
-{text}	MIxS string serialization	DH datatype	xs:token			FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE
-{text}:{text}	MIxS string serialization	DH pattern regex	[^\:\n\r]+\:[^\:\n\r]+			FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE
-{text < 20 characters}	NMDC string serialization	DH pattern regex	"^.{1,20}$"																
-"{time, seconds optional}"	NMDC string serialization	DH pattern regex	^([01]?\d|2[0-3]|24(?=:00?:00?$)):([0-5]\d)(:([0-5]\d))?$																
-HH:MM:SS	MIxS string serialization	DH pattern regex	^(2[0-3]|[01]?[0-9]):([0-5]?[0-9])$			FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	TRUE	FALSE	FALSE	TRUE	FALSE	FALSE	FALSE
-{lat lon}	NMDC string serialization	DH pattern regex	^[-+]?([1-8]?\d(\.\d+)?|90(\.0+)?)\s[-+]?(180(\.0+)?|((1[0-7]\d)|([1-9]?\d))(\.\d+)?)$																
-quantity value	linkml range	DH pattern regex	^[-+]?[0-9]*\.?[0-9]+ +\S.*$			FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE
-{float} {float}	MIxS string serialization	DH pattern regex	^[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)? [-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?$	not a good validation for lat lon / lat_lon	1.3 4.5	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	TRUE	FALSE	FALSE
-{float} {unit}	MIxS string serialization	DH pattern regex	^[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)? \S+$			FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	TRUE	FALSE	FALSE
-{float}|{float}-{float}	MIxS string serialization	DH pattern regex	^[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?$|^[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?-[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?$			FALSE	TRUE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	TRUE	FALSE	TRUE
-"{text}: {text}, {text}"	NMDC string serialization	DH pattern regex	"^[^:, ][^:]*: [^:, ][^,]*, [^:, ].*$"																
-"{date, arbitrary precision}"	NMDC string serialization	DH pattern regex	^[12]\d{3}(?:(?:-(?:0[1-9]|1[0-2]))(?:-(?:0[1-9]|[12]\d|3[01]))?)?$																
-{text};{float} {unit}	MIxS string serialization	DH pattern regex	^\S*;[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)? \S+$			TRUE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	TRUE	FALSE	FALSE
-{termLabel} {[termID]}; {timestamp}	MIxS string serialization	DH pattern regex	"^\S+.*\S+ \[ENVO:\d+\]; ([\+-]?\d{4}(?!\d{2}\b))((-?)((0[1-9]|1[0-2])(\3([12]\d|0[1-9]|3[01]))?|W([0-4]\d|5[0-2])(-?[1-7])?|(00[1-9]|0[1-9]\d|[12]\d{2}|3([0-5]\d|6[1-6])))([T\s]((([01]\d|2[0-3])((:?)[0-5]\d)?|24\:?00)([\.,]\d+(?!:))?)?(\17[0-5]\d([\.,]\d+)?)?([zZ]|([\+-])([01]\d|2[0-3]):?([0-5]\d)?)?)?)?$"		some words [ENVO:123]; 2022-01-01T10:10:10	TRUE	FALSE	FALSE	FALSE	FALSE	TRUE	TRUE	FALSE	FALSE	FALSE	TRUE	FALSE	FALSE	FALSE
-{termLabel} {[termID]};{timestamp}	MIxS string serialization	DH pattern regex	"^\S+.*\S+ \[ENVO:\d+\]; ([\+-]?\d{4}(?!\d{2}\b))((-?)((0[1-9]|1[0-2])(\3([12]\d|0[1-9]|3[01]))?|W([0-4]\d|5[0-2])(-?[1-7])?|(00[1-9]|0[1-9]\d|[12]\d{2}|3([0-5]\d|6[1-6])))([T\s]((([01]\d|2[0-3])((:?)[0-5]\d)?|24\:?00)([\.,]\d+(?!:))?)?(\17[0-5]\d([\.,]\d+)?)?([zZ]|([\+-])([01]\d|2[0-3]):?([0-5]\d)?)?)?)?$"		some words [ENVO:123]; 2022-01-01T10:10:10	TRUE	FALSE	FALSE	FALSE	FALSE	TRUE	TRUE	FALSE	FALSE	FALSE	TRUE	FALSE	FALSE	FALSE
-{termLabel} {[termID]}	MIxS string serialization	DH pattern regex	^\S+.*\S+ \[ENVO:\d+\]$	shouldn't really just be for ENVO		FALSE	FALSE	FALSE	FALSE	FALSE	TRUE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE
-{text};{timestamp}	MIxS string serialization	DH pattern regex	"^\S+.*\S+;([\+-]?\d{4}(?!\d{2}\b))((-?)((0[1-9]|1[0-2])(\3([12]\d|0[1-9]|3[01]))?|W([0-4]\d|5[0-2])(-?[1-7])?|(00[1-9]|0[1-9]\d|[12]\d{2}|3([0-5]\d|6[1-6])))([T\s]((([01]\d|2[0-3])((:?)[0-5]\d)?|24\:?00)([\.,]\d+(?!:))?)?(\17[0-5]\d([\.,]\d+)?)?([zZ]|([\+-])([01]\d|2[0-3]):?([0-5]\d)?)?)?)?$"		some words;2022-01-01T10:10:10	TRUE	FALSE	FALSE	FALSE	FALSE	FALSE	TRUE	FALSE	FALSE	FALSE	TRUE	FALSE	FALSE	FALSE
-{text};{float} {unit};{timestamp}	MIxS string serialization	DH pattern regex	"^\S+.*\S+;[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)? \S+;([\+-]?\d{4}(?!\d{2}\b))((-?)((0[1-9]|1[0-2])(\3([12]\d|0[1-9]|3[01]))?|W([0-4]\d|5[0-2])(-?[1-7])?|(00[1-9]|0[1-9]\d|[12]\d{2}|3([0-5]\d|6[1-6])))([T\s]((([01]\d|2[0-3])((:?)[0-5]\d)?|24\:?00)([\.,]\d+(?!:))?)?(\17[0-5]\d([\.,]\d+)?)?([zZ]|([\+-])([01]\d|2[0-3]):?([0-5]\d)?)?)?)?$"		some words;1.3 mg;2022-01	TRUE	FALSE	FALSE	FALSE	FALSE	FALSE	TRUE	FALSE	FALSE	FALSE	TRUE	TRUE	FALSE	FALSE
-{text};{float} {unit};{float} {unit}	MIxS string serialization	DH pattern regex	^\S+.*\S+;[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)? \S+;[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)? \S+$		some words;1.3 mg;1.3 mg	TRUE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	TRUE	FALSE	FALSE
-{{text}|{float} {unit}};{float} {unit}	MIxS string serialization					TRUE	TRUE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	TRUE	TRUE	FALSE
-{boolean};{Rn/start_time/end_time/duration}	MIxS string serialization					TRUE	FALSE	FALSE	FALSE	FALSE	FALSE	TRUE	FALSE	FALSE	FALSE	TRUE	FALSE	FALSE	FALSE
-{float} {unit};{Rn/start_time/end_time/duration}	MIxS string serialization					TRUE	FALSE	FALSE	FALSE	FALSE	FALSE	TRUE	FALSE	FALSE	FALSE	TRUE	TRUE	FALSE	FALSE
-{PMID}|{DOI}|{URL}	MIxS string serialization					FALSE	TRUE	TRUE	TRUE	TRUE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE
-{PMID}|{DOI}|{URL}|{text}	MIxS string serialization					FALSE	TRUE	TRUE	TRUE	TRUE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE
-"{term}: {term}, {text}"	MIxS string serialization					FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE
-{termLabel} {[termID]}|{text}	MIxS string serialization					FALSE	TRUE	FALSE	FALSE	FALSE	TRUE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE	FALSE
-{text};{float} {unit};{Rn/start_time/end_time/duration}	MIxS string serialization					TRUE	FALSE	FALSE	FALSE	FALSE	FALSE	TRUE	FALSE	FALSE	FALSE	TRUE	TRUE	FALSE	FALSE
-{text};{Rn/start_time/end_time/duration}	MIxS string serialization					TRUE	FALSE	FALSE	FALSE	FALSE	FALSE	TRUE	FALSE	FALSE	FALSE	TRUE	FALSE	FALSE	FALSE
+from_val	from_type	to_type	to_val	len
+{float}	MIxS string serialization	DH datatype	float	5
+{text}	MIxS string serialization	DH datatype	string	6
+{integer}	MIxS string serialization	DH datatype	integer	7
+{text < 20 characters}	NMDC string serialization	DH pattern regex	"^.{1,20}$"	9
+{text}:{text}	MIxS string serialization	DH pattern regex	[^\:\n\r]+\:[^\:\n\r]+	22
+"{text}: {text}, {text}"	NMDC string serialization	DH pattern regex	"^[^:, ][^:]*: [^:, ][^,]*, [^:, ].*$"	36
+{float} {unit}	MIxS string serialization	DH pattern regex	^[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)? \S+$	44
+"{time, seconds optional}"	NMDC string serialization	DH pattern regex	^([01]?\d|2[0-3]|24(?=:00?:00?$)):([0-5]\d)(:([0-5]\d))?$	57
+"{date, arbitrary precision}"	NMDC string serialization	DH pattern regex	^[12]\d{3}(?:(?:-(?:0[1-9]|1[0-2]))(?:-(?:0[1-9]|[12]\d|3[01]))?)?$	67
+{float} {float}	MIxS string serialization	DH pattern regex	^[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)? [-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?$	79
+{lat lon}	NMDC string serialization	DH pattern regex	^[-+]?([1-8]?\d(\.\d+)?|90(\.0+)?)\s[-+]?(180(\.0+)?|((1[0-7]\d)|([1-9]?\d))(\.\d+)?)$	86
+{text};{float} {unit};{float} {unit}	MIxS string serialization	DH pattern regex	^\S+.*\S+;[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)? \S+;[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)? \S+$	96
+{float}|{float}-{float}	MIxS string serialization	DH pattern regex	^([-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)? to )?[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?$	#REF!
+{text};{timestamp}	MIxS string serialization	DH pattern regex	"^\S+.*\S+;([\+-]?\d{4}(?!\d{2}\b))((-?)((0[1-9]|1[0-2])(\3([12]\d|0[1-9]|3[01]))?|W([0-4]\d|5[0-2])(-?[1-7])?|(00[1-9]|0[1-9]\d|[12]\d{2}|3([0-5]\d|6[1-6])))([T\s]((([01]\d|2[0-3])((:?)[0-5]\d)?|24\:?00)([\.,]\d+(?!:))?)?(\17[0-5]\d([\.,]\d+)?)?([zZ]|([\+-])([01]\d|2[0-3]):?([0-5]\d)?)?)?)?$"	292
+{termLabel} {[termID]}; {timestamp}	MIxS string serialization	DH pattern regex	"^\S+.*\S+ \[[a-zA-Z]{2,}:\d+\]; ([\+-]?\d{4}(?!\d{2}\b))((-?)((0[1-9]|1[0-2])(\3([12]\d|0[1-9]|3[01]))?|W([0-4]\d|5[0-2])(-?[1-7])?|(00[1-9]|0[1-9]\d|[12]\d{2}|3([0-5]\d|6[1-6])))([T\s]((([01]\d|2[0-3])((:?)[0-5]\d)?|24\:?00)([\.,]\d+(?!:))?)?(\17[0-5]\d([\.,]\d+)?)?([zZ]|([\+-])([01]\d|2[0-3]):?([0-5]\d)?)?)?)?$"	314
+{termLabel} {[termID]};{timestamp}	MIxS string serialization	DH pattern regex	"^\S+.*\S+ \[[a-zA-Z]{2,}:\d+\];([\+-]?\d{4}(?!\d{2}\b))((-?)((0[1-9]|1[0-2])(\3([12]\d|0[1-9]|3[01]))?|W([0-4]\d|5[0-2])(-?[1-7])?|(00[1-9]|0[1-9]\d|[12]\d{2}|3([0-5]\d|6[1-6])))([T\s]((([01]\d|2[0-3])((:?)[0-5]\d)?|24\:?00)([\.,]\d+(?!:))?)?(\17[0-5]\d([\.,]\d+)?)?([zZ]|([\+-])([01]\d|2[0-3]):?([0-5]\d)?)?)?)?$"	313
+{text};{float} {unit};{timestamp}	MIxS string serialization	DH pattern regex	.*	2

src/data/invalid/SampleData-water-data-alkalinity-list.yaml

@@ -6,6 +6,9 @@ water_data:
     env_local_scale: sand [ONTO:1234]
     env_medium: sand [ONTO:1234]
     env_package: xyz
+    samp_name: xyz
+    source_mat_id: x:1
     alkalinity:
       - 50 milligram per liter
       - 100 spoons per bucket
+

src/data/invalid/SampleData-water-data-broad-label-only.yaml

@@ -6,3 +6,5 @@ water_data:
     env_local_scale: sand [ONTO:1234]
     env_medium: sand [ONTO:1234]
     env_package: xyz
+    samp_name: xyz
+    source_mat_id: x:1

src/data/invalid/SampleData-water-data-broad-term-only.yaml

@@ -5,3 +5,5 @@ water_data:
     env_local_scale: [ONTO:1234]
     env_medium: sand [ONTO:1234]
     env_package: xyz
+    samp_name: xyz
+    source_mat_id: x:1

src/data/invalid/SampleData-water-data-depth-invalid-range.yaml

@@ -7,3 +7,5 @@ water_data:
     env_medium: sand [ONTO:1234]
     env_package: xyz
     depth: 1.5-2.5
+    samp_name: xyz
+    source_mat_id: x:1

src/data/invalid/SampleData-water-data-incomplete-organsim_count.yaml

@@ -7,3 +7,5 @@ water_data:
     env_medium: sand [ONTO:1234]
     env_package: xyz
     organism_count: xyz;100 units
+    samp_name: xyz
+    source_mat_id: x:1

src/data/invalid/SampleData-water-data-invalid-analysis-type.yaml

@@ -6,3 +6,5 @@ water_data:
     env_local_scale: sand [ONTO:1234]
     env_medium: sand [ONTO:1234]
     env_package: xyz
+    samp_name: xyz
+    source_mat_id: x:1

src/data/invalid/SampleData-water-data-invalid-rel_to_oxygen.yaml

@@ -7,3 +7,5 @@ water_data:
     env_medium: sand [ONTO:1234]
     env_package: xyz
     rel_to_oxygen: xyz
+    samp_name: xyz
+    source_mat_id: x:1