From 16c77c547387a75557f62a65669008360a4d8d3e Mon Sep 17 00:00:00 2001 From: Mark Andrew Miller Date: Fri, 29 Sep 2023 13:12:28 -0400 Subject: [PATCH] placeholder --- nmdc_schema/migration_recursion.py | 16 +++++++++--- project.Makefile | 41 ++++++++++++++++-------------- 2 files changed, 34 insertions(+), 23 deletions(-) diff --git a/nmdc_schema/migration_recursion.py b/nmdc_schema/migration_recursion.py index 27b71aac3a..28c7a243b1 100644 --- a/nmdc_schema/migration_recursion.py +++ b/nmdc_schema/migration_recursion.py @@ -14,6 +14,10 @@ doi_url_pattern = r'^https?:\/\/[a-zA-Z\.]+\/10\.' curie_pattern = r'^[a-zA-Z_][a-zA-Z0-9_-]*:[a-zA-Z0-9_][a-zA-Z0-9_.-]*$' +migration_plan = { + +} + class Migrator: def __init__(self): @@ -198,18 +202,22 @@ def main(schema_path, input_path, output_path, salvage_prefix): # logger.info(f"Starting {tdk}-specific migrations") # for current_study in tdv: # migrator.migrate_studies_7_7_2_to_7_8(current_study) + + if tdk == "biosample_set": + logger.info(f"Starting {tdk}-specific migrations") + for current_biosample in tdv: + migrator.migrate_uc_gold_biosample_identifiers_7_8_0_to_8_0_0(current_biosample) + if tdk == "extraction_set": logger.info(f"Starting {tdk}-specific migrations") for current_extraction in tdv: migrator.migrate_extractions_7_8_0_to_8_0_0(current_extraction) + if tdk == "omics_processing_set": logger.info(f"Starting {tdk}-specific migrations") for current_omics_processing in tdv: migrator.migrate_uc_gold_sequencing_project_identifiers_7_8_0_to_8_0_0(current_omics_processing) - if tdk == "biosample_set": - logger.info(f"Starting {tdk}-specific migrations") - for current_biosample in tdv: - migrator.migrate_uc_gold_biosample_identifiers_7_8_0_to_8_0_0(current_biosample) + if tdk == "study_set": logger.info(f"Starting {tdk}-specific migrations") for current_study in tdv: diff --git a/project.Makefile b/project.Makefile index afa639abcf..a504979951 100644 --- a/project.Makefile +++ b/project.Makefile @@ -421,33 +421,36 @@ make-rdf: rdf-clean local/mongo_as_nmdc_database_validation.log local/mongo_as_n temp: -# --selected-collections functional_annotation_agg \ # huge, no publically avaiaible reference data (kegg) +# --selected-collections functional_annotation_agg \ # huge, no publicly available reference data (kegg) # --selected-collections metaproteomics_analysis_activity_set \ # next slowest +# when connecting to the dev MongoDB, also use --client-base-url https://api-dev.microbiomedata.org +# make pre-composed prod and dev makefile tasks! since they require the user to provide a matching mongo port and api url + local/mongo_as_unvalidated_nmdc_database.yaml: date # 276.50 seconds on 2023-08-30 without functional_annotation_agg or metaproteomics_analysis_activity_set time $(RUN) pure-export \ --max-docs-per-coll 10000000 \ --output-yaml $@ \ --page-size 10000 \ - --selected-collections biosample_set \ - --selected-collections data_object_set \ - --selected-collections extraction_set \ - --selected-collections field_research_site_set \ - --selected-collections library_preparation_set \ - --selected-collections mags_activity_set \ - --selected-collections metabolomics_analysis_activity_set \ - --selected-collections metagenome_annotation_activity_set \ - --selected-collections metagenome_assembly_set \ - --selected-collections metagenome_sequencing_activity_set \ - --selected-collections metatranscriptome_activity_set \ - --selected-collections nom_analysis_activity_set \ - --selected-collections omics_processing_set \ - --selected-collections pooling_set \ - --selected-collections processed_sample_set \ - --selected-collections read_based_taxonomy_analysis_activity_set \ - --selected-collections read_qc_analysis_activity_set \ - --selected-collections study_set + --selected-collections biosample_set \ + --selected-collections data_object_set \ + --selected-collections extraction_set \ + --selected-collections field_research_site_set \ + --selected-collections library_preparation_set \ + --selected-collections mags_activity_set \ + --selected-collections metabolomics_analysis_activity_set \ + --selected-collections metagenome_annotation_activity_set \ + --selected-collections metagenome_assembly_set \ + --selected-collections metagenome_sequencing_activity_set \ + --selected-collections metatranscriptome_activity_set \ + --selected-collections nom_analysis_activity_set \ + --selected-collections omics_processing_set \ + --selected-collections pooling_set \ + --selected-collections processed_sample_set \ + --selected-collections read_based_taxonomy_analysis_activity_set \ + --selected-collections read_qc_analysis_activity_set \ + --selected-collections study_set local/mongo_as_nmdc_database_rdf_safe.yaml: nmdc_schema/nmdc_schema_accepting_legacy_ids.yaml local/mongo_as_unvalidated_nmdc_database.yaml