From 0adaec55806d8208127712e1613a098ea43f8ac3 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Wed, 16 Aug 2023 15:59:25 -0700 Subject: [PATCH 01/30] Allow putting a few new keys into a different map. --- otc/ctrie/ctrie_db.cpp | 84 ++++++++++++++++++++++++++++++++++++++++++ otc/ctrie/ctrie_db.h | 13 ++++++- 2 files changed, 95 insertions(+), 2 deletions(-) diff --git a/otc/ctrie/ctrie_db.cpp b/otc/ctrie/ctrie_db.cpp index 888b7f6a..8d1db2b4 100644 --- a/otc/ctrie/ctrie_db.cpp +++ b/otc/ctrie/ctrie_db.cpp @@ -27,6 +27,10 @@ std::set CompressedTrieBasedDB::fuzzy_ auto from_full = wide_trie.fuzzy_matches(conv_query, max_dist); sorted.insert(std::begin(from_full), std::end(from_full)); + + auto from_new = new_trie->fuzzy_matches(conv_query, max_dist); + sorted.insert(std::begin(from_new), std::end(from_new)); + return sorted; } @@ -42,6 +46,9 @@ std::set CompressedTrieBasedDB::exact_ auto from_full = wide_trie.fuzzy_matches(conv_query, 0); sorted.insert(std::begin(from_full), std::end(from_full)); + auto from_new = new_trie->fuzzy_matches(conv_query, 0); + sorted.insert(std::begin(from_new), std::end(from_new)); + return sorted; } @@ -54,12 +61,86 @@ vector CompressedTrieBasedDB::prefix_query(const std::string & query_str auto from_full = wide_trie.prefix_query(conv_query); sorted.insert(sorted.end(), std::begin(from_full), std::end(from_full)); + auto from_new = new_trie->prefix_query(conv_query); + sorted.insert(sorted.end(), std::begin(from_new), std::end(from_new)); + // I'm not sure this is a good idea... std::sort(sorted.begin(), sorted.end()); return sorted; } +void CompressedTrieBasedDB::add_key(const std::string& s) +{ + new_keys.insert(s); + rebuild_new_trie(); +} + +void CompressedTrieBasedDB::rebuild_new_trie() +{ + LOG(INFO)<<"Rebuilding ctree for additions"; + LOG(INFO)<< new_keys.size() << " keys\n"; + + ctrie_init_set_t for_new; + std::map letter_counts; + std::set new_letter_set; + // unsigned mem_str = 0; + for (auto i : new_keys) + { + // mem_str += i.length(); + auto widestr = to_u32string(i); + + for_new.insert(widestr); + + for (auto letter : widestr) { + new_letter_set.insert(letter); + } + //LOG(INFO) << glob_conv8.to_bytes(widestr) << '\n'; + } + LOG(INFO)<> by_count; + + LOG(INFO)<<"new letters: "<(); + new_trie->init(for_new, new_letters); +} + void CompressedTrieBasedDB::initialize(const std::set & keys) { ctrie_init_set_t for_wide; ctrie_init_set_t for_thin; @@ -138,6 +219,9 @@ void CompressedTrieBasedDB::initialize(const std::set & keys) { wide_trie.init(for_wide, wide_letters); thin_trie.init(for_thin, thin_letters); + new_trie = std::make_shared(); + new_trie->init({},thin_letters); + /* wide_trie.db_write_words(std::cerr); thin_trie.db_write_words(std::cerr); diff --git a/otc/ctrie/ctrie_db.h b/otc/ctrie/ctrie_db.h index c828008d..364296fc 100644 --- a/otc/ctrie/ctrie_db.h +++ b/otc/ctrie/ctrie_db.h @@ -3,18 +3,27 @@ #include "otc/ctrie/ctrie.h" +#include namespace otc { class CompressedTrieBasedDB { - public: +public: void initialize(const std::set & keys); std::set fuzzy_query(const std::string & query_str) const; std::set exact_query(const std::string & query_str) const; std::vector prefix_query(const std::string & query_str) const; - private: + + void add_key(const std::string& s); + + void rebuild_new_trie(); + +private: CompressedTrie wide_trie; CompressedTrie thin_trie; + + std::shared_ptr new_trie; + std::set new_keys; }; From 1124bdd707d5ec7be77d1fdd19a48619e450f35d Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 29 Aug 2023 15:01:05 -0400 Subject: [PATCH 02/30] Make TreeToServe hold a PatchableTaxonomy. --- otc/ws/trees_to_serve.cpp | 8 ++++---- otc/ws/trees_to_serve.h | 10 ++++++---- ws/tolwsbooting.cpp | 3 ++- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/otc/ws/trees_to_serve.cpp b/otc/ws/trees_to_serve.cpp index 8b575346..4844f429 100644 --- a/otc/ws/trees_to_serve.cpp +++ b/otc/ws/trees_to_serve.cpp @@ -43,20 +43,20 @@ const string * TreesToServe::get_stored_string(const string & k) { return v; } -void TreesToServe::set_taxonomy(RichTaxonomy &taxonomy) { +void TreesToServe::set_taxonomy(PatchableTaxonomy &taxonomy) { assert(taxonomy_ptr == nullptr); taxonomy_ptr = &taxonomy; taxonomy_tree = &(taxonomy.get_tax_tree()); } -ReadableTaxonomy TreesToServe::get_readable_taxonomy() const { +TreesToServe::ReadableTaxonomy TreesToServe::get_readable_taxonomy() const { assert(taxonomy_ptr != nullptr); return {*taxonomy_ptr, std::make_unique(taxonomy_thread_safety)}; } -WritableTaxonomy TreesToServe::get_writable_taxonomy() { +TreesToServe::WritableTaxonomy TreesToServe::get_writable_taxonomy() { assert(taxonomy_ptr != nullptr); return {*taxonomy_ptr, std::make_unique(taxonomy_thread_safety)}; @@ -197,7 +197,7 @@ void TreesToServe::final_tree_added() { const auto & sft = annot.suppressed_from_tree; assert(taxonomy_ptr != nullptr); // TODO: make taxonomy_ptr non-const or make the relevant field mutable. Probably the former. - auto nct = const_cast(taxonomy_ptr); + auto nct = const_cast(taxonomy_ptr); nct->set_ids_suppressed_from_summary_tree_alias(&sft); } diff --git a/otc/ws/trees_to_serve.h b/otc/ws/trees_to_serve.h index 36e4b339..bb21b20c 100644 --- a/otc/ws/trees_to_serve.h +++ b/otc/ws/trees_to_serve.h @@ -4,6 +4,8 @@ #include #include #include "otc/ws/tolws.h" +#include "otc/taxonomy/patching.h" + namespace otc { using ReadableTaxonomy = std::pair id_to_tree; std::map id_to_annotations; std::string default_synth_id; - RichTaxonomy * taxonomy_ptr = nullptr; + PatchableTaxonomy * taxonomy_ptr = nullptr; std::map stored_strings; std::list stored_strings_list; const RichTaxTree * taxonomy_tree = nullptr; @@ -35,10 +37,10 @@ class TreesToServe { const std::string * get_stored_string(const std::string & k); - void set_taxonomy(RichTaxonomy &taxonomy); + void set_taxonomy(PatchableTaxonomy &taxonomy); - using ReadableTaxonomy = std::pair >; - using WritableTaxonomy = std::pair >; + using ReadableTaxonomy = std::pair >; + using WritableTaxonomy = std::pair >; ReadableTaxonomy get_readable_taxonomy() const; WritableTaxonomy get_writable_taxonomy(); diff --git a/ws/tolwsbooting.cpp b/ws/tolwsbooting.cpp index 3ef5e0c0..2df285d4 100644 --- a/ws/tolwsbooting.cpp +++ b/ws/tolwsbooting.cpp @@ -16,6 +16,7 @@ #include "otc/ctrie/context_ctrie_db.h" #include "otc/tnrs/context.h" #include "otc/supertree_util.h" +#include "otc/taxonomy/patching.h" #include "config.h" #ifdef HAVE_SYS_RESOURCE_H #include @@ -831,7 +832,7 @@ int run_server(const po::variables_map & args) { // Must load taxonomy before trees LOG(INFO) << "reading taxonomy..."; - RichTaxonomy taxonomy = load_rich_taxonomy(args); + PatchableTaxonomy taxonomy = load_patchable_taxonomy(args); auto nc = Context::cull_contexts_to_taxonomy(taxonomy); LOG(INFO) << nc << " taxonomy contexts retained..."; From 48c22eb8f9332c155ac685eca5e77c8d4252db06 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 5 Sep 2023 12:50:53 -0400 Subject: [PATCH 03/30] Add simple taxon addition route. --- otc/ws/taxonomyws.cpp | 18 ++++++++++++++++++ otc/ws/tolws.h | 8 ++++++++ ws/tolwsbooting.cpp | 17 +++++++++++++++++ 3 files changed, 43 insertions(+) diff --git a/otc/ws/taxonomyws.cpp b/otc/ws/taxonomyws.cpp index 5d946fda..95a5d2e8 100644 --- a/otc/ws/taxonomyws.cpp +++ b/otc/ws/taxonomyws.cpp @@ -242,4 +242,22 @@ string taxon_subtree_ws_method(const TreesToServe & tts, return response.dump(1); } +std::string taxon_addition_ws_method(const TreesToServe & tts, + PatchableTaxonomy & taxonomy, + OttId ott_id, + OttId parent_id, + const string& name, + const string& rank) +{ + const auto & taxonomy_tree = taxonomy.get_tax_tree(); + + auto parent_node = taxonomy.included_taxon_from_id(parent_id); + if (parent_node == nullptr) { + throw OTCBadRequest() << "Unrecognized OTT ID: " << parent_id; + } + json response; + taxonomy.add_new_taxon(ott_id, parent_id, name, rank, "", "", {}); + return response.dump(1); +} + } // namespace otc diff --git a/otc/ws/tolws.h b/otc/ws/tolws.h index caf67c0d..74e95c78 100644 --- a/otc/ws/tolws.h +++ b/otc/ws/tolws.h @@ -17,6 +17,7 @@ #include "otc/tree.h" #include "otc/error.h" #include "otc/taxonomy/taxonomy.h" +#include "otc/taxonomy/patching.h" #include "otc/taxonomy/flags.h" #include "otc/ws/parallelreadserialwrite.h" #include "otc/ws/otc_web_error.h" @@ -313,6 +314,13 @@ std::string taxon_subtree_ws_method(const TreesToServe & tts, const RTRichTaxNode * taxon_node, NodeNameStyle label_format); +std::string taxon_addition_ws_method(const TreesToServe & tts, + PatchableTaxonomy & taxonomy, + OttId ottid, + OttId parent_id, + const std::string& name, + const std::string& rank); + std::string tnrs_match_names_ws_method(const std::vector& names, const std::optional& context_name, bool do_approximate_matching, diff --git a/ws/tolwsbooting.cpp b/ws/tolwsbooting.cpp index 2df285d4..359eae64 100644 --- a/ws/tolwsbooting.cpp +++ b/ws/tolwsbooting.cpp @@ -456,6 +456,19 @@ string taxon_subtree_method_handler( const json& parsedargs ) { return taxon_subtree_ws_method(tts, taxonomy, taxon_node, nns); } +string taxon_addition_method_handler( const json& parsedargs ) +{ + // Actually, I think the amendments handle multiple operations. + // But this just handles one addition. + + auto [locked_taxonomy,lock] = tts.get_writable_taxonomy(); + auto name = extract_required_argument(parsedargs, "name"); + auto ott_id = extract_required_argument(parsedargs, "ott_id"); + auto parent_id = extract_required_argument(parsedargs, "parent"); + auto rank = extract_required_argument(parsedargs, "rank"); + return taxon_addition_ws_method(tts, locked_taxonomy, ott_id, parent_id, name, rank); +} + // See taxomachine/src/main/java/org/opentree/taxonomy/plugins/tnrs_v3.java // 10,000 queries at .0016 second per query = 16 seconds @@ -874,6 +887,8 @@ int run_server(const po::variables_map & args) { auto v3_r_taxon_mrca = path_handler(v3_prefix + "/taxonomy/mrca", taxon_mrca_method_handler ); auto v3_r_taxon_subtree = path_handler(v3_prefix + "/taxonomy/subtree", taxon_subtree_method_handler ); + auto v3_r_taxon_addition = path_handler(v3_prefix + "/taxonomy/process_additions", taxon_addition_method_handler ); + // tnrs auto v3_r_tnrs_match_names = path_handler(v3_prefix + "/tnrs/match_names", tnrs_match_names_handler ); auto v3_r_tnrs_autocomplete_name = path_handler(v3_prefix + "/tnrs/autocomplete_name", tnrs_autocomplete_name_handler ); @@ -909,6 +924,8 @@ int run_server(const po::variables_map & args) { auto v4_r_taxon_mrca = path_handler(v4_prefix + "/taxonomy/mrca", taxon_mrca_method_handler ); auto v4_r_taxon_subtree = path_handler(v4_prefix + "/taxonomy/subtree", taxon_subtree_method_handler ); + auto v4_r_taxon_addition = path_handler(v4_prefix + "/taxonomy/process_additions", taxon_addition_method_handler ); + // tnrs auto v4_r_tnrs_match_names = path_handler(v4_prefix + "/tnrs/match_names", tnrs_match_names_handler ); auto v4_r_tnrs_autocomplete_name = path_handler(v4_prefix + "/tnrs/autocomplete_name", tnrs_autocomplete_name_handler ); From d7cc25b05b672358df90e68e000b9ab2720da471 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 5 Sep 2023 13:19:29 -0400 Subject: [PATCH 04/30] Publish the routes. --- ws/tolwsbooting.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ws/tolwsbooting.cpp b/ws/tolwsbooting.cpp index 359eae64..5a9c1dee 100644 --- a/ws/tolwsbooting.cpp +++ b/ws/tolwsbooting.cpp @@ -954,6 +954,7 @@ int run_server(const po::variables_map & args) { service.publish( v3_r_taxon_flags ); service.publish( v3_r_taxon_mrca ); service.publish( v3_r_taxon_subtree ); + service.publish( v3_r_taxon_addition ); service.publish( v3_r_tnrs_match_names ); service.publish( v3_r_tnrs_autocomplete_name ); service.publish( v3_r_tnrs_contexts ); @@ -972,6 +973,7 @@ int run_server(const po::variables_map & args) { service.publish( v4_r_taxon_flags ); service.publish( v4_r_taxon_mrca ); service.publish( v4_r_taxon_subtree ); + service.publish( v4_r_taxon_addition ); service.publish( v4_r_tnrs_match_names ); service.publish( v4_r_tnrs_autocomplete_name ); service.publish( v4_r_tnrs_contexts ); From 14bd1576068b84f0522804dbb2d89cf6ecfddf0f Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 5 Sep 2023 13:29:10 -0400 Subject: [PATCH 05/30] Add basic error reporting. --- otc/ws/taxonomyws.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/otc/ws/taxonomyws.cpp b/otc/ws/taxonomyws.cpp index 95a5d2e8..6ddfb72c 100644 --- a/otc/ws/taxonomyws.cpp +++ b/otc/ws/taxonomyws.cpp @@ -256,7 +256,10 @@ std::string taxon_addition_ws_method(const TreesToServe & tts, throw OTCBadRequest() << "Unrecognized OTT ID: " << parent_id; } json response; - taxonomy.add_new_taxon(ott_id, parent_id, name, rank, "", "", {}); + auto [ok,error] = taxonomy.add_new_taxon(ott_id, parent_id, name, rank, "", "", {}); + if (not ok) + throw OTCBadRequest() << error; + response["ok"] = true; return response.dump(1); } From eea7bef494ce767cd1aa1760cd9f108daf5f18e6 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 5 Sep 2023 15:16:44 -0400 Subject: [PATCH 06/30] Add comments. --- otc/taxonomy/patching.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/otc/taxonomy/patching.cpp b/otc/taxonomy/patching.cpp index 315c3e3c..6321f2e1 100644 --- a/otc/taxonomy/patching.cpp +++ b/otc/taxonomy/patching.cpp @@ -288,6 +288,8 @@ bool_str_t PatchableTaxonomy::add_new_taxon(OttId oid, OttId * homonym_of) { auto & tree = this->get_mutable_tax_tree(); auto & rt_data = tree.get_data(); + + // 1. Check if the new taxon is a homonym of an existing taxon. auto nm_nd_it = rt_data.name_to_node.find(name); if (nm_nd_it == rt_data.name_to_node.end()) { if (homonym_of != nullptr) { @@ -298,6 +300,8 @@ bool_str_t PatchableTaxonomy::add_new_taxon(OttId oid, expl += " is a homonym of " + std::to_string(nm_nd_it->second->get_ott_id()); return bool_str_t{false, expl}; } + + // 2. Check if the OTT ID is already used. auto itnit = included_taxon_from_id(oid); auto itrit = rt_data.id_to_record.find(oid); if (itnit != nullptr || itrit != rt_data.id_to_record.end()) { @@ -305,6 +309,8 @@ bool_str_t PatchableTaxonomy::add_new_taxon(OttId oid, expl += " is already used."; return bool_str_t{false, expl}; } + + // 3. Find the parent taxon by its OTT ID. RTRichTaxNode * par_ptr = const_cast(included_taxon_from_id(parent_id)); if (par_ptr == nullptr) { itrit = rt_data.id_to_record.find(parent_id); @@ -316,13 +322,18 @@ bool_str_t PatchableTaxonomy::add_new_taxon(OttId oid, } return bool_str_t{false, expl}; } + + // 4. Complain if the new taxon has a uniqname if (uniqname.length() > 0) { return bool_str_t{false, "handling of uniqname not supported"}; } + // 5. Create the new taxon record. const auto & tr = get_new_tax_rec(oid, parent_id, name, rank, sourceinfo, uniqname, flags); + + // 6. Create the new tree node auto nnd = tree.create_child(par_ptr); reg_or_rereg_nd(nnd, tr, tree); return bool_str_t{true, ""}; From 1489dd2c96312e070fa84e826f989294fad15a31 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 5 Sep 2023 15:16:58 -0400 Subject: [PATCH 07/30] Remove warnings. --- otc/taxonomy/patching.cpp | 6 +++--- tools/source-taxonomy-patcher.cpp | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/otc/taxonomy/patching.cpp b/otc/taxonomy/patching.cpp index 6321f2e1..081e0840 100644 --- a/otc/taxonomy/patching.cpp +++ b/otc/taxonomy/patching.cpp @@ -210,13 +210,13 @@ bool_str_t PatchableTaxonomy::add_forward(OttId former_id, OttId redirect_to_id) return bool_str_t{true, ""}; } -bool_str_t PatchableTaxonomy::delete_forward(OttId former_id, OttId redirect_to_id) { +bool_str_t PatchableTaxonomy::delete_forward(OttId /* former_id */, OttId /*redirect_to_id*/) { auto & tree = this->get_mutable_tax_tree(); auto & rt_data = tree.get_data(); throw OTCError() << "delete_forward not implemented"; } -bool_str_t PatchableTaxonomy::delete_taxon(OttId ott_id) { +bool_str_t PatchableTaxonomy::delete_taxon(OttId /* ott_id */) { auto & tree = this->get_mutable_tax_tree(); auto & rt_data = tree.get_data(); throw OTCError() << "delete_taxon not implemented"; @@ -230,7 +230,7 @@ bool_str_t PatchableTaxonomy::edit_taxon(OttId oid, const std::string & uniqname, const std::string & flags, bool flags_edited, - OttId * homonym_of) { + OttId * /* homonym_of */) { auto & tree = this->get_mutable_tax_tree(); auto & rt_data = tree.get_data(); auto nd_ptr = const_cast(included_taxon_from_id(oid)); diff --git a/tools/source-taxonomy-patcher.cpp b/tools/source-taxonomy-patcher.cpp index 63e09d84..545236d6 100644 --- a/tools/source-taxonomy-patcher.cpp +++ b/tools/source-taxonomy-patcher.cpp @@ -115,7 +115,7 @@ std::pair get_array_property(const json & j, return std::pair(false, nullptr); } -const unsigned parse_as_unsigned(const json & j) { +unsigned parse_as_unsigned(const json & j) { if (!j.is_number_unsigned()) { throw OTCError() << "Expecting a non negative integer"; } From f09902ebb066e04251c0af541e0a772560c6c220 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Tue, 5 Sep 2023 16:20:24 -0400 Subject: [PATCH 08/30] Modify the CTrie when we add a name. --- otc/ctrie/context_ctrie_db.cpp | 17 +++++++++++++++++ otc/ctrie/context_ctrie_db.h | 4 +++- otc/taxonomy/patching.cpp | 10 ++++++++++ otc/taxonomy/taxonomy.h | 10 +++++++--- 4 files changed, 37 insertions(+), 4 deletions(-) diff --git a/otc/ctrie/context_ctrie_db.cpp b/otc/ctrie/context_ctrie_db.cpp index 43df3418..ac5f8443 100644 --- a/otc/ctrie/context_ctrie_db.cpp +++ b/otc/ctrie/context_ctrie_db.cpp @@ -272,4 +272,21 @@ vec_fqr_w_t ContextAwareCTrieBasedDB::fuzzy_query_to_taxa(const std::string & qu return to_taxa(fuzzy_query(query_str), context_root, taxonomy, include_suppressed); } +// how do trav_enter and such things work? +// what are they used for? + +void ContextAwareCTrieBasedDB::add_key(const std::string& s, OttId id, const RichTaxonomy& taxonomy) +{ + // how are names split between this context and its children? + + auto node = taxonomy.included_taxon_from_id(id); + if (not node) + throw OTCError()<<"add_key: id "<add_key(s, id, taxonomy); +} + } // namespace otc diff --git a/otc/ctrie/context_ctrie_db.h b/otc/ctrie/context_ctrie_db.h index b1293329..96cca302 100644 --- a/otc/ctrie/context_ctrie_db.h +++ b/otc/ctrie/context_ctrie_db.h @@ -45,9 +45,11 @@ class ContextAwareCTrieBasedDB { const RichTaxonomy & taxonomy, bool include_suppressed) const; + void add_key(const std::string& s, OttId id, const RichTaxonomy&); + private: const Context & context; - std::vector children; + std::vector children; CompressedTrieBasedDB trie; std::uint32_t filter_trav_enter = 0; std::uint32_t trav_exit = UINT32_MAX; diff --git a/otc/taxonomy/patching.cpp b/otc/taxonomy/patching.cpp index 081e0840..211e3850 100644 --- a/otc/taxonomy/patching.cpp +++ b/otc/taxonomy/patching.cpp @@ -3,6 +3,7 @@ namespace fs = std::filesystem; #include "otc/taxonomy/patching.h" #include "otc/otc_base_includes.h" +#include "otc/ctrie/context_ctrie_db.h" using namespace otc; @@ -336,6 +337,15 @@ bool_str_t PatchableTaxonomy::add_new_taxon(OttId oid, // 6. Create the new tree node auto nnd = tree.create_child(par_ptr); reg_or_rereg_nd(nnd, tr, tree); + + // 7. Update the fuzzy match databases + if (auto f = get_fuzzy_matcher()) + f->add_key(name, oid, *this); + + // TODO: fix trav_enter, etc. + // TODO: fix allocation of names between contexts + // TODO: read amendment blob + return bool_str_t{true, ""}; } diff --git a/otc/taxonomy/taxonomy.h b/otc/taxonomy/taxonomy.h index 68180cd2..0757f7e4 100644 --- a/otc/taxonomy/taxonomy.h +++ b/otc/taxonomy/taxonomy.h @@ -405,12 +405,16 @@ class RichTaxonomy: public BaseTaxonomy { const std::list & get_synonyms_list() const { return synonyms; } - + const ContextAwareCTrieBasedDB * get_fuzzy_matcher() const { return fuzzy_match_db; } - void set_fuzzy_matcher(const ContextAwareCTrieBasedDB * match_db) { + ContextAwareCTrieBasedDB * get_fuzzy_matcher() { + return fuzzy_match_db; + } + + void set_fuzzy_matcher(ContextAwareCTrieBasedDB * match_db) { fuzzy_match_db = match_db; } @@ -433,7 +437,7 @@ class RichTaxonomy: public BaseTaxonomy { // can pass a non-nullptr pointer in using the setter. This // will allow the services to report "is_suppressed_from_synth" option. const OttIdSet * is_suppressed_from_synth = nullptr; - const ContextAwareCTrieBasedDB * fuzzy_match_db = nullptr; + ContextAwareCTrieBasedDB * fuzzy_match_db = nullptr; RichTaxonomy(const RichTaxonomy &) = delete; private: void read_input_synonyms_stream(std::istream & synonyms_file); From 4a3121668e765c4fdf546f6fd2cbf9f2fea29c55 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Wed, 13 Sep 2023 15:09:32 -0400 Subject: [PATCH 09/30] Remove unused data member ContextAwareCTrieBasedDB::children --- otc/ctrie/context_ctrie_db.cpp | 11 ----------- otc/ctrie/context_ctrie_db.h | 1 - otc/tnrs/context.cpp | 10 +++++----- 3 files changed, 5 insertions(+), 17 deletions(-) diff --git a/otc/ctrie/context_ctrie_db.cpp b/otc/ctrie/context_ctrie_db.cpp index ac5f8443..1cd00e08 100644 --- a/otc/ctrie/context_ctrie_db.cpp +++ b/otc/ctrie/context_ctrie_db.cpp @@ -83,12 +83,6 @@ std::set ContextAwareCTrieBasedDB::fuz if (context.name_matcher != nullptr) { sorted = context.name_matcher->fuzzy_query(query_str); } - for (auto c :children) { - if (c->context.name_matcher) { - auto csorted = c->context.name_matcher->fuzzy_query(query_str); - sorted.insert(std::begin(csorted), std::end(csorted)); - } - } return sorted; } @@ -277,16 +271,11 @@ vec_fqr_w_t ContextAwareCTrieBasedDB::fuzzy_query_to_taxa(const std::string & qu void ContextAwareCTrieBasedDB::add_key(const std::string& s, OttId id, const RichTaxonomy& taxonomy) { - // how are names split between this context and its children? - auto node = taxonomy.included_taxon_from_id(id); if (not node) throw OTCError()<<"add_key: id "<add_key(s, id, taxonomy); } } // namespace otc diff --git a/otc/ctrie/context_ctrie_db.h b/otc/ctrie/context_ctrie_db.h index 96cca302..2f15b48f 100644 --- a/otc/ctrie/context_ctrie_db.h +++ b/otc/ctrie/context_ctrie_db.h @@ -49,7 +49,6 @@ class ContextAwareCTrieBasedDB { private: const Context & context; - std::vector children; CompressedTrieBasedDB trie; std::uint32_t filter_trav_enter = 0; std::uint32_t trav_exit = UINT32_MAX; diff --git a/otc/tnrs/context.cpp b/otc/tnrs/context.cpp index f270483b..aacbf238 100644 --- a/otc/tnrs/context.cpp +++ b/otc/tnrs/context.cpp @@ -177,15 +177,15 @@ std::vector _fill_and_sort_ranges(const RichTaxonomy & taxonomy, c const auto & tree = taxonomy.get_tax_tree(); const auto & tax_tree_data = tree.get_data(); std::vector ret; - for (auto & p : vec_ids_names) { + for (auto & [id,name] : vec_ids_names) { const RTRichTaxNode * nd; try { - nd = tax_tree_data.id_to_node.at(p.first); + nd = tax_tree_data.id_to_node.at(id); } catch (...) { - throw OTCError() << "Barrier taxon with id=" << p.first << " not found"; + throw OTCError() << "Barrier taxon with id=" << id << " not found"; } - if (nd->get_name() != p.second) { - throw OTCError() << "Barrier taxon with id=" << p.first << " had name \"" << nd->get_name() << "\", but we expected \"" << p.second << "\"."; + if (nd->get_name() != name) { + throw OTCError() << "Barrier taxon with id=" << id << " had name \"" << nd->get_name() << "\", but we expected \"" << name << "\"."; } const auto & ndd = nd->get_data(); ret.push_back(trav_range_t{ndd.trav_enter, ndd.trav_exit}); From 1296ab932b46d60bd1d2b3fb329205d3f5af711d Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Wed, 13 Sep 2023 15:13:35 -0400 Subject: [PATCH 10/30] Use structured binding. --- otc/ctrie/context_ctrie_db.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/otc/ctrie/context_ctrie_db.cpp b/otc/ctrie/context_ctrie_db.cpp index 1cd00e08..ead19cb2 100644 --- a/otc/ctrie/context_ctrie_db.cpp +++ b/otc/ctrie/context_ctrie_db.cpp @@ -121,19 +121,19 @@ vec_fqr_w_t ContextAwareCTrieBasedDB::to_taxa(const set vec size = " << vec_taxon_and_syn_ptrs.size(); - for (auto & tax_and_syn_pair : vec_taxon_and_syn_ptrs) { - auto tax_ptr = tax_and_syn_pair.first; + for (auto & [tax_ptr, tax_thing] : vec_taxon_and_syn_ptrs) + { if (tax_ptr == nullptr) { LOG(DEBUG) << "matched suppressed and include_suppressed = " << include_suppressed; if (include_suppressed) { - const TaxonomyRecord * tr = (const TaxonomyRecord *)(tax_and_syn_pair.second); + const TaxonomyRecord * tr = (const TaxonomyRecord *)(tax_thing); results.push_back(FuzzyQueryResultWithTaxon(fqr, tr)); } } else { const auto & res_tax_data = tax_ptr->get_data(); LOG(DEBUG) << "matched taxon trav = (" << res_tax_data.trav_enter << ", " << res_tax_data.trav_exit << "). filter.trav = (" << filter_trav_enter << ", " << filter_trav_exit << ")"; if (res_tax_data.trav_exit <= filter_trav_exit && res_tax_data.trav_enter >= filter_trav_enter) { - const TaxonomicJuniorSynonym * syn_ptr = (const TaxonomicJuniorSynonym *)(tax_and_syn_pair.second); + const TaxonomicJuniorSynonym * syn_ptr = (const TaxonomicJuniorSynonym *)(tax_thing); if (syn_ptr == nullptr) { LOG(DEBUG) << "pushing non-syn"; results.push_back(FuzzyQueryResultWithTaxon(fqr, tax_ptr)); From 447ed1eb896443981176e65fd85ebe59e8d799a6 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Wed, 13 Sep 2023 15:57:35 -0400 Subject: [PATCH 11/30] Move some of the depth-based operations to otc/tree_operations --- otc/induced_tree.h | 41 ------------------------------------- otc/tree_operations.h | 47 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 41 deletions(-) diff --git a/otc/induced_tree.h b/otc/induced_tree.h index 553259e4..d4b02575 100644 --- a/otc/induced_tree.h +++ b/otc/induced_tree.h @@ -8,18 +8,6 @@ #include "otc/otc_base_includes.h" namespace otc { -template -inline int depth(const N* node) { - assert(node->get_data().depth > 0); - return node->get_data().depth; -} - - -template -inline int& depth(N* node) { - assert(node->get_data().depth > 0); - return node->get_data().depth; -} template node_t* trace_to_parent(node_t* node, std::unordered_set& nodes) { @@ -29,35 +17,6 @@ node_t* trace_to_parent(node_t* node, std::unordered_set& nodes) { return node; } -template -N* mrca_from_depth(N* node1, N* node2) { - assert(node1 or node2); - if (not node1) { - return node2; - } - if (not node2) { - return node1; - } - assert(node1 and node2); - assert(get_root(node1) == get_root(node2)); - while (depth(node1) > depth(node2)) { - node1 = node1->get_parent(); - } - while (depth(node1) < depth(node2)) { - node2 = node2->get_parent(); - } - assert(depth(node1) == depth(node2)); - while (node1 != node2) { - assert(node1->get_parent()); - assert(node2->get_parent()); - node1 = node1->get_parent(); - node2 = node2->get_parent(); - } - assert(node1 == node2); - return node1; -} - - template N* MRCA_of_group(const std::vector& leaves, const F& MRCA_of_pair) { diff --git a/otc/tree_operations.h b/otc/tree_operations.h index 710ab261..2bdef996 100644 --- a/otc/tree_operations.h +++ b/otc/tree_operations.h @@ -321,6 +321,53 @@ void compute_depth(T& tree) { } } +template +inline int depth(const N* node) { + assert(node->get_data().depth > 0); + return node->get_data().depth; +} + + +template +inline int& depth(N* node) { + assert(node->get_data().depth > 0); + return node->get_data().depth; +} + +template +N* mrca_from_depth(N* node1, N* node2) { + assert(node1 or node2); + if (not node1) { + return node2; + } + if (not node2) { + return node1; + } + assert(node1 and node2); + assert(get_root(node1) == get_root(node2)); + while (depth(node1) > depth(node2)) { + node1 = node1->get_parent(); + } + while (depth(node1) < depth(node2)) { + node2 = node2->get_parent(); + } + assert(depth(node1) == depth(node2)); + while (node1 != node2) { + assert(node1->get_parent()); + assert(node2->get_parent()); + node1 = node1->get_parent(); + node2 = node2->get_parent(); + } + assert(node1 == node2); + return node1; +} + +template +bool is_ancestor_of_using_depth(N* node1, N* node2) +{ + return mrca_from_depth(node1, node2) == node1; +} + // uses ottID->node mapping, but not the split sets of the nodes template typename T::node_type * find_mrca_from_id_set(T & tree, const OttIdSet & idSet, OttId trigger) { From 5fdb6c4d2ecbfa5cef7ba469ff2e4c283c04cce2 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Wed, 13 Sep 2023 15:58:00 -0400 Subject: [PATCH 12/30] Use is_ancestor_of_using_depth( ) --- otc/ctrie/context_ctrie_db.cpp | 5 +++-- otc/ctrie/context_ctrie_db.h | 2 -- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/otc/ctrie/context_ctrie_db.cpp b/otc/ctrie/context_ctrie_db.cpp index ead19cb2..df0898ae 100644 --- a/otc/ctrie/context_ctrie_db.cpp +++ b/otc/ctrie/context_ctrie_db.cpp @@ -131,8 +131,9 @@ vec_fqr_w_t ContextAwareCTrieBasedDB::to_taxa(const setget_data(); - LOG(DEBUG) << "matched taxon trav = (" << res_tax_data.trav_enter << ", " << res_tax_data.trav_exit << "). filter.trav = (" << filter_trav_enter << ", " << filter_trav_exit << ")"; - if (res_tax_data.trav_exit <= filter_trav_exit && res_tax_data.trav_enter >= filter_trav_enter) { + + if (is_ancestor_of_using_depth(context_root, tax_ptr)) + { const TaxonomicJuniorSynonym * syn_ptr = (const TaxonomicJuniorSynonym *)(tax_thing); if (syn_ptr == nullptr) { LOG(DEBUG) << "pushing non-syn"; diff --git a/otc/ctrie/context_ctrie_db.h b/otc/ctrie/context_ctrie_db.h index 2f15b48f..9d11cf25 100644 --- a/otc/ctrie/context_ctrie_db.h +++ b/otc/ctrie/context_ctrie_db.h @@ -50,8 +50,6 @@ class ContextAwareCTrieBasedDB { private: const Context & context; CompressedTrieBasedDB trie; - std::uint32_t filter_trav_enter = 0; - std::uint32_t trav_exit = UINT32_MAX; std::map match_name_to_taxon; }; From 502397811667ab669b8f70905cbbcb87ddab9779 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Wed, 13 Sep 2023 16:20:35 -0400 Subject: [PATCH 13/30] Update compilers for testing. --- .github/workflows/build.yml | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f772d2cb..5ab415bb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,12 +12,12 @@ jobs: build: strategy: matrix: - name: [ubuntu-gcc-9, - ubuntu-gcc-10, + name: [ubuntu-gcc-10, ubuntu-gcc-11, # ubuntu-gcc-11-sanitize, - ubuntu-gcc-11-debugoptimized, - ubuntu-clang-11, + ubuntu-gcc-12, + ubuntu-gcc-12-debugoptimized, + ubuntu-clang-13, # macos-xcode-12.5 ] @@ -40,6 +40,12 @@ jobs: version: "11" buildtype: "release" + - name: ubuntu-gcc-12 + os: ubuntu-latest + compiler: gcc + version: "12" + buildtype: "release" + # To enable this, we'd probably need to add `export ASAN_OPTIONS=detect_odr_violation=0`. # # - name: ubuntu-gcc-11-sanitize @@ -50,16 +56,16 @@ jobs: # sanitize: "address" # - - name: ubuntu-gcc-11-debugoptimized + - name: ubuntu-gcc-12-debugoptimized os: ubuntu-latest compiler: gcc - version: "11" + version: "12" buildtype: "debugoptimized" - - name: ubuntu-clang-11 + - name: ubuntu-clang-13 os: ubuntu-latest compiler: clang - version: "11" + version: "13" buildtype: "release" # Currently we get "dyld: Library not loaded: @rpath/librestbed.4.dylib" From b99a518ded79a49e38c93e8ccc9eaba250984b08 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Wed, 13 Sep 2023 16:45:30 -0400 Subject: [PATCH 14/30] Cast char32_t to uint32_t to print as integer. --- otc/ctrie/ctrie_db.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/otc/ctrie/ctrie_db.cpp b/otc/ctrie/ctrie_db.cpp index 8d1db2b4..9a2f389b 100644 --- a/otc/ctrie/ctrie_db.cpp +++ b/otc/ctrie/ctrie_db.cpp @@ -117,7 +117,7 @@ void CompressedTrieBasedDB::rebuild_new_trie() LOG(INFO)<<" "< & keys) { std::cerr<<" "< Date: Mon, 18 Sep 2023 13:53:52 -0400 Subject: [PATCH 15/30] Add the depth for the new node. --- otc/taxonomy/patching.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/otc/taxonomy/patching.cpp b/otc/taxonomy/patching.cpp index 211e3850..9bc6a103 100644 --- a/otc/taxonomy/patching.cpp +++ b/otc/taxonomy/patching.cpp @@ -357,6 +357,9 @@ void PatchableTaxonomy::reg_or_rereg_nd(RTRichTaxNode * nnd, auto & rt_data = tree.get_data(); rt_data.name_to_node[tr.name] = nnd; rt_data.id_to_node[nnd->get_ott_id()] = nnd; + + // Fill out depth field for node. + nnd->get_data().depth = nnd->get_parent()->get_data().depth + 1; } TaxonomyRecord & PatchableTaxonomy::get_new_tax_rec(OttId oid, From ce40a03183aac7d710bf8864094199b32d4da107 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 18 Sep 2023 15:23:48 -0400 Subject: [PATCH 16/30] Use depth instead of traversal indices for ancestor/mrca --- otc/ctrie/context_ctrie_db.cpp | 10 +++++----- otc/taxonomy/taxonomy.cpp | 2 +- otc/ws/conflictws.cpp | 2 +- otc/ws/find_node.cpp | 2 +- otc/ws/taxonomyws.cpp | 2 +- otc/ws/tolws.cpp | 12 ++++++------ otc/ws/tolws.h | 1 + otc/ws/trees_to_serve.cpp | 1 + 8 files changed, 17 insertions(+), 15 deletions(-) diff --git a/otc/ctrie/context_ctrie_db.cpp b/otc/ctrie/context_ctrie_db.cpp index df0898ae..8d7a9e62 100644 --- a/otc/ctrie/context_ctrie_db.cpp +++ b/otc/ctrie/context_ctrie_db.cpp @@ -120,7 +120,7 @@ vec_fqr_w_t ContextAwareCTrieBasedDB::to_taxa(const set vec size = " << vec_taxon_and_syn_ptrs.size(); +// LOG(DEBUG) << "FuzzyQueryResult(match=\"" << fqr.match() << "\", score = " << fqr.score << ") -> vec size = " << vec_taxon_and_syn_ptrs.size(); for (auto & [tax_ptr, tax_thing] : vec_taxon_and_syn_ptrs) { if (tax_ptr == nullptr) { @@ -182,8 +182,8 @@ ContextAwareCTrieBasedDB::to_taxa(const optional& n_query, else { const auto & res_tax_data = tax_ptr->get_data(); - LOG(DEBUG) << "matched taxon trav = (" << res_tax_data.trav_enter << ", " << res_tax_data.trav_exit << "). filter.trav = (" << filter_trav_enter << ", " << filter_trav_exit << ")"; - if (res_tax_data.trav_exit <= filter_trav_exit && res_tax_data.trav_enter >= filter_trav_enter) +// LOG(DEBUG) << "matched taxon trav = (" << res_tax_data.trav_enter << ", " << res_tax_data.trav_exit << "). filter.trav = (" << filter_trav_enter << ", " << filter_trav_exit << ")"; + if (is_ancestor_of_using_depth(context_root, tax_ptr)) { const TaxonomicJuniorSynonym * syn_ptr = (const TaxonomicJuniorSynonym *) rec_or_syn_ptr; if (syn_ptr == nullptr) @@ -238,8 +238,8 @@ ContextAwareCTrieBasedDB::to_taxa(const vector& n_queries, else { const auto & res_tax_data = tax_ptr->get_data(); - LOG(DEBUG) << "matched taxon trav = (" << res_tax_data.trav_enter << ", " << res_tax_data.trav_exit << "). filter.trav = (" << filter_trav_enter << ", " << filter_trav_exit << ")"; - if (res_tax_data.trav_exit <= filter_trav_exit && res_tax_data.trav_enter >= filter_trav_enter) +// LOG(DEBUG) << "matched taxon trav = (" << res_tax_data.trav_enter << ", " << res_tax_data.trav_exit << "). filter.trav = (" << filter_trav_enter << ", " << filter_trav_exit << ")"; + if (is_ancestor_of_using_depth(context_root, tax_ptr)) { const TaxonomicJuniorSynonym * syn_ptr = (const TaxonomicJuniorSynonym *) rec_or_syn_ptr; if (syn_ptr == nullptr) diff --git a/otc/taxonomy/taxonomy.cpp b/otc/taxonomy/taxonomy.cpp index a276c352..832b13ea 100644 --- a/otc/taxonomy/taxonomy.cpp +++ b/otc/taxonomy/taxonomy.cpp @@ -1098,7 +1098,7 @@ const RTRichTaxNode* taxonomy_mrca(const std::vector& node } auto focal = nodes[0]; for(auto& node: nodes) { - focal = find_mrca_via_traversal_indices(focal, node); + focal = mrca_from_depth(focal, node); if (not focal) { throw OTCError() << "MRCA of taxa was not found. Please report this bug!\n"; } diff --git a/otc/ws/conflictws.cpp b/otc/ws/conflictws.cpp index 8a6b79b4..7882a890 100644 --- a/otc/ws/conflictws.cpp +++ b/otc/ws/conflictws.cpp @@ -529,7 +529,7 @@ json conflict_with_summary(const ConflictTree& query_tree, return mrca_from_depth(n1,n2); }; std::function summary_mrca = [](const snode_type* n1, const snode_type* n2) { - return find_mrca_via_traversal_indices(n1,n2); + return mrca_from_depth(n1,n2); }; witness_namer_t witness_namer = [&](const string& w) {return synth_witness_namer(w,summary,Tax);}; return conflict_with_tree_impl(query_tree, summary, query_mrca, summary_mrca, witness_namer); diff --git a/otc/ws/find_node.cpp b/otc/ws/find_node.cpp index 7418a666..88c07820 100644 --- a/otc/ws/find_node.cpp +++ b/otc/ws/find_node.cpp @@ -113,7 +113,7 @@ MRCANameToSynth find_node_by_mrca_str(const SummaryTree_t & tree, const RichTaxo const SumTreeNode_t* mrca = nullptr; if (result1.node() and result2.node()) - mrca = find_mrca_via_traversal_indices(result1.node(), result2.node()); + mrca = mrca_from_depth(result1.node(), result2.node()); return MRCANameToSynth{result1, result2, mrca}; } diff --git a/otc/ws/taxonomyws.cpp b/otc/ws/taxonomyws.cpp index 6ddfb72c..7c9bf4f7 100644 --- a/otc/ws/taxonomyws.cpp +++ b/otc/ws/taxonomyws.cpp @@ -210,7 +210,7 @@ string taxonomy_mrca_ws_method(const RichTaxonomy & taxonomy, first = false; focal = n; } else { - focal = find_mrca_via_traversal_indices(focal, n); + focal = mrca_from_depth(focal, n); if (focal == nullptr) { break; } diff --git a/otc/ws/tolws.cpp b/otc/ws/tolws.cpp index 040a6abc..13d873af 100644 --- a/otc/ws/tolws.cpp +++ b/otc/ws/tolws.cpp @@ -502,7 +502,7 @@ const SumTreeNode_t * mrca(const T & nodes) { first = false; focal = n; } else { - focal = find_mrca_via_traversal_indices(focal, n); + focal = mrca_from_depth(focal, n); if (focal == nullptr) { break; } @@ -515,16 +515,16 @@ const SumTreeNode_t * mrca(const T & nodes) { // Check if the excluded node is inside the include group, and update the MRCA of the exclude group if not. bool check_node_and_update_excluded_ancestor(const SumTreeNode_t* mrca_included, const SumTreeNode_t* excluded_node, const SumTreeNode_t* &closest_excluded_ancestor) { - auto mrca = find_mrca_via_traversal_indices(mrca_included, excluded_node); + auto mrca = mrca_from_depth(mrca_included, excluded_node); // Return false if the excluded node is actually inside the include group. if (mrca == mrca_included) return false; - assert(not closest_excluded_ancestor or is_ancestor_of(closest_excluded_ancestor, mrca_included) or is_ancestor_of(mrca_included, closest_excluded_ancestor)); + assert(not closest_excluded_ancestor or is_ancestor_of_using_depth(closest_excluded_ancestor, mrca_included) or is_ancestor_of_using_depth(mrca_included, closest_excluded_ancestor)); // If this is the first excluded taxon OR the mrca is closer, the update the closest excluded ancestor - if (not closest_excluded_ancestor or is_ancestor_of(closest_excluded_ancestor, mrca)) + if (not closest_excluded_ancestor or is_ancestor_of_using_depth(closest_excluded_ancestor, mrca)) closest_excluded_ancestor = mrca; return true; @@ -577,7 +577,7 @@ string mrca_ws_method(const TreesToServe & tts, // 4. Do the phylo-ref response here, if we had any excluded ids. if (not excluded_node_ids.empty()) { - assert(is_ancestor_of(closest_excluded_ancestor, mrca_included)); + assert(is_ancestor_of_using_depth(closest_excluded_ancestor, mrca_included)); json nodes; for(auto node = mrca_included; node != closest_excluded_ancestor; node = node->get_parent()) nodes.push_back(node_id_for_summary_tree_node(*node)); @@ -709,7 +709,7 @@ string induced_subtree_ws_method(const TreesToServe & tts, first = false; focal = n; } else { - focal = find_mrca_via_traversal_indices(focal, n); + focal = mrca_from_depth(focal, n); if (focal == nullptr) { break; } diff --git a/otc/ws/tolws.h b/otc/ws/tolws.h index 74e95c78..964214eb 100644 --- a/otc/ws/tolws.h +++ b/otc/ws/tolws.h @@ -54,6 +54,7 @@ typedef RTRichTaxNode Taxon; class SumTreeNodeData { public: + int depth = 0; std::uint32_t trav_enter = UINT32_MAX; std::uint32_t trav_exit = UINT32_MAX; # if defined(JOINT_MAPPING_VEC) diff --git a/otc/ws/trees_to_serve.cpp b/otc/ws/trees_to_serve.cpp index 4844f429..cfc86343 100644 --- a/otc/ws/trees_to_serve.cpp +++ b/otc/ws/trees_to_serve.cpp @@ -97,6 +97,7 @@ TreesToServe::SumTreeInitPair TreesToServe::get_new_tree_and_annotations(const s unique_ptr nt = first_newick_tree_from_file(filename, parsingRules); index_by_name_or_id(*nt); + compute_depth(*nt); set_traversal_entry_exit_and_num_tips(*nt); tree_list.push_back(move(nt)); annotation_list.emplace(annotation_list.end()); From 30ddfdd8049bceaf3e513d4cb1211c26009cb58d Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Wed, 20 Sep 2023 14:38:51 -0400 Subject: [PATCH 17/30] Find name codes w/o using traversal indices. --- otc/ctrie/context_ctrie_db.cpp | 4 +- otc/tnrs/context.cpp | 82 +++++++++------------------------- otc/tnrs/context.h | 2 +- 3 files changed, 24 insertions(+), 64 deletions(-) diff --git a/otc/ctrie/context_ctrie_db.cpp b/otc/ctrie/context_ctrie_db.cpp index 8d7a9e62..1790e424 100644 --- a/otc/ctrie/context_ctrie_db.cpp +++ b/otc/ctrie/context_ctrie_db.cpp @@ -14,7 +14,7 @@ namespace otc { ContextAwareCTrieBasedDB::ContextAwareCTrieBasedDB(const Context &context_arg, const RichTaxonomy &taxonomy) :context(context_arg) { - Context::init_nom_codes_to_traversal(taxonomy); + Context::init_nom_codes_boundaries(taxonomy); if (context_arg.name_matcher != nullptr) { return; // already initialized } @@ -73,7 +73,7 @@ ContextAwareCTrieBasedDB::ContextAwareCTrieBasedDB(const Context &context_arg, const RichTaxonomy & taxonomy, const std::set &) :context(context_arg) { - Context::init_nom_codes_to_traversal(taxonomy); + Context::init_nom_codes_boundaries(taxonomy); throw OTCError() << "partitioning by context is not implemented yet..."; } diff --git a/otc/tnrs/context.cpp b/otc/tnrs/context.cpp index aacbf238..a0c10b7c 100644 --- a/otc/tnrs/context.cpp +++ b/otc/tnrs/context.cpp @@ -164,37 +164,12 @@ int Context::cull_contexts_to_taxonomy(const RichTaxonomy & taxonomy) { return num_retained; } -using trav_range_t = std::pair; -using nom_cod_and_ranges_t = std::pair >; -using vec_nom_cod_and_ranges_t = std::vector; -vec_nom_cod_and_ranges_t global_nom_code_and_ranges; +std::map nom_code_roots; using id_name_pair_t = std::pair; using vec_id_name_pair_t = std::vector; -std::vector _fill_and_sort_ranges(const RichTaxonomy & taxonomy, const vec_id_name_pair_t & vec_ids_names) { - const auto & tree = taxonomy.get_tax_tree(); - const auto & tax_tree_data = tree.get_data(); - std::vector ret; - for (auto & [id,name] : vec_ids_names) { - const RTRichTaxNode * nd; - try { - nd = tax_tree_data.id_to_node.at(id); - } catch (...) { - throw OTCError() << "Barrier taxon with id=" << id << " not found"; - } - if (nd->get_name() != name) { - throw OTCError() << "Barrier taxon with id=" << id << " had name \"" << nd->get_name() << "\", but we expected \"" << name << "\"."; - } - const auto & ndd = nd->get_data(); - ret.push_back(trav_range_t{ndd.trav_enter, ndd.trav_exit}); - } - std::sort(ret.begin(), ret.end()); - return ret; -} - - /* From BarrierNodes.java in taxomachine... ICN 352914 "Fungi" @@ -212,9 +187,11 @@ ICZN {691846, "Metazoa"}, {202765, "Choanoflagellida"} */ -void Context::init_nom_codes_to_traversal(const RichTaxonomy & taxonomy) { - global_nom_code_and_ranges.clear(); - if (taxonomy.get_tax_tree().get_root()->get_ott_id() != 805080) { +void Context::init_nom_codes_boundaries(const RichTaxonomy & taxonomy) { + nom_code_roots.clear(); + + auto root_ott_id = taxonomy.get_tax_tree().get_root()->get_ott_id(); + if (root_ott_id != 805080) { LOG(WARNING) << "taxonomy root did not have ID=805080, assuming that this is a taxonomy for testing purposes only... detection of nomenclatural code will be non-functional."; return; } @@ -228,43 +205,26 @@ void Context::init_nom_codes_to_traversal(const RichTaxonomy & taxonomy) { {878953, "Rhodophyta"}, {664970, "Glaucophyta"}, {151014, "Haptophyta"}}; - const auto viczn = _fill_and_sort_ranges(taxonomy, iczn); - const auto vinp = _fill_and_sort_ranges(taxonomy, icnp); - const auto vicn = _fill_and_sort_ranges(taxonomy, icn); - global_nom_code_and_ranges.push_back(nom_cod_and_ranges_t{&Nomenclature::ICZN, viczn}); - global_nom_code_and_ranges.push_back(nom_cod_and_ranges_t{&Nomenclature::ICNP, vinp}); - global_nom_code_and_ranges.push_back(nom_cod_and_ranges_t{&Nomenclature::ICN, vicn}); -} -const vec_nom_cod_and_ranges_t & get_nom_code_to_trav_ranges() { - return global_nom_code_and_ranges; -} + for(auto& [tax_id,tax_name]: iczn) + nom_code_roots.insert({tax_id, Nomenclature::ICZN.name}); + for(auto& [tax_id,tax_name]: icnp) + nom_code_roots.insert({tax_id, Nomenclature::ICNP.name}); + for(auto& [tax_id,tax_name]: icn) + nom_code_roots.insert({tax_id, Nomenclature::ICN.name}); -inline bool contains_trav(const std::vector & sorted_trav_pairs, std::uint32_t query) { - for (const auto & p : sorted_trav_pairs) { - if (p.first > query) { - return false; - } - if (p.second >= query) { - return true; - } - } - return false; + if (not nom_code_roots.count(root_ott_id)) + nom_code_roots.insert({root_ott_id, Nomenclature::Undefined.name}); } -const Nomenclature::Code & get_nom_code_for_trav(std::uint32_t trav) { - const auto & nom_code_ranges = get_nom_code_to_trav_ranges(); - for (const auto & c_and_r : nom_code_ranges) { - if (contains_trav(c_and_r.second, trav)) { - return *(c_and_r.first); - } - } - return Nomenclature::Undefined; -} +const std::string & Context::get_code_name(const RichTaxonomy & , const RTRichTaxNode * taxon) +{ + // Walk up the tree until we find an ancestor that is the root of a naming system. + while(taxon and not nom_code_roots.count(taxon->get_ott_id())) + taxon = taxon->get_parent(); -const std::string & Context::get_code_name(const RichTaxonomy & , const RTRichTaxNode * taxon) { - const auto & nom = get_nom_code_for_trav(taxon->get_data().trav_enter); - return nom.name; + assert(taxon); + return nom_code_roots.at(taxon->get_ott_id()); } const RTRichTaxNode * get_closest_anc_with_node(const RichTaxonomy & taxonomy, const TaxonomyRecord * record) { diff --git a/otc/tnrs/context.h b/otc/tnrs/context.h index 08542e05..89406a8e 100644 --- a/otc/tnrs/context.h +++ b/otc/tnrs/context.h @@ -38,7 +38,7 @@ struct Context name_matcher(nullptr) {} - static void init_nom_codes_to_traversal(const RichTaxonomy &); + static void init_nom_codes_boundaries(const RichTaxonomy &); static const std::string & get_code_name(const RichTaxonomy & taxonomy, const RTRichTaxNode * taxon); static const std::string & get_code_name(const RichTaxonomy & taxonomy, const TaxonomyRecord * record); // MUST CALL this FIRST. IMPORTANT GLOBAL SIDE EFFECTS From a523d3fe21416910baf5c4a3d217725773a0800d Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Wed, 20 Sep 2023 14:39:10 -0400 Subject: [PATCH 18/30] Do verbose logging during tests of otc-tol-ws --- ws/test_web_services.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ws/test_web_services.py b/ws/test_web_services.py index 08bda929..cbe3d928 100644 --- a/ws/test_web_services.py +++ b/ws/test_web_services.py @@ -267,7 +267,8 @@ def launch_server(exe_dir, taxonomy_dir, synth_par, server_threads=4): "-D" + synth_par, "-p{}".format(pidfile_path), "-P{}".format(SERVER_PORT), - "--num-threads={}".format(server_threads)] + "--num-threads={}".format(server_threads), + "-v"] _LOG.debug('Launching with: "{}"'.format('" "'.join(invocation))) with open(server_std_out, 'w') as sstdoe: RUNNING_SERVER = subprocess.Popen(invocation, From afdac22f4940c1a5913cb3550d4b4d0d01e06e2a Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Wed, 20 Sep 2023 14:47:51 -0400 Subject: [PATCH 19/30] Stop computing traversal indices. --- otc/ctrie/context_ctrie_db.cpp | 11 ----------- otc/taxonomy/patching.cpp | 1 - otc/taxonomy/taxonomy.cpp | 1 - otc/tree_operations.h | 24 +----------------------- otc/ws/trees_to_serve.cpp | 2 +- 5 files changed, 2 insertions(+), 37 deletions(-) diff --git a/otc/ctrie/context_ctrie_db.cpp b/otc/ctrie/context_ctrie_db.cpp index 1790e424..54ca8c82 100644 --- a/otc/ctrie/context_ctrie_db.cpp +++ b/otc/ctrie/context_ctrie_db.cpp @@ -112,8 +112,6 @@ vec_fqr_w_t ContextAwareCTrieBasedDB::to_taxa(const setget_data(); - const auto filter_trav_enter = tax_data.trav_enter; - const auto filter_trav_exit = tax_data.trav_exit; if (sorted.empty()) { LOG(DEBUG) << "no matches"; @@ -163,8 +161,6 @@ ContextAwareCTrieBasedDB::to_taxa(const optional& n_query, vector results; const auto & tax_data = context_root->get_data(); - const auto filter_trav_enter = tax_data.trav_enter; - const auto filter_trav_exit = tax_data.trav_exit; const auto & vec_taxon_and_syn_ptrs = match_name_to_taxon.at(*n_query); LOG(DEBUG) << "exact_query(match=\"" << *n_query << ") -> vec size = " << vec_taxon_and_syn_ptrs.size(); @@ -182,7 +178,6 @@ ContextAwareCTrieBasedDB::to_taxa(const optional& n_query, else { const auto & res_tax_data = tax_ptr->get_data(); -// LOG(DEBUG) << "matched taxon trav = (" << res_tax_data.trav_enter << ", " << res_tax_data.trav_exit << "). filter.trav = (" << filter_trav_enter << ", " << filter_trav_exit << ")"; if (is_ancestor_of_using_depth(context_root, tax_ptr)) { const TaxonomicJuniorSynonym * syn_ptr = (const TaxonomicJuniorSynonym *) rec_or_syn_ptr; @@ -217,8 +212,6 @@ ContextAwareCTrieBasedDB::to_taxa(const vector& n_queries, vector results; const auto & tax_data = context_root->get_data(); - const auto filter_trav_enter = tax_data.trav_enter; - const auto filter_trav_exit = tax_data.trav_exit; for(auto& n_query: n_queries) { @@ -238,7 +231,6 @@ ContextAwareCTrieBasedDB::to_taxa(const vector& n_queries, else { const auto & res_tax_data = tax_ptr->get_data(); -// LOG(DEBUG) << "matched taxon trav = (" << res_tax_data.trav_enter << ", " << res_tax_data.trav_exit << "). filter.trav = (" << filter_trav_enter << ", " << filter_trav_exit << ")"; if (is_ancestor_of_using_depth(context_root, tax_ptr)) { const TaxonomicJuniorSynonym * syn_ptr = (const TaxonomicJuniorSynonym *) rec_or_syn_ptr; @@ -267,9 +259,6 @@ vec_fqr_w_t ContextAwareCTrieBasedDB::fuzzy_query_to_taxa(const std::string & qu return to_taxa(fuzzy_query(query_str), context_root, taxonomy, include_suppressed); } -// how do trav_enter and such things work? -// what are they used for? - void ContextAwareCTrieBasedDB::add_key(const std::string& s, OttId id, const RichTaxonomy& taxonomy) { auto node = taxonomy.included_taxon_from_id(id); diff --git a/otc/taxonomy/patching.cpp b/otc/taxonomy/patching.cpp index 9bc6a103..aa2ea19a 100644 --- a/otc/taxonomy/patching.cpp +++ b/otc/taxonomy/patching.cpp @@ -342,7 +342,6 @@ bool_str_t PatchableTaxonomy::add_new_taxon(OttId oid, if (auto f = get_fuzzy_matcher()) f->add_key(name, oid, *this); - // TODO: fix trav_enter, etc. // TODO: fix allocation of names between contexts // TODO: read amendment blob diff --git a/otc/taxonomy/taxonomy.cpp b/otc/taxonomy/taxonomy.cpp index 832b13ea..6ad74d7a 100644 --- a/otc/taxonomy/taxonomy.cpp +++ b/otc/taxonomy/taxonomy.cpp @@ -692,7 +692,6 @@ RichTaxonomy::RichTaxonomy(const std::string& dir, } } compute_depth(*tree); - set_traversal_entry_exit(*tree); _fill_ids_to_suppress_set(); this->read_synonyms(); const auto & td = tree->get_data(); diff --git a/otc/tree_operations.h b/otc/tree_operations.h index 2bdef996..0e86ef55 100644 --- a/otc/tree_operations.h +++ b/otc/tree_operations.h @@ -1535,36 +1535,14 @@ void index_nodes_by_name(T & tree) { } template -void set_traversal_entry_exit(T & tree) { +void set_num_tips(T & tree) { std::uint32_t ind = 0; - for (auto nd : iter_pre(tree)) { - nd->get_data().trav_enter = ind++; - } - for (auto pnd : iter_post(tree)) { - auto fc = pnd->get_last_child(); - auto & d = pnd->get_data(); - if (fc == nullptr) { - d.trav_exit = d.trav_enter; - } else { - d.trav_exit = fc->get_data().trav_exit; - } - } -} - -template -void set_traversal_entry_exit_and_num_tips(T & tree) { - std::uint32_t ind = 0; - for (auto nd : iter_pre(tree)) { - nd->get_data().trav_enter = ind++; - } for (auto pnd : iter_post(tree)) { auto fc = pnd->get_last_child(); auto & d = pnd->get_data(); if (fc == nullptr) { - d.trav_exit = d.trav_enter; d.num_tips = 1; } else { - d.trav_exit = fc->get_data().trav_exit; d.num_tips = 0; for (auto c : iter_child_const(*pnd)) { d.num_tips += c->get_data().num_tips; diff --git a/otc/ws/trees_to_serve.cpp b/otc/ws/trees_to_serve.cpp index cfc86343..116a512f 100644 --- a/otc/ws/trees_to_serve.cpp +++ b/otc/ws/trees_to_serve.cpp @@ -98,7 +98,7 @@ TreesToServe::SumTreeInitPair TreesToServe::get_new_tree_and_annotations(const s index_by_name_or_id(*nt); compute_depth(*nt); - set_traversal_entry_exit_and_num_tips(*nt); + set_num_tips(*nt); tree_list.push_back(move(nt)); annotation_list.emplace(annotation_list.end()); auto & sta = annotation_list.back(); From bf425b8bea21f894599a503f47860c72d22fb280 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Wed, 20 Sep 2023 14:50:54 -0400 Subject: [PATCH 20/30] Remove trav_{enter,exit} fields from tax tree and sum tree. --- otc/taxonomy/taxonomy.h | 22 ---------------------- otc/ws/tolws.cpp | 11 ----------- otc/ws/tolws.h | 2 -- 3 files changed, 35 deletions(-) diff --git a/otc/taxonomy/taxonomy.h b/otc/taxonomy/taxonomy.h index 0757f7e4..efa5896a 100644 --- a/otc/taxonomy/taxonomy.h +++ b/otc/taxonomy/taxonomy.h @@ -272,8 +272,6 @@ class RTRichTaxNodeData { //TaxonomicRank rank = TaxonomicRank::RANK_NO_RANK; //nlohmann::json sources; std::vector junior_synonyms; - std::uint32_t trav_enter = UINT32_MAX; - std::uint32_t trav_exit = UINT32_MAX; TaxonomicRank rank = TaxonomicRank::RANK_NO_RANK; std::bitset<32> flags; std::string source_info; @@ -661,25 +659,5 @@ std::vector exact_name_search(const RichTaxonomy& taxonomy const std::string& query, std::function ok = [](const RTRichTaxNode*){return true;}); - -template -N * find_mrca_via_traversal_indices(N *f, N *s); - -template -inline N * find_mrca_via_traversal_indices(N *f, N *s) { - const auto * fdata = &(f->get_data()); - const auto sec_ind = s->get_data().trav_enter; - while (sec_ind < fdata->trav_enter || sec_ind > fdata->trav_exit) { - f = f->get_parent(); - if (f == nullptr) { - assert(false); - return nullptr; - } - fdata = &(f->get_data()); - } - return f; -} - - } // namespace #endif diff --git a/otc/ws/tolws.cpp b/otc/ws/tolws.cpp index 13d873af..9cc8d9a9 100644 --- a/otc/ws/tolws.cpp +++ b/otc/ws/tolws.cpp @@ -276,17 +276,6 @@ void add_node_support_info(const TreesToServe & tts, } } - -template -bool is_ancestor_of(N* f, N* s) -{ - assert(f); - assert(s); - const auto * fdata = &(f->get_data()); - const auto sec_ind = s->get_data().trav_enter; - return (sec_ind >= fdata->trav_enter and sec_ind <= fdata->trav_exit); -} - // See API docs at https://github.com/OpenTreeOfLife/germinator/wiki/Synthetic-tree-API-v3 string available_trees_ws_method(const TreesToServe &tts) { diff --git a/otc/ws/tolws.h b/otc/ws/tolws.h index 964214eb..165f94cf 100644 --- a/otc/ws/tolws.h +++ b/otc/ws/tolws.h @@ -55,8 +55,6 @@ typedef RTRichTaxNode Taxon; class SumTreeNodeData { public: int depth = 0; - std::uint32_t trav_enter = UINT32_MAX; - std::uint32_t trav_exit = UINT32_MAX; # if defined(JOINT_MAPPING_VEC) vec_src_node_ids source_edge_mappings; # else From c697f2d091d262e778c345ecde7416bf25f632d7 Mon Sep 17 00:00:00 2001 From: Benjamin Redelings Date: Mon, 25 Sep 2023 14:28:54 -0400 Subject: [PATCH 21/30] Update nlohmann json from 3.5.0 to 3.11.2 --- otc/nlohmann/json.hpp | 31567 +++++++++++++++++++++++----------------- 1 file changed, 17919 insertions(+), 13648 deletions(-) diff --git a/otc/nlohmann/json.hpp b/otc/nlohmann/json.hpp index c9af0bed..ac4a1d08 100644 --- a/otc/nlohmann/json.hpp +++ b/otc/nlohmann/json.hpp @@ -1,8459 +1,14984 @@ -/* - __ _____ _____ _____ - __| | __| | | | JSON for Modern C++ -| | |__ | | | | | | version 3.5.0 -|_____|_____|_____|_|___| https://github.com/nlohmann/json - -Licensed under the MIT License . -SPDX-License-Identifier: MIT -Copyright (c) 2013-2018 Niels Lohmann . - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -*/ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.11.2 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-License-Identifier: MIT -#ifndef NLOHMANN_JSON_HPP -#define NLOHMANN_JSON_HPP +/****************************************************************************\ + * Note on documentation: The source files contain links to the online * + * documentation of the public API at https://json.nlohmann.me. This URL * + * contains the most recent documentation and should also be applicable to * + * previous versions; documentation for deprecated functions is not * + * removed, but marked deprecated. See "Generate documentation" section in * + * file docs/README.md. * +\****************************************************************************/ -#define NLOHMANN_JSON_VERSION_MAJOR 3 -#define NLOHMANN_JSON_VERSION_MINOR 5 -#define NLOHMANN_JSON_VERSION_PATCH 0 +#ifndef INCLUDE_NLOHMANN_JSON_HPP_ +#define INCLUDE_NLOHMANN_JSON_HPP_ #include // all_of, find, for_each -#include // assert -#include // and, not, or #include // nullptr_t, ptrdiff_t, size_t #include // hash, less #include // initializer_list -#include // istream, ostream +#ifndef JSON_NO_IO + #include // istream, ostream +#endif // JSON_NO_IO #include // random_access_iterator_tag -#include // accumulate +#include // unique_ptr #include // string, stoi, to_string #include // declval, forward, move, pair, swap - -// #include -#ifndef NLOHMANN_JSON_FWD_HPP -#define NLOHMANN_JSON_FWD_HPP - -#include // int64_t, uint64_t -#include // map -#include // allocator -#include // string #include // vector -/*! -@brief namespace for Niels Lohmann -@see https://github.com/nlohmann -@since version 1.0.0 -*/ -namespace nlohmann -{ -/*! -@brief default JSONSerializer template argument - -This serializer ignores the template arguments and uses ADL -([argument-dependent lookup](https://en.cppreference.com/w/cpp/language/adl)) -for serialization. -*/ -template -struct adl_serializer; - -template class ObjectType = - std::map, - template class ArrayType = std::vector, - class StringType = std::string, class BooleanType = bool, - class NumberIntegerType = std::int64_t, - class NumberUnsignedType = std::uint64_t, - class NumberFloatType = double, - template class AllocatorType = std::allocator, - template class JSONSerializer = - adl_serializer> -class basic_json; - -/*! -@brief JSON Pointer - -A JSON pointer defines a string syntax for identifying a specific value -within a JSON document. It can be used with functions `at` and -`operator[]`. Furthermore, JSON pointers are the base for JSON patches. - -@sa [RFC 6901](https://tools.ietf.org/html/rfc6901) - -@since version 2.0.0 -*/ -template -class json_pointer; +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.11.2 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-License-Identifier: MIT -/*! -@brief default JSON class -This type is the default specialization of the @ref basic_json class which -uses the standard template types. -@since version 1.0.0 -*/ -using json = basic_json<>; -} // namespace nlohmann +#include -#endif +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.11.2 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-License-Identifier: MIT -// #include -// This file contains all internal macro definitions -// You MUST include macro_unscope.hpp at the end of json.hpp to undef all of them +// This file contains all macro definitions affecting or depending on the ABI -// exclude unsupported compilers -#if !defined(JSON_SKIP_UNSUPPORTED_COMPILER_CHECK) - #if defined(__clang__) - #if (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) < 30400 - #error "unsupported Clang version - see https://github.com/nlohmann/json#supported-compilers" - #endif - #elif defined(__GNUC__) && !(defined(__ICC) || defined(__INTEL_COMPILER)) - #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) < 40800 - #error "unsupported GCC version - see https://github.com/nlohmann/json#supported-compilers" +#ifndef JSON_SKIP_LIBRARY_VERSION_CHECK + #if defined(NLOHMANN_JSON_VERSION_MAJOR) && defined(NLOHMANN_JSON_VERSION_MINOR) && defined(NLOHMANN_JSON_VERSION_PATCH) + #if NLOHMANN_JSON_VERSION_MAJOR != 3 || NLOHMANN_JSON_VERSION_MINOR != 11 || NLOHMANN_JSON_VERSION_PATCH != 2 + #warning "Already included a different version of the library!" #endif #endif #endif -// disable float-equal warnings on GCC/clang -#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) - #pragma GCC diagnostic push - #pragma GCC diagnostic ignored "-Wfloat-equal" +#define NLOHMANN_JSON_VERSION_MAJOR 3 // NOLINT(modernize-macro-to-enum) +#define NLOHMANN_JSON_VERSION_MINOR 11 // NOLINT(modernize-macro-to-enum) +#define NLOHMANN_JSON_VERSION_PATCH 2 // NOLINT(modernize-macro-to-enum) + +#ifndef JSON_DIAGNOSTICS + #define JSON_DIAGNOSTICS 0 #endif -// disable documentation warnings on clang -#if defined(__clang__) - #pragma GCC diagnostic push - #pragma GCC diagnostic ignored "-Wdocumentation" +#ifndef JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON + #define JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON 0 #endif -// allow for portable deprecation warnings -#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) - #define JSON_DEPRECATED __attribute__((deprecated)) -#elif defined(_MSC_VER) - #define JSON_DEPRECATED __declspec(deprecated) +#if JSON_DIAGNOSTICS + #define NLOHMANN_JSON_ABI_TAG_DIAGNOSTICS _diag #else - #define JSON_DEPRECATED + #define NLOHMANN_JSON_ABI_TAG_DIAGNOSTICS #endif -// allow to disable exceptions -#if (defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)) && !defined(JSON_NOEXCEPTION) - #define JSON_THROW(exception) throw exception - #define JSON_TRY try - #define JSON_CATCH(exception) catch(exception) - #define JSON_INTERNAL_CATCH(exception) catch(exception) +#if JSON_USE_LEGACY_DISCARDED_VALUE_COMPARISON + #define NLOHMANN_JSON_ABI_TAG_LEGACY_DISCARDED_VALUE_COMPARISON _ldvcmp #else - #define JSON_THROW(exception) std::abort() - #define JSON_TRY if(true) - #define JSON_CATCH(exception) if(false) - #define JSON_INTERNAL_CATCH(exception) if(false) + #define NLOHMANN_JSON_ABI_TAG_LEGACY_DISCARDED_VALUE_COMPARISON #endif -// override exception macros -#if defined(JSON_THROW_USER) - #undef JSON_THROW - #define JSON_THROW JSON_THROW_USER -#endif -#if defined(JSON_TRY_USER) - #undef JSON_TRY - #define JSON_TRY JSON_TRY_USER -#endif -#if defined(JSON_CATCH_USER) - #undef JSON_CATCH - #define JSON_CATCH JSON_CATCH_USER - #undef JSON_INTERNAL_CATCH - #define JSON_INTERNAL_CATCH JSON_CATCH_USER -#endif -#if defined(JSON_INTERNAL_CATCH_USER) - #undef JSON_INTERNAL_CATCH - #define JSON_INTERNAL_CATCH JSON_INTERNAL_CATCH_USER +#ifndef NLOHMANN_JSON_NAMESPACE_NO_VERSION + #define NLOHMANN_JSON_NAMESPACE_NO_VERSION 0 #endif -// manual branch prediction -#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) - #define JSON_LIKELY(x) __builtin_expect(!!(x), 1) - #define JSON_UNLIKELY(x) __builtin_expect(!!(x), 0) -#else - #define JSON_LIKELY(x) x - #define JSON_UNLIKELY(x) x -#endif +// Construct the namespace ABI tags component +#define NLOHMANN_JSON_ABI_TAGS_CONCAT_EX(a, b) json_abi ## a ## b +#define NLOHMANN_JSON_ABI_TAGS_CONCAT(a, b) \ + NLOHMANN_JSON_ABI_TAGS_CONCAT_EX(a, b) -// C++ language standard detection -#if (defined(__cplusplus) && __cplusplus >= 201703L) || (defined(_HAS_CXX17) && _HAS_CXX17 == 1) // fix for issue #464 - #define JSON_HAS_CPP_17 - #define JSON_HAS_CPP_14 -#elif (defined(__cplusplus) && __cplusplus >= 201402L) || (defined(_HAS_CXX14) && _HAS_CXX14 == 1) - #define JSON_HAS_CPP_14 -#endif +#define NLOHMANN_JSON_ABI_TAGS \ + NLOHMANN_JSON_ABI_TAGS_CONCAT( \ + NLOHMANN_JSON_ABI_TAG_DIAGNOSTICS, \ + NLOHMANN_JSON_ABI_TAG_LEGACY_DISCARDED_VALUE_COMPARISON) -/*! -@brief macro to briefly define a mapping between an enum and JSON -@def NLOHMANN_JSON_SERIALIZE_ENUM -@since version 3.4.0 -*/ -#define NLOHMANN_JSON_SERIALIZE_ENUM(ENUM_TYPE, ...) \ - template \ - inline void to_json(BasicJsonType& j, const ENUM_TYPE& e) \ - { \ - static_assert(std::is_enum::value, #ENUM_TYPE " must be an enum!"); \ - static const std::pair m[] = __VA_ARGS__; \ - auto it = std::find_if(std::begin(m), std::end(m), \ - [e](const std::pair& ej_pair) -> bool \ - { \ - return ej_pair.first == e; \ - }); \ - j = ((it != std::end(m)) ? it : std::begin(m))->second; \ - } \ - template \ - inline void from_json(const BasicJsonType& j, ENUM_TYPE& e) \ - { \ - static_assert(std::is_enum::value, #ENUM_TYPE " must be an enum!"); \ - static const std::pair m[] = __VA_ARGS__; \ - auto it = std::find_if(std::begin(m), std::end(m), \ - [j](const std::pair& ej_pair) -> bool \ - { \ - return ej_pair.second == j; \ - }); \ - e = ((it != std::end(m)) ? it : std::begin(m))->first; \ - } +// Construct the namespace version component +#define NLOHMANN_JSON_NAMESPACE_VERSION_CONCAT_EX(major, minor, patch) \ + _v ## major ## _ ## minor ## _ ## patch +#define NLOHMANN_JSON_NAMESPACE_VERSION_CONCAT(major, minor, patch) \ + NLOHMANN_JSON_NAMESPACE_VERSION_CONCAT_EX(major, minor, patch) -// Ugly macros to avoid uglier copy-paste when specializing basic_json. They -// may be removed in the future once the class is split. +#if NLOHMANN_JSON_NAMESPACE_NO_VERSION +#define NLOHMANN_JSON_NAMESPACE_VERSION +#else +#define NLOHMANN_JSON_NAMESPACE_VERSION \ + NLOHMANN_JSON_NAMESPACE_VERSION_CONCAT(NLOHMANN_JSON_VERSION_MAJOR, \ + NLOHMANN_JSON_VERSION_MINOR, \ + NLOHMANN_JSON_VERSION_PATCH) +#endif -#define NLOHMANN_BASIC_JSON_TPL_DECLARATION \ - template class ObjectType, \ - template class ArrayType, \ - class StringType, class BooleanType, class NumberIntegerType, \ - class NumberUnsignedType, class NumberFloatType, \ - template class AllocatorType, \ - template class JSONSerializer> +// Combine namespace components +#define NLOHMANN_JSON_NAMESPACE_CONCAT_EX(a, b) a ## b +#define NLOHMANN_JSON_NAMESPACE_CONCAT(a, b) \ + NLOHMANN_JSON_NAMESPACE_CONCAT_EX(a, b) -#define NLOHMANN_BASIC_JSON_TPL \ - basic_json +#ifndef NLOHMANN_JSON_NAMESPACE +#define NLOHMANN_JSON_NAMESPACE \ + nlohmann::NLOHMANN_JSON_NAMESPACE_CONCAT( \ + NLOHMANN_JSON_ABI_TAGS, \ + NLOHMANN_JSON_NAMESPACE_VERSION) +#endif -// #include +#ifndef NLOHMANN_JSON_NAMESPACE_BEGIN +#define NLOHMANN_JSON_NAMESPACE_BEGIN \ + namespace nlohmann \ + { \ + inline namespace NLOHMANN_JSON_NAMESPACE_CONCAT( \ + NLOHMANN_JSON_ABI_TAGS, \ + NLOHMANN_JSON_NAMESPACE_VERSION) \ + { +#endif +#ifndef NLOHMANN_JSON_NAMESPACE_END +#define NLOHMANN_JSON_NAMESPACE_END \ + } /* namespace (inline namespace) NOLINT(readability/namespace) */ \ + } // namespace nlohmann +#endif -#include // not -#include // size_t -#include // conditional, enable_if, false_type, integral_constant, is_constructible, is_integral, is_same, remove_cv, remove_reference, true_type +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.11.2 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-License-Identifier: MIT -namespace nlohmann -{ -namespace detail -{ -// alias templates to reduce boilerplate -template -using enable_if_t = typename std::enable_if::type; -template -using uncvref_t = typename std::remove_cv::type>::type; -// implementation of C++14 index_sequence and affiliates -// source: https://stackoverflow.com/a/32223343 -template -struct index_sequence -{ - using type = index_sequence; - using value_type = std::size_t; - static constexpr std::size_t size() noexcept - { - return sizeof...(Ints); - } -}; +#include // transform +#include // array +#include // forward_list +#include // inserter, front_inserter, end +#include // map +#include // string +#include // tuple, make_tuple +#include // is_arithmetic, is_same, is_enum, underlying_type, is_convertible +#include // unordered_map +#include // pair, declval +#include // valarray -template -struct merge_and_renumber; +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.11.2 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-License-Identifier: MIT -template -struct merge_and_renumber, index_sequence> - : index_sequence < I1..., (sizeof...(I1) + I2)... > {}; -template -struct make_index_sequence - : merge_and_renumber < typename make_index_sequence < N / 2 >::type, - typename make_index_sequence < N - N / 2 >::type > {}; -template<> struct make_index_sequence<0> : index_sequence<> {}; -template<> struct make_index_sequence<1> : index_sequence<0> {}; +#include // nullptr_t +#include // exception +#if JSON_DIAGNOSTICS + #include // accumulate +#endif +#include // runtime_error +#include // to_string +#include // vector -template -using index_sequence_for = make_index_sequence; +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.11.2 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-License-Identifier: MIT -// dispatch utility (taken from ranges-v3) -template struct priority_tag : priority_tag < N - 1 > {}; -template<> struct priority_tag<0> {}; -// taken from ranges-v3 -template -struct static_const -{ - static constexpr T value{}; -}; -template -constexpr T static_const::value; -} // namespace detail -} // namespace nlohmann +#include // array +#include // size_t +#include // uint8_t +#include // string -// #include +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.11.2 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-License-Identifier: MIT -#include // not -#include // numeric_limits -#include // false_type, is_constructible, is_integral, is_same, true_type -#include // declval -// #include +#include // declval, pair +// #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.11.2 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-License-Identifier: MIT -// #include -#include // random_access_iterator_tag +#include // #include +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ +// | | |__ | | | | | | version 3.11.2 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2022 Niels Lohmann +// SPDX-License-Identifier: MIT -namespace nlohmann -{ -namespace detail -{ -template struct make_void -{ - using type = void; -}; -template using void_t = typename make_void::type; -} // namespace detail -} // namespace nlohmann -// #include +// #include -namespace nlohmann -{ +NLOHMANN_JSON_NAMESPACE_BEGIN namespace detail { -template -struct iterator_types {}; - -template -struct iterator_types < - It, - void_t> -{ - using difference_type = typename It::difference_type; - using value_type = typename It::value_type; - using pointer = typename It::pointer; - using reference = typename It::reference; - using iterator_category = typename It::iterator_category; -}; - -// This is required as some compilers implement std::iterator_traits in a way that -// doesn't work with SFINAE. See https://github.com/nlohmann/json/issues/1341. -template -struct iterator_traits -{ -}; - -template -struct iterator_traits < T, enable_if_t < !std::is_pointer::value >> - : iterator_types -{ -}; -template -struct iterator_traits::value>> +template struct make_void { - using iterator_category = std::random_access_iterator_tag; - using value_type = T; - using difference_type = ptrdiff_t; - using pointer = T*; - using reference = T&; + using type = void; }; -} -} - -// #include +template using void_t = typename make_void::type; -// #include - - -#include - -// #include +} // namespace detail +NLOHMANN_JSON_NAMESPACE_END -// http://en.cppreference.com/w/cpp/experimental/is_detected -namespace nlohmann -{ +NLOHMANN_JSON_NAMESPACE_BEGIN namespace detail { + +// https://en.cppreference.com/w/cpp/experimental/is_detected struct nonesuch { nonesuch() = delete; ~nonesuch() = delete; nonesuch(nonesuch const&) = delete; + nonesuch(nonesuch const&&) = delete; void operator=(nonesuch const&) = delete; + void operator=(nonesuch&&) = delete; }; -template class Op, - class... Args> +template class Op, + class... Args> struct detector { using value_t = std::false_type; using type = Default; }; -template class Op, class... Args> +template class Op, class... Args> struct detector>, Op, Args...> { using value_t = std::true_type; using type = Op; }; -template