From b043965755e3789e2196c45742ff6b1d3277cd62 Mon Sep 17 00:00:00 2001 From: Alberto Islas Date: Fri, 18 Oct 2024 19:10:43 -0600 Subject: [PATCH] fix(elasticsearch): Restored use of partial for update_es_document task - Fixed failing tests. --- cl/lib/es_signal_processor.py | 54 +++++++++++++--------- cl/search/tests/tests_es_opinion.py | 24 +++++----- cl/search/tests/tests_es_oral_arguments.py | 12 ++--- cl/search/tests/tests_es_parenthetical.py | 8 ++-- cl/search/tests/tests_es_person.py | 16 +++---- cl/search/tests/tests_es_recap.py | 42 ++++++++--------- cl/tests/cases.py | 7 +-- 7 files changed, 86 insertions(+), 77 deletions(-) diff --git a/cl/lib/es_signal_processor.py b/cl/lib/es_signal_processor.py index 693fd9f023..d46a735e07 100644 --- a/cl/lib/es_signal_processor.py +++ b/cl/lib/es_signal_processor.py @@ -310,7 +310,8 @@ def update_es_documents( # Update main document in ES, including fields to be # extracted from a related instance. transaction.on_commit( - lambda: update_es_document.si( + partial( + update_es_document.delay, es_document.__name__, fields_to_update, ( @@ -319,7 +320,7 @@ def update_es_documents( ), (compose_app_label(instance), instance.pk), fields_map, - ).delay() + ) ) @@ -367,7 +368,8 @@ def update_m2m_field_in_es_document( :return: None """ transaction.on_commit( - lambda: update_es_document.si( + partial( + update_es_document.delay, es_document.__name__, [ affected_field, @@ -375,7 +377,7 @@ def update_m2m_field_in_es_document( (compose_app_label(instance), instance.pk), None, None, - ).delay() + ) ) if es_document is OpinionClusterDocument and isinstance( @@ -430,17 +432,19 @@ def update_reverse_related_documents( if isinstance(main_object, Person) and not main_object.is_judge: continue transaction.on_commit( - lambda: chain( - update_es_document.si( - es_document.__name__, - affected_fields, - (compose_app_label(main_object), main_object.pk), - related_instance, - fields_map_to_pass, - ), - send_or_schedule_search_alerts.s(), - percolator_response_processing.s(), - ).apply_async() + partial( + chain( + update_es_document.si( + es_document.__name__, + affected_fields, + (compose_app_label(main_object), main_object.pk), + related_instance, + fields_map_to_pass, + ), + send_or_schedule_search_alerts.s(), + percolator_response_processing.s(), + ).apply_async + ) ) match instance: @@ -511,13 +515,14 @@ def delete_reverse_related_documents( # Update the Person document after the reverse instanced is deleted # Update parent document in ES. transaction.on_commit( - lambda: update_es_document.si( + partial( + update_es_document.delay, es_document.__name__, affected_fields, (compose_app_label(instance), instance.pk), None, None, - ).delay() + ) ) # Avoid calling update_children_docs_by_query if the Person # doesn't have any positions or is not a Judge. @@ -537,13 +542,14 @@ def delete_reverse_related_documents( # Update parent document in ES. transaction.on_commit( - lambda: update_es_document.si( + partial( + update_es_document.delay, es_document.__name__, affected_fields, (compose_app_label(instance), instance.pk), None, None, - ).delay() + ) ) # Avoid calling update_children_docs_by_query if the Docket # doesn't have any entries. @@ -561,13 +567,14 @@ def delete_reverse_related_documents( case OpinionCluster() if es_document is OpinionClusterDocument: # type: ignore # Update parent document in ES. transaction.on_commit( - lambda: update_es_document.si( + partial( + update_es_document.delay, es_document.__name__, affected_fields, (compose_app_label(instance), instance.pk), None, None, - ).delay() + ) ) # Then update all their child documents (Positions) transaction.on_commit( @@ -585,13 +592,14 @@ def delete_reverse_related_documents( for main_object in main_objects: # Update main document in ES. transaction.on_commit( - lambda: update_es_document.si( + partial( + update_es_document.delay, es_document.__name__, affected_fields, (compose_app_label(main_object), main_object.pk), None, None, - ).delay() + ) ) diff --git a/cl/search/tests/tests_es_opinion.py b/cl/search/tests/tests_es_opinion.py index 24f95886d9..c7d9c2568d 100644 --- a/cl/search/tests/tests_es_opinion.py +++ b/cl/search/tests/tests_es_opinion.py @@ -3072,7 +3072,7 @@ def test_child_document_update_properly(self) -> None: with mock.patch( "cl.lib.es_signal_processor.es_save_document.si", side_effect=lambda *args, **kwargs: self.count_task_calls( - es_save_document, *args, **kwargs + es_save_document, True, *args, **kwargs ), ): opinion_cluster = OpinionClusterFactory.create( @@ -3106,7 +3106,7 @@ def test_child_document_update_properly(self) -> None: with mock.patch( "cl.lib.es_signal_processor.update_es_document.si", side_effect=lambda *args, **kwargs: self.count_task_calls( - update_es_document, *args, **kwargs + update_es_document, True, *args, **kwargs ), ): # Update the author field in the opinion record. @@ -3119,7 +3119,7 @@ def test_child_document_update_properly(self) -> None: with mock.patch( "cl.lib.es_signal_processor.update_es_document.si", side_effect=lambda *args, **kwargs: self.count_task_calls( - update_es_document, *args, **kwargs + update_es_document, True, *args, **kwargs ), ): opinion.joined_by_str = "Joined Lorem" @@ -3194,7 +3194,7 @@ def test_child_document_update_properly(self) -> None: with mock.patch( "cl.lib.es_signal_processor.update_es_document.si", side_effect=lambda *args, **kwargs: self.count_task_calls( - update_es_document, *args, **kwargs + update_es_document, True, *args, **kwargs ), ): # Add OpinionsCited using save() as in add_manual_citations command @@ -3211,7 +3211,7 @@ def test_child_document_update_properly(self) -> None: with mock.patch( "cl.lib.es_signal_processor.update_es_document.si", side_effect=lambda *args, **kwargs: self.count_task_calls( - update_es_document, *args, **kwargs + update_es_document, True, *args, **kwargs ), ): # Add OpinionsCited using bulk_create as in store_opinion_citations_and_update_parentheticals @@ -3295,9 +3295,9 @@ def test_parent_document_update_fields_properly(self) -> None: ) with mock.patch( - "cl.lib.es_signal_processor.update_es_document.si", + "cl.lib.es_signal_processor.update_es_document.delay", side_effect=lambda *args, **kwargs: self.count_task_calls( - update_es_document, *args, **kwargs + update_es_document, False, *args, **kwargs ), ): # Update the court field in the docket record. @@ -3311,9 +3311,9 @@ def test_parent_document_update_fields_properly(self) -> None: # Update a opinion_cluster untracked field. with mock.patch( - "cl.lib.es_signal_processor.update_es_document.si", + "cl.lib.es_signal_processor.update_es_document.delay", side_effect=lambda *args, **kwargs: self.count_task_calls( - update_es_document, *args, **kwargs + update_es_document, False, *args, **kwargs ), ): opinion_cluster.other_dates = "January 12" @@ -3386,9 +3386,9 @@ def test_update_shared_fields_related_documents(self) -> None: ) with mock.patch( - "cl.lib.es_signal_processor.update_es_document.si", + "cl.lib.es_signal_processor.update_es_document.delay", side_effect=lambda *args, **kwargs: self.count_task_calls( - update_es_document, *args, **kwargs + update_es_document, False, *args, **kwargs ), ): # update docket number in parent document @@ -3411,7 +3411,7 @@ def test_update_shared_fields_related_documents(self) -> None: with mock.patch( "cl.lib.es_signal_processor.update_children_docs_by_query.delay", side_effect=lambda *args, **kwargs: self.count_task_calls( - update_children_docs_by_query, *args, **kwargs + update_children_docs_by_query, False, *args, **kwargs ), ): # update docket number in parent document diff --git a/cl/search/tests/tests_es_oral_arguments.py b/cl/search/tests/tests_es_oral_arguments.py index a0764b4bda..24f3970fba 100644 --- a/cl/search/tests/tests_es_oral_arguments.py +++ b/cl/search/tests/tests_es_oral_arguments.py @@ -2601,7 +2601,7 @@ def test_oa_indexing_and_tasks_count(self) -> None: with mock.patch( "cl.lib.es_signal_processor.es_save_document.si", side_effect=lambda *args, **kwargs: self.count_task_calls( - es_save_document, *args, **kwargs + es_save_document, True, *args, **kwargs ), ): audio = AudioFactory.create( @@ -2617,7 +2617,7 @@ def test_oa_indexing_and_tasks_count(self) -> None: with mock.patch( "cl.lib.es_signal_processor.es_save_document.si", side_effect=lambda *args, **kwargs: self.count_task_calls( - es_save_document, *args, **kwargs + es_save_document, True, *args, **kwargs ), ): audio.save( @@ -2633,7 +2633,7 @@ def test_oa_indexing_and_tasks_count(self) -> None: with mock.patch( "cl.lib.es_signal_processor.update_es_document.si", side_effect=lambda *args, **kwargs: self.count_task_calls( - update_es_document, *args, **kwargs + update_es_document, True, *args, **kwargs ), ): audio.save() @@ -2644,7 +2644,7 @@ def test_oa_indexing_and_tasks_count(self) -> None: with mock.patch( "cl.lib.es_signal_processor.update_es_document.si", side_effect=lambda *args, **kwargs: self.count_task_calls( - update_es_document, *args, **kwargs + update_es_document, True, *args, **kwargs ), ): audio.case_name = "Bank vs America" @@ -2657,7 +2657,7 @@ def test_oa_indexing_and_tasks_count(self) -> None: with mock.patch( "cl.lib.es_signal_processor.update_es_document.si", side_effect=lambda *args, **kwargs: self.count_task_calls( - update_es_document, *args, **kwargs + update_es_document, True, *args, **kwargs ), ): audio.docket = self.docket_2 @@ -2677,7 +2677,7 @@ def test_oa_indexing_and_tasks_count(self) -> None: with mock.patch( "cl.lib.es_signal_processor.update_es_document.si", side_effect=lambda *args, **kwargs: self.count_task_calls( - update_es_document, *args, **kwargs + update_es_document, True, *args, **kwargs ), ): audio.case_name = "Lorem Ipsum" diff --git a/cl/search/tests/tests_es_parenthetical.py b/cl/search/tests/tests_es_parenthetical.py index 894947314f..95e97a9f8d 100644 --- a/cl/search/tests/tests_es_parenthetical.py +++ b/cl/search/tests/tests_es_parenthetical.py @@ -728,7 +728,7 @@ def test_parenthetical_indexing_and_tasks_count(self) -> None: with mock.patch( "cl.lib.es_signal_processor.es_save_document.si", side_effect=lambda *args, **kwargs: self.count_task_calls( - es_save_document, *args, **kwargs + es_save_document, True, *args, **kwargs ), ): cluster_1 = OpinionClusterFactory( @@ -767,7 +767,7 @@ def test_parenthetical_indexing_and_tasks_count(self) -> None: with mock.patch( "cl.lib.es_signal_processor.update_es_document.si", side_effect=lambda *args, **kwargs: self.count_task_calls( - update_es_document, *args, **kwargs + update_es_document, True, *args, **kwargs ), ): pg_test.save() @@ -778,7 +778,7 @@ def test_parenthetical_indexing_and_tasks_count(self) -> None: with mock.patch( "cl.lib.es_signal_processor.update_es_document.si", side_effect=lambda *args, **kwargs: self.count_task_calls( - update_es_document, *args, **kwargs + update_es_document, True, *args, **kwargs ), ): p6 = ParentheticalFactory( @@ -809,7 +809,7 @@ def test_parenthetical_indexing_and_tasks_count(self) -> None: with mock.patch( "cl.lib.es_signal_processor.update_es_document.si", side_effect=lambda *args, **kwargs: self.count_task_calls( - update_es_document, *args, **kwargs + update_es_document, True, *args, **kwargs ), ): pg_test.opinion = o diff --git a/cl/search/tests/tests_es_person.py b/cl/search/tests/tests_es_person.py index 5291be3537..d15ff1f685 100644 --- a/cl/search/tests/tests_es_person.py +++ b/cl/search/tests/tests_es_person.py @@ -2664,7 +2664,7 @@ def test_person_indexing_and_tasks_count(self) -> None: with mock.patch( "cl.lib.es_signal_processor.es_save_document.si", side_effect=lambda *args, **kwargs: self.count_task_calls( - es_save_document, *args, **kwargs + es_save_document, True, *args, **kwargs ), ): person = PersonFactory.create( @@ -2685,7 +2685,7 @@ def test_person_indexing_and_tasks_count(self) -> None: with mock.patch( "cl.lib.es_signal_processor.es_save_document.si", side_effect=lambda *args, **kwargs: self.count_task_calls( - es_save_document, *args, **kwargs + es_save_document, True, *args, **kwargs ), ): position = PositionFactory.create( @@ -2713,7 +2713,7 @@ def test_person_indexing_and_tasks_count(self) -> None: with mock.patch( "cl.lib.es_signal_processor.update_es_document.si", side_effect=lambda *args, **kwargs: self.count_task_calls( - update_es_document, *args, **kwargs + update_es_document, True, *args, **kwargs ), ): person.save() @@ -2724,7 +2724,7 @@ def test_person_indexing_and_tasks_count(self) -> None: with mock.patch( "cl.lib.es_signal_processor.update_es_document.si", side_effect=lambda *args, **kwargs: self.count_task_calls( - update_es_document, *args, **kwargs + update_es_document, True, *args, **kwargs ), ): position.save() @@ -2735,7 +2735,7 @@ def test_person_indexing_and_tasks_count(self) -> None: with mock.patch( "cl.lib.es_signal_processor.update_es_document.si", side_effect=lambda *args, **kwargs: self.count_task_calls( - update_es_document, *args, **kwargs + update_es_document, True, *args, **kwargs ), ): person.name_first = "Barack" @@ -2759,7 +2759,7 @@ def test_person_indexing_and_tasks_count(self) -> None: with mock.patch( "cl.lib.es_signal_processor.update_es_document.si", side_effect=lambda *args, **kwargs: self.count_task_calls( - update_es_document, *args, **kwargs + update_es_document, True, *args, **kwargs ), ): person.religion = "pr" @@ -2774,7 +2774,7 @@ def test_person_indexing_and_tasks_count(self) -> None: with mock.patch( "cl.lib.es_signal_processor.update_es_document.si", side_effect=lambda *args, **kwargs: self.count_task_calls( - update_es_document, *args, **kwargs + update_es_document, True, *args, **kwargs ), ): position.nomination_process = "state_senate" @@ -2789,7 +2789,7 @@ def test_person_indexing_and_tasks_count(self) -> None: with mock.patch( "cl.lib.es_signal_processor.update_es_document.si", side_effect=lambda *args, **kwargs: self.count_task_calls( - update_es_document, *args, **kwargs + update_es_document, True, *args, **kwargs ), ): position.predecessor = self.person_2 diff --git a/cl/search/tests/tests_es_recap.py b/cl/search/tests/tests_es_recap.py index 9b6236468d..82a266ca7d 100644 --- a/cl/search/tests/tests_es_recap.py +++ b/cl/search/tests/tests_es_recap.py @@ -6435,7 +6435,7 @@ def test_docket_indexing_and_tasks_count(self) -> None: with mock.patch( "cl.lib.es_signal_processor.es_save_document.si", side_effect=lambda *args, **kwargs: self.count_task_calls( - es_save_document, *args, **kwargs + es_save_document, True, *args, **kwargs ), ): non_recap_docket = DocketFactory( @@ -6454,7 +6454,7 @@ def test_docket_indexing_and_tasks_count(self) -> None: with mock.patch( "cl.lib.es_signal_processor.update_es_document.si", side_effect=lambda *args, **kwargs: self.count_task_calls( - update_es_document, *args, **kwargs + update_es_document, True, *args, **kwargs ), ): non_recap_docket.source = Docket.HARVARD @@ -6467,7 +6467,7 @@ def test_docket_indexing_and_tasks_count(self) -> None: with mock.patch( "cl.lib.es_signal_processor.update_es_document.si", side_effect=lambda *args, **kwargs: self.count_task_calls( - update_es_document, *args, **kwargs + update_es_document, True, *args, **kwargs ), ): non_recap_docket.source = Docket.RECAP_AND_IDB_AND_HARVARD @@ -6481,7 +6481,7 @@ def test_docket_indexing_and_tasks_count(self) -> None: with mock.patch( "cl.lib.es_signal_processor.es_save_document.si", side_effect=lambda *args, **kwargs: self.count_task_calls( - es_save_document, *args, **kwargs + es_save_document, True, *args, **kwargs ), ): docket = DocketFactory( @@ -6500,7 +6500,7 @@ def test_docket_indexing_and_tasks_count(self) -> None: with mock.patch( "cl.lib.es_signal_processor.update_es_document.si", side_effect=lambda *args, **kwargs: self.count_task_calls( - update_es_document, *args, **kwargs + update_es_document, True, *args, **kwargs ), ): docket_2 = DocketFactory( @@ -6518,7 +6518,7 @@ def test_docket_indexing_and_tasks_count(self) -> None: with mock.patch( "cl.lib.es_signal_processor.update_es_document.si", side_effect=lambda *args, **kwargs: self.count_task_calls( - update_es_document, *args, **kwargs + update_es_document, True, *args, **kwargs ), ): docket.save() @@ -6529,7 +6529,7 @@ def test_docket_indexing_and_tasks_count(self) -> None: with mock.patch( "cl.lib.es_signal_processor.es_save_document.si", side_effect=lambda *args, **kwargs: self.count_task_calls( - es_save_document, *args, **kwargs + es_save_document, True, *args, **kwargs ), ): docket.save() @@ -6541,7 +6541,7 @@ def test_docket_indexing_and_tasks_count(self) -> None: with mock.patch( "cl.lib.es_signal_processor.update_es_document.si", side_effect=lambda *args, **kwargs: self.count_task_calls( - update_es_document, *args, **kwargs + update_es_document, True, *args, **kwargs ), ): docket.blocked = True @@ -6554,7 +6554,7 @@ def test_docket_indexing_and_tasks_count(self) -> None: with mock.patch( "cl.lib.es_signal_processor.update_es_document.si", side_effect=lambda *args, **kwargs: self.count_task_calls( - update_es_document, *args, **kwargs + update_es_document, True, *args, **kwargs ), ): docket.docket_number = "21-43434" @@ -6574,7 +6574,7 @@ def test_docket_indexing_and_tasks_count(self) -> None: with mock.patch( "cl.lib.es_signal_processor.update_es_document.si", side_effect=lambda *args, **kwargs: self.count_task_calls( - update_es_document, *args, **kwargs + update_es_document, True, *args, **kwargs ), ): docket.docket_number = "21-43435" @@ -6606,7 +6606,7 @@ def test_recap_document_indexing_and_tasks_count(self) -> None: with mock.patch( "cl.lib.es_signal_processor.es_save_document.si", side_effect=lambda *args, **kwargs: self.count_task_calls( - es_save_document, *args, **kwargs + es_save_document, True, *args, **kwargs ), ): de_1 = DocketEntryWithParentsFactory( @@ -6631,7 +6631,7 @@ def test_recap_document_indexing_and_tasks_count(self) -> None: with mock.patch( "cl.lib.es_signal_processor.update_es_document.si", side_effect=lambda *args, **kwargs: self.count_task_calls( - update_es_document, *args, **kwargs + update_es_document, True, *args, **kwargs ), ): de_2 = DocketEntryWithParentsFactory( @@ -6648,7 +6648,7 @@ def test_recap_document_indexing_and_tasks_count(self) -> None: with mock.patch( "cl.lib.es_signal_processor.update_es_document.si", side_effect=lambda *args, **kwargs: self.count_task_calls( - update_es_document, *args, **kwargs + update_es_document, True, *args, **kwargs ), ): rd_1.save() @@ -6659,7 +6659,7 @@ def test_recap_document_indexing_and_tasks_count(self) -> None: with mock.patch( "cl.lib.es_signal_processor.update_es_document.si", side_effect=lambda *args, **kwargs: self.count_task_calls( - update_es_document, *args, **kwargs + update_es_document, True, *args, **kwargs ), ): rd_1.is_sealed = True @@ -6672,7 +6672,7 @@ def test_recap_document_indexing_and_tasks_count(self) -> None: with mock.patch( "cl.lib.es_signal_processor.update_es_document.si", side_effect=lambda *args, **kwargs: self.count_task_calls( - update_es_document, *args, **kwargs + update_es_document, True, *args, **kwargs ), ): rd_1.description = "Lorem Ipsum" @@ -6688,7 +6688,7 @@ def test_recap_document_indexing_and_tasks_count(self) -> None: with mock.patch( "cl.lib.es_signal_processor.es_save_document.si", side_effect=lambda *args, **kwargs: self.count_task_calls( - es_save_document, *args, **kwargs + es_save_document, True, *args, **kwargs ), ): rd_1.page_count = 6 @@ -6711,7 +6711,7 @@ def test_recap_document_indexing_and_tasks_count(self) -> None: with mock.patch( "cl.lib.es_signal_processor.update_es_document.si", side_effect=lambda *args, **kwargs: self.count_task_calls( - update_es_document, *args, **kwargs + update_es_document, True, *args, **kwargs ), ): rd_1.docket_entry = de_2 @@ -6739,7 +6739,7 @@ def test_recap_document_indexing_and_tasks_count(self) -> None: with mock.patch( "cl.lib.es_signal_processor.update_es_document.si", side_effect=lambda *args, **kwargs: self.count_task_calls( - update_es_document, *args, **kwargs + update_es_document, True, *args, **kwargs ), ): rd_1.pacer_doc_id = "99999999" @@ -6756,7 +6756,7 @@ def test_recap_document_indexing_and_tasks_count(self) -> None: with mock.patch( "cl.lib.es_signal_processor.update_es_document.si", side_effect=lambda *args, **kwargs: self.count_task_calls( - update_es_document, *args, **kwargs + update_es_document, True, *args, **kwargs ), ): OpinionsCitedByRECAPDocument.objects.bulk_create( @@ -6783,7 +6783,7 @@ def test_recap_document_indexing_and_tasks_count(self) -> None: with mock.patch( "cl.lib.es_signal_processor.update_es_document.si", side_effect=lambda *args, **kwargs: self.count_task_calls( - update_es_document, *args, **kwargs + update_es_document, True, *args, **kwargs ), ): OpinionsCitedByRECAPDocument.objects.filter( @@ -6800,7 +6800,7 @@ def test_recap_document_indexing_and_tasks_count(self) -> None: with mock.patch( "cl.lib.es_signal_processor.update_es_document.si", side_effect=lambda *args, **kwargs: self.count_task_calls( - update_es_document, *args, **kwargs + update_es_document, True, *args, **kwargs ), ): o_cited = OpinionsCitedByRECAPDocument( diff --git a/cl/tests/cases.py b/cl/tests/cases.py index d848bd9347..5b0c03e374 100644 --- a/cl/tests/cases.py +++ b/cl/tests/cases.py @@ -227,13 +227,14 @@ class CountESTasksTestCase(SimpleTestCase): def setUp(self): self.task_call_count = 0 - def count_task_calls(self, task, *args, **kwargs) -> None: + def count_task_calls( + self, task, immutable_signature, *args, **kwargs + ) -> None: """Wraps the task to count its calls and assert the expected count.""" # Increment the call count self.task_call_count += 1 - # Call the task - if task.__name__ in ["es_save_document", "update_es_document"]: + if immutable_signature: return task.s(*args, **kwargs) else: task.apply_async(args=args, kwargs=kwargs)