From a555bb59c311f894333a3de35e383d04ea05f81d Mon Sep 17 00:00:00 2001 From: nkirby-va Date: Thu, 21 Sep 2023 10:46:28 -0400 Subject: [PATCH 01/32] get something working - refactor later --- .../populate_end_product_sync_queue.sql | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 db/scripts/populate_end_product_sync_queue.sql diff --git a/db/scripts/populate_end_product_sync_queue.sql b/db/scripts/populate_end_product_sync_queue.sql new file mode 100644 index 00000000000..b67df2180b9 --- /dev/null +++ b/db/scripts/populate_end_product_sync_queue.sql @@ -0,0 +1,37 @@ +drop trigger if exists update_claim_status_trigger on vbms_ext_claim; + +truncate table priority_end_product_sync_queue; + +create or replace function public.update_claim_status_trigger_function() +returns trigger as $$ + begin + if (NEW."LEVEL_STATUS_CODE" = 'CLR' OR NEW."LEVEL_STATUS_CODE" = 'CAN') + and (NEW."EP_CODE" LIKE '04%' + OR NEW."EP_CODE" LIKE '03%' + OR NEW."EP_CODE" LIKE '93%' + OR NEW."EP_CODE" LIKE '68%') then + if exists ( + select 1 + from end_product_establishments + where (reference_id = cast(NEW."CLAIM_ID" as varchar) + and (synced_status is null or synced_status <> NEW."LEVEL_STATUS_CODE")) + ) then + if not exists ( + select 1 + from priority_end_product_sync_queue + where end_product_establishment_id = (select id from end_product_establishments where reference_id = cast(NEW."CLAIM_ID" as varchar)) + ) then + insert into priority_end_product_sync_queue (created_at, end_product_establishment_id, updated_at) + values (now(), (select id from end_product_establishments where reference_id = cast(NEW."CLAIM_ID" as varchar)), now()); + end if; + end if; + end if; + return null; + end; +$$ +language plpgsql; + +create trigger update_claim_status_trigger +after update or insert on vbms_ext_claim +for each row +execute procedure public.update_claim_status_trigger_function(); From fdf817fb4df8ae13697d519d798dd79d38531c80 Mon Sep 17 00:00:00 2001 From: nkirby-va Date: Thu, 21 Sep 2023 15:14:16 -0400 Subject: [PATCH 02/32] partial refactor --- .../populate_end_product_sync_queue.sql | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/db/scripts/populate_end_product_sync_queue.sql b/db/scripts/populate_end_product_sync_queue.sql index b67df2180b9..a34671c8a68 100644 --- a/db/scripts/populate_end_product_sync_queue.sql +++ b/db/scripts/populate_end_product_sync_queue.sql @@ -1,28 +1,35 @@ +-- TODO: remove line 2 and 4 drop trigger if exists update_claim_status_trigger on vbms_ext_claim; truncate table priority_end_product_sync_queue; create or replace function public.update_claim_status_trigger_function() returns trigger as $$ + declare + string_claim_id varchar(25); + epe_id integer; begin + string_claim_id := cast(NEW."CLAIM_ID" as varchar); + + select id into epe_id + from end_product_establishments + where (reference_id = string_claim_id + and (synced_status is null or synced_status <> NEW."LEVEL_STATUS_CODE")); + if (NEW."LEVEL_STATUS_CODE" = 'CLR' OR NEW."LEVEL_STATUS_CODE" = 'CAN') and (NEW."EP_CODE" LIKE '04%' OR NEW."EP_CODE" LIKE '03%' OR NEW."EP_CODE" LIKE '93%' OR NEW."EP_CODE" LIKE '68%') then - if exists ( - select 1 - from end_product_establishments - where (reference_id = cast(NEW."CLAIM_ID" as varchar) - and (synced_status is null or synced_status <> NEW."LEVEL_STATUS_CODE")) - ) then + if epe_id > 0 + then if not exists ( select 1 from priority_end_product_sync_queue - where end_product_establishment_id = (select id from end_product_establishments where reference_id = cast(NEW."CLAIM_ID" as varchar)) + where end_product_establishment_id = (select id from end_product_establishments where reference_id = string_claim_id) -- can this sub query be replaced with 'epe_id'? ) then insert into priority_end_product_sync_queue (created_at, end_product_establishment_id, updated_at) - values (now(), (select id from end_product_establishments where reference_id = cast(NEW."CLAIM_ID" as varchar)), now()); + values (now(), epe_id, now()); end if; end if; end if; From 7abdf77c3790205718ef7a7007c0c39e7982e962 Mon Sep 17 00:00:00 2001 From: nkirby-va Date: Fri, 22 Sep 2023 10:20:14 -0400 Subject: [PATCH 03/32] refactor --- db/scripts/populate_end_product_sync_queue.sql | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/db/scripts/populate_end_product_sync_queue.sql b/db/scripts/populate_end_product_sync_queue.sql index a34671c8a68..bc9867ae23f 100644 --- a/db/scripts/populate_end_product_sync_queue.sql +++ b/db/scripts/populate_end_product_sync_queue.sql @@ -9,18 +9,19 @@ returns trigger as $$ string_claim_id varchar(25); epe_id integer; begin - string_claim_id := cast(NEW."CLAIM_ID" as varchar); - - select id into epe_id - from end_product_establishments - where (reference_id = string_claim_id - and (synced_status is null or synced_status <> NEW."LEVEL_STATUS_CODE")); - if (NEW."LEVEL_STATUS_CODE" = 'CLR' OR NEW."LEVEL_STATUS_CODE" = 'CAN') and (NEW."EP_CODE" LIKE '04%' OR NEW."EP_CODE" LIKE '03%' OR NEW."EP_CODE" LIKE '93%' OR NEW."EP_CODE" LIKE '68%') then + + string_claim_id := cast(NEW."CLAIM_ID" as varchar); + + select id into epe_id + from end_product_establishments + where (reference_id = string_claim_id + and (synced_status is null or synced_status <> NEW."LEVEL_STATUS_CODE")); + if epe_id > 0 then if not exists ( From 1eea17777f3d1420e05c1850e3ad9d6ace91059a Mon Sep 17 00:00:00 2001 From: nkirby-va Date: Mon, 25 Sep 2023 10:42:17 -0400 Subject: [PATCH 04/32] refactor, add rspec tests --- Makefile.example | 11 ++ db/scripts/add_pepsq_populate_trigger.rb | 51 ++++++ ...eue.sql => add_pepsq_populate_trigger.sql} | 0 db/scripts/drop_pepsq_populate_trigger.rb | 11 ++ db/scripts/drop_pepsq_populate_trigger.sql | 2 + .../populate_end_product_sync_queue_spec.rb | 149 ++++++++++++++++++ 6 files changed, 224 insertions(+) create mode 100644 db/scripts/add_pepsq_populate_trigger.rb rename db/scripts/{populate_end_product_sync_queue.sql => add_pepsq_populate_trigger.sql} (100%) create mode 100644 db/scripts/drop_pepsq_populate_trigger.rb create mode 100644 db/scripts/drop_pepsq_populate_trigger.sql create mode 100644 spec/sql/triggers/populate_end_product_sync_queue_spec.rb diff --git a/Makefile.example b/Makefile.example index 3d187a5428f..92a25dbd5f3 100644 --- a/Makefile.example +++ b/Makefile.example @@ -177,9 +177,11 @@ audit-remove: ## Remove caseflow_audit schema, tables and triggers in postgres # These tables should not be included as part of migrations external-db-create: ## Creates external_vbms_ext_claim table bundle exec rails r db/scripts/external/create_vbms_ext_claim_table.rb + make add-populate-pepsq-trigger external-db-remove: ## Remove external_vbms_ext_claim table bundle exec rails r db/scripts/external/remove_vbms_ext_claim_table.rb + make remove-populate-pepsq-trigger # This needs to be manually run after make reset/migrate in order for local tests involving external tables to pass. # Otherwise the caseflow_certification_test schema will not create these tables and will error out. @@ -194,6 +196,15 @@ remove-vbms-ext-claim-seeds: ## Drops audit tables, removes all PriorityEndProdu reseed-vbms-ext-claim: remove-vbms-ext-claim-seeds seed-vbms-ext-claim ## Re-seeds database with records created from seed-vbms-ext-claim +# Add trigger to vbms_ext_claim to populate pepsq table +add-populate-pepsq-trigger: + bundle exec rails r db/scripts/add_pepsq_populate_trigger.rb + +# Remove populate pepsq trigger from vbms_ext_claim table +remove-populate-pepsq-trigger: + bundle exec rails r db/scripts/drop_pepsq_populate_trigger.rb + + c: ## Start rails console bundle exec rails console diff --git a/db/scripts/add_pepsq_populate_trigger.rb b/db/scripts/add_pepsq_populate_trigger.rb new file mode 100644 index 00000000000..cc87d9be17c --- /dev/null +++ b/db/scripts/add_pepsq_populate_trigger.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +require "pg" + +conn = CaseflowRecord.connection +conn.execute(" + drop trigger if exists update_claim_status_trigger on vbms_ext_claim; + + create or replace function public.update_claim_status_trigger_function() + returns trigger as $$ + declare + string_claim_id varchar(25); + epe_id integer; + begin + if (NEW.\"LEVEL_STATUS_CODE\" = 'CLR' OR NEW.\"LEVEL_STATUS_CODE\" = 'CAN') + and (NEW.\"EP_CODE\" LIKE '04%' + OR NEW.\"EP_CODE\" LIKE '03%' + OR NEW.\"EP_CODE\" LIKE '93%' + OR NEW.\"EP_CODE\" LIKE '68%') then + + string_claim_id := cast(NEW.\"CLAIM_ID\" as varchar); + + select id into epe_id + from end_product_establishments + where (reference_id = string_claim_id + and (synced_status is null or synced_status <> NEW.\"LEVEL_STATUS_CODE\")); + + if epe_id > 0 + then + if not exists ( + select 1 + from priority_end_product_sync_queue + where end_product_establishment_id = (select id from end_product_establishments where reference_id = string_claim_id) -- can this sub query be replaced with 'epe_id'? + ) then + insert into priority_end_product_sync_queue (created_at, end_product_establishment_id, updated_at) + values (now(), epe_id, now()); + end if; + end if; + end if; + return null; + end; + $$ + language plpgsql; + + create trigger update_claim_status_trigger + after update or insert on vbms_ext_claim + for each row + execute procedure public.update_claim_status_trigger_function(); + ") + +conn.close diff --git a/db/scripts/populate_end_product_sync_queue.sql b/db/scripts/add_pepsq_populate_trigger.sql similarity index 100% rename from db/scripts/populate_end_product_sync_queue.sql rename to db/scripts/add_pepsq_populate_trigger.sql diff --git a/db/scripts/drop_pepsq_populate_trigger.rb b/db/scripts/drop_pepsq_populate_trigger.rb new file mode 100644 index 00000000000..8173564ce36 --- /dev/null +++ b/db/scripts/drop_pepsq_populate_trigger.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +require "pg" + +conn = CaseflowRecord.connection +conn.execute(" + drop trigger if exists update_claim_status_trigger on vbms_ext_claim; + drop function if exists public.update_claim_status_trigger_function(); + ") + +conn.close diff --git a/db/scripts/drop_pepsq_populate_trigger.sql b/db/scripts/drop_pepsq_populate_trigger.sql new file mode 100644 index 00000000000..97b50402069 --- /dev/null +++ b/db/scripts/drop_pepsq_populate_trigger.sql @@ -0,0 +1,2 @@ +drop trigger if exists update_claim_status_trigger on vbms_ext_claim; +drop function if exists public.update_claim_status_trigger_function(); diff --git a/spec/sql/triggers/populate_end_product_sync_queue_spec.rb b/spec/sql/triggers/populate_end_product_sync_queue_spec.rb new file mode 100644 index 00000000000..dc3bfcf3878 --- /dev/null +++ b/spec/sql/triggers/populate_end_product_sync_queue_spec.rb @@ -0,0 +1,149 @@ +# frozen_string_literal: true + +describe "vbms_ext_claim trigger to populate end_product_sync_que table", :postgres do + context "when the trigger is added to the vbms_ext_claim table before the creation new records" do + before(:all) do + system("make add-populate-pepsq-trigger") + end + before do + PriorityEndProductSyncQueue.delete_all + end + after(:all) do + system("remove add-populate-pepsq-trigger") + end + + context "we only log inserted vbms_ext_claims" do + let(:logged_epe1) { create(:end_product_establishment, :active, reference_id: 300_000) } + let(:logged_ext_claim1) { create(:vbms_ext_claim, :cleared, :slc, id: 300_000) } + + it "that are cleared, have a \"04%\" EP_CODE, + different sync status, and are not in pepsq table" do + logged_epe1 + logged_ext_claim1 + expect(PriorityEndProductSyncQueue.count).to eq 1 + expect(PriorityEndProductSyncQueue.first.end_product_establishment_id).to eq logged_epe1.id + end + + let(:logged_epe2) { create(:end_product_establishment, synced_status: nil, reference_id: 300_000) } + let(:logged_ext_claim2) { create(:vbms_ext_claim, :canceled, :hlr, id: 300_000) } + + it "that are cancelled, have a \"03%\" EP_CODE, + with out sync status, not in pepsq table " do + logged_epe2 + logged_ext_claim2 + expect(PriorityEndProductSyncQueue.count).to eq 1 + expect(PriorityEndProductSyncQueue.first.end_product_establishment_id).to eq logged_epe2.id + end + end + + context "we do not log inserted (on creation) vbms_ext_claims" do + let(:logged_epe3) { create(:end_product_establishment, synced_status: nil, reference_id: 300_000) } + let(:logged_ext_claim3) { create(:vbms_ext_claim, :rdc, :hlr, id: 300_000) } + + it "that are rdc, have a \"03%\" EP_CODE, + with out sync status, not in pepsq table " do + logged_epe3 + logged_ext_claim3 + expect(PriorityEndProductSyncQueue.count).to eq 0 + end + + let(:logged_epe4) { create(:end_product_establishment, synced_status: nil, reference_id: 300_000) } + let(:logged_ext_claim4) { create(:vbms_ext_claim, :canceled, EP_CODE: "999", id: 300_000) } + + it "that are canceled, have a wrong EP_CODE, + with a nil sync status, not in pepsq table " do + logged_epe4 + logged_ext_claim4 + expect(PriorityEndProductSyncQueue.count).to eq 0 + end + + let(:logged_epe5) { create(:end_product_establishment, synced_status: nil, reference_id: 300_000) } + let(:logged_ext_claim5) { create(:vbms_ext_claim, :canceled, :slc, id: 300_000) } + + it "that are canceled, have a wrong EP_CODE, + with a nil sync status, already in the pepsq table " do + logged_epe5 + PriorityEndProductSyncQueue.create(end_product_establishment_id: logged_epe5.id) + logged_ext_claim5 + expect(PriorityEndProductSyncQueue.count).to eq 1 + end + end + end + + context "when the trigger is added and records already exist in the vbms_ext_claim table" do + before(:all) do + @logged_epe = create(:end_product_establishment, :active, reference_id: 300_000) + @logged_ext_claim = create(:vbms_ext_claim, :rdc, :slc, id: 300_000) + system("make add-populate-pepsq-trigger") + end + before do + PriorityEndProductSyncQueue.delete_all + end + after(:all) do + EndProductEstablishment.delete(@logged_epe) + VbmsExtClaim.delete(@logged_ext_claim) + system("remove add-populate-pepsq-trigger") + end + + context "we only log updated vbms_ext_claims" do + it "that are cleared, have a \"04%\" EP_CODE, + different sync status, and are not in pepsq table" do + @logged_ext_claim.update(LEVEL_STATUS_CODE: "CLR") + expect(PriorityEndProductSyncQueue.count).to eq 1 + expect(PriorityEndProductSyncQueue.first.end_product_establishment_id).to eq @logged_epe.id + end + + it "that are cancelled, have a \"03%\" EP_CODE, + with out sync status, not in pepsq table " do + @logged_epe.update(synced_status: nil) + @logged_ext_claim.update(LEVEL_STATUS_CODE: "CAN", EP_CODE: "030") + expect(PriorityEndProductSyncQueue.count).to eq 1 + expect(PriorityEndProductSyncQueue.first.end_product_establishment_id).to eq @logged_epe.id + end + end + + context "we do not log updated vbms_ext_claims" do + it "that are rdc, have a \"03%\" EP_CODE, + with out sync status, not in pepsq table " do + @logged_ext_claim.update(LEVEL_STATUS_CODE: "RDC", EP_CODE: "030") + expect(PriorityEndProductSyncQueue.count).to eq 0 + end + + it "that are canceled, have a wrong EP_CODE, + with a nil sync status, not in pepsq table " do + @logged_epe.update(synced_status: nil) + @logged_ext_claim.update(LEVEL_STATUS_CODE: "CAN", EP_CODE: "999") + expect(PriorityEndProductSyncQueue.count).to eq 0 + end + + it "that are canceled, have a wrong EP_CODE, + with a nil sync status, already in the pepsq table " do + PriorityEndProductSyncQueue.create(end_product_establishment_id: @logged_epe.id) + expect(PriorityEndProductSyncQueue.count).to eq 1 + end + end + end + + context "when the trigger is removed from the vbms_ext_claim table" do + before(:all) do + system("make remove-populate-pepsq-trigger") + end + + let(:logged_epe) { create(:end_product_establishment, :active, reference_id: 300_000) } + let(:logged_ext_claim) { create(:vbms_ext_claim, :cleared, :slc, id: 300_000) } + + it "no records should be inserted into pepsq on creation of new vbms_ext_claim records" do + logged_epe + logged_ext_claim + expect(PriorityEndProductSyncQueue.count).to eq 0 + end + + it "no records should be inserted into pepsq on update of existing vbms_ext_claim records" do + logged_epe + logged_ext_claim + logged_epe.update(synced_status: nil) + logged_ext_claim.update(LEVEL_STATUS_CODE: "CAN", EP_CODE: "030") + expect(PriorityEndProductSyncQueue.count).to eq 0 + end + end +end From 684a29a77f8a989aac2997799c57859243f01949 Mon Sep 17 00:00:00 2001 From: nkirby-va Date: Mon, 25 Sep 2023 10:44:33 -0400 Subject: [PATCH 05/32] add trigger to external external-db-create-test --- Makefile.example | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile.example b/Makefile.example index 92a25dbd5f3..f40f648393b 100644 --- a/Makefile.example +++ b/Makefile.example @@ -187,6 +187,7 @@ external-db-remove: ## Remove external_vbms_ext_claim table # Otherwise the caseflow_certification_test schema will not create these tables and will error out. external-db-create-test: ## Creates table in caseflow_certification_test DB for local RSPEC tests bundle exec rails r -e test db/scripts/external/create_vbms_ext_claim_table.rb + make add-populate-pepsq-trigger remove-vbms-ext-claim-seeds: ## Drops audit tables, removes all PriorityEndProductSyncQueue, BatchProcess, and seed-vbms-ext-claim records, then rebuilds audit tables make audit-remove From ac75094a8b96ff2d07cd4ff0ec0403f1761a2f6c Mon Sep 17 00:00:00 2001 From: nkirby-va Date: Mon, 25 Sep 2023 10:57:36 -0400 Subject: [PATCH 06/32] remove env variables for job --- config/environments/demo.rb | 5 ----- config/environments/development.rb | 5 ----- config/environments/test.rb | 5 ----- 3 files changed, 15 deletions(-) diff --git a/config/environments/demo.rb b/config/environments/demo.rb index 1815b62083b..2215958bce8 100644 --- a/config/environments/demo.rb +++ b/config/environments/demo.rb @@ -93,11 +93,6 @@ ENV["BATCH_PROCESS_ERROR_DELAY"] ||= "12" # In number of hours ENV["BATCH_PROCESS_MAX_ERRORS_BEFORE_STUCK"] ||= "3" # When record errors for X time, it's declared stuck - # Populate End Product Sync Queue ENVs - ENV["END_PRODUCT_QUEUE_JOB_DURATION"] ||= "1" # Number of hours the job will run for - ENV["END_PRODUCT_QUEUE_SLEEP_DURATION"] ||= "5" # Number of seconds between loop iterations - ENV["END_PRODUCT_QUEUE_BATCH_LIMIT"] ||= "500" # Max number of records in a batch - # Setup S3 config.s3_enabled = ENV["AWS_BUCKET_NAME"].present? config.s3_bucket_name = ENV["AWS_BUCKET_NAME"] diff --git a/config/environments/development.rb b/config/environments/development.rb index 9822cd7b692..6c2dfb02f7f 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -100,11 +100,6 @@ # Quarterly Notifications Batch Sizes ENV["QUARTERLY_NOTIFICATIONS_JOB_BATCH_SIZE"] ||= "1000" - # Populate End Product Sync Queue ENVs - ENV["END_PRODUCT_QUEUE_JOB_DURATION"] ||= "50" # Number of minutes the job will run for - ENV["END_PRODUCT_QUEUE_SLEEP_DURATION"] ||= "5" # Number of seconds between loop iterations - ENV["END_PRODUCT_QUEUE_BATCH_LIMIT"] ||= "500" # Max number of records in a batch - # Travel Board Sync Batch Size ENV["TRAVEL_BOARD_HEARING_SYNC_BATCH_LIMIT"] ||= "250" diff --git a/config/environments/test.rb b/config/environments/test.rb index fff81c6c7a2..426c6f8fb64 100644 --- a/config/environments/test.rb +++ b/config/environments/test.rb @@ -115,11 +115,6 @@ # Quarterly Notifications Batch Sizes ENV["QUARTERLY_NOTIFICATIONS_JOB_BATCH_SIZE"] ||= "1000" - # Populate End Product Sync Queue ENVs - ENV["END_PRODUCT_QUEUE_JOB_DURATION"] ||= "50" # Number of minutes the job will run for - ENV["END_PRODUCT_QUEUE_SLEEP_DURATION"] ||= "0" # Number of seconds between loop iterations - ENV["END_PRODUCT_QUEUE_BATCH_LIMIT"] ||= "250" # Max number of records in a batch - # Travel Board Sync Batch Size ENV["TRAVEL_BOARD_HEARING_SYNC_BATCH_LIMIT"] ||= "250" From d8854ff840430769a96dbd76fa07cb563b702dd4 Mon Sep 17 00:00:00 2001 From: nkirby-va Date: Mon, 25 Sep 2023 11:23:38 -0400 Subject: [PATCH 07/32] modify logic --- db/scripts/add_pepsq_populate_trigger.rb | 6 +++--- db/scripts/add_pepsq_populate_trigger.sql | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/db/scripts/add_pepsq_populate_trigger.rb b/db/scripts/add_pepsq_populate_trigger.rb index cc87d9be17c..fced0251748 100644 --- a/db/scripts/add_pepsq_populate_trigger.rb +++ b/db/scripts/add_pepsq_populate_trigger.rb @@ -12,11 +12,11 @@ string_claim_id varchar(25); epe_id integer; begin - if (NEW.\"LEVEL_STATUS_CODE\" = 'CLR' OR NEW.\"LEVEL_STATUS_CODE\" = 'CAN') - and (NEW.\"EP_CODE\" LIKE '04%' + if (NEW.\"EP_CODE\" LIKE '04%' OR NEW.\"EP_CODE\" LIKE '03%' OR NEW.\"EP_CODE\" LIKE '93%' - OR NEW.\"EP_CODE\" LIKE '68%') then + OR NEW.\"EP_CODE\" LIKE '68%') + and (NEW.\"LEVEL_STATUS_CODE\" = 'CLR' OR NEW.\"LEVEL_STATUS_CODE\" = 'CAN') then string_claim_id := cast(NEW.\"CLAIM_ID\" as varchar); diff --git a/db/scripts/add_pepsq_populate_trigger.sql b/db/scripts/add_pepsq_populate_trigger.sql index bc9867ae23f..7ec71f2a6be 100644 --- a/db/scripts/add_pepsq_populate_trigger.sql +++ b/db/scripts/add_pepsq_populate_trigger.sql @@ -9,11 +9,11 @@ returns trigger as $$ string_claim_id varchar(25); epe_id integer; begin - if (NEW."LEVEL_STATUS_CODE" = 'CLR' OR NEW."LEVEL_STATUS_CODE" = 'CAN') - and (NEW."EP_CODE" LIKE '04%' + if (NEW."EP_CODE" LIKE '04%' OR NEW."EP_CODE" LIKE '03%' OR NEW."EP_CODE" LIKE '93%' - OR NEW."EP_CODE" LIKE '68%') then + OR NEW."EP_CODE" LIKE '68%') + and (NEW."LEVEL_STATUS_CODE" = 'CLR' OR NEW."LEVEL_STATUS_CODE" = 'CAN') then string_claim_id := cast(NEW."CLAIM_ID" as varchar); From afe9ea932ee5158f6b5c9475b34fd110d0de03ea Mon Sep 17 00:00:00 2001 From: nkirby-va Date: Mon, 25 Sep 2023 11:23:57 -0400 Subject: [PATCH 08/32] add more tests for ep_codes --- .../populate_end_product_sync_queue_spec.rb | 73 ++++++++++++++----- 1 file changed, 55 insertions(+), 18 deletions(-) diff --git a/spec/sql/triggers/populate_end_product_sync_queue_spec.rb b/spec/sql/triggers/populate_end_product_sync_queue_spec.rb index dc3bfcf3878..79aca4179d1 100644 --- a/spec/sql/triggers/populate_end_product_sync_queue_spec.rb +++ b/spec/sql/triggers/populate_end_product_sync_queue_spec.rb @@ -34,37 +34,59 @@ expect(PriorityEndProductSyncQueue.count).to eq 1 expect(PriorityEndProductSyncQueue.first.end_product_establishment_id).to eq logged_epe2.id end - end - context "we do not log inserted (on creation) vbms_ext_claims" do let(:logged_epe3) { create(:end_product_establishment, synced_status: nil, reference_id: 300_000) } - let(:logged_ext_claim3) { create(:vbms_ext_claim, :rdc, :hlr, id: 300_000) } + let(:logged_ext_claim3) { create(:vbms_ext_claim, :canceled, :hlr, id: 300_000, ep_code: '930') } - it "that are rdc, have a \"03%\" EP_CODE, + it "that are cancelled, have a \"93%\" EP_CODE, with out sync status, not in pepsq table " do logged_epe3 logged_ext_claim3 - expect(PriorityEndProductSyncQueue.count).to eq 0 + expect(PriorityEndProductSyncQueue.count).to eq 1 + expect(PriorityEndProductSyncQueue.first.end_product_establishment_id).to eq logged_epe3.id end let(:logged_epe4) { create(:end_product_establishment, synced_status: nil, reference_id: 300_000) } - let(:logged_ext_claim4) { create(:vbms_ext_claim, :canceled, EP_CODE: "999", id: 300_000) } + let(:logged_ext_claim4) { create(:vbms_ext_claim, :cleared, id: 300_000, ep_code: '680') } - it "that are canceled, have a wrong EP_CODE, - with a nil sync status, not in pepsq table " do + it "that are cleared, have a \"68%\" EP_CODE, + with out sync status, not in pepsq table " do logged_epe4 logged_ext_claim4 - expect(PriorityEndProductSyncQueue.count).to eq 0 + expect(PriorityEndProductSyncQueue.count).to eq 1 + expect(PriorityEndProductSyncQueue.first.end_product_establishment_id).to eq logged_epe4.id end + end + context "we do not log inserted (on creation) vbms_ext_claims" do let(:logged_epe5) { create(:end_product_establishment, synced_status: nil, reference_id: 300_000) } - let(:logged_ext_claim5) { create(:vbms_ext_claim, :canceled, :slc, id: 300_000) } + let(:logged_ext_claim5) { create(:vbms_ext_claim, :rdc, :hlr, id: 300_000) } - it "that are canceled, have a wrong EP_CODE, - with a nil sync status, already in the pepsq table " do + it "that are rdc, have a \"03%\" EP_CODE, + with out sync status, not in pepsq table " do logged_epe5 - PriorityEndProductSyncQueue.create(end_product_establishment_id: logged_epe5.id) logged_ext_claim5 + expect(PriorityEndProductSyncQueue.count).to eq 0 + end + + let(:logged_epe6) { create(:end_product_establishment, synced_status: nil, reference_id: 300_000) } + let(:logged_ext_claim6) { create(:vbms_ext_claim, :canceled, EP_CODE: "999", id: 300_000) } + + it "that are canceled, have a wrong EP_CODE, + with a nil sync status, not in pepsq table " do + logged_epe6 + logged_ext_claim6 + expect(PriorityEndProductSyncQueue.count).to eq 0 + end + + let(:logged_epe7) { create(:end_product_establishment, synced_status: nil, reference_id: 300_000) } + let(:logged_ext_claim7) { create(:vbms_ext_claim, :canceled, :slc, id: 300_000) } + + it "that are canceled, have a wrong EP_CODE, + with a nil sync status, already in the pepsq table " do + logged_epe7 + PriorityEndProductSyncQueue.create(end_product_establishment_id: logged_epe7.id) + logged_ext_claim7 expect(PriorityEndProductSyncQueue.count).to eq 1 end end @@ -86,30 +108,45 @@ end context "we only log updated vbms_ext_claims" do - it "that are cleared, have a \"04%\" EP_CODE, + it "that are cleared, *have a \"04%\" EP_CODE, different sync status, and are not in pepsq table" do @logged_ext_claim.update(LEVEL_STATUS_CODE: "CLR") expect(PriorityEndProductSyncQueue.count).to eq 1 expect(PriorityEndProductSyncQueue.first.end_product_establishment_id).to eq @logged_epe.id end - it "that are cancelled, have a \"03%\" EP_CODE, + it "that are cancelled, *have a \"03%\" EP_CODE, with out sync status, not in pepsq table " do @logged_epe.update(synced_status: nil) @logged_ext_claim.update(LEVEL_STATUS_CODE: "CAN", EP_CODE: "030") expect(PriorityEndProductSyncQueue.count).to eq 1 expect(PriorityEndProductSyncQueue.first.end_product_establishment_id).to eq @logged_epe.id end + + it "that are cleared, *have a \"93%\" EP_CODE, + different sync status, and are not in pepsq table" do + @logged_ext_claim.update(LEVEL_STATUS_CODE: "CLR", EP_CODE: "930") + expect(PriorityEndProductSyncQueue.count).to eq 1 + expect(PriorityEndProductSyncQueue.first.end_product_establishment_id).to eq @logged_epe.id + end + + it "that are cancelled, *have a \"68%\" EP_CODE, + with out sync status, not in pepsq table " do + @logged_epe.update(synced_status: nil) + @logged_ext_claim.update(LEVEL_STATUS_CODE: "CAN", EP_CODE: "680") + expect(PriorityEndProductSyncQueue.count).to eq 1 + expect(PriorityEndProductSyncQueue.first.end_product_establishment_id).to eq @logged_epe.id + end end context "we do not log updated vbms_ext_claims" do - it "that are rdc, have a \"03%\" EP_CODE, + it "*that are rdc, have a \"03%\" EP_CODE, with out sync status, not in pepsq table " do @logged_ext_claim.update(LEVEL_STATUS_CODE: "RDC", EP_CODE: "030") expect(PriorityEndProductSyncQueue.count).to eq 0 end - it "that are canceled, have a wrong EP_CODE, + it "that are canceled, *have a wrong EP_CODE, with a nil sync status, not in pepsq table " do @logged_epe.update(synced_status: nil) @logged_ext_claim.update(LEVEL_STATUS_CODE: "CAN", EP_CODE: "999") @@ -117,7 +154,7 @@ end it "that are canceled, have a wrong EP_CODE, - with a nil sync status, already in the pepsq table " do + with a nil sync status, *already in the pepsq table " do PriorityEndProductSyncQueue.create(end_product_establishment_id: @logged_epe.id) expect(PriorityEndProductSyncQueue.count).to eq 1 end From 517b345df975a285e2c26e2ce65789c31713cdb2 Mon Sep 17 00:00:00 2001 From: nkirby-va Date: Mon, 25 Sep 2023 11:34:50 -0400 Subject: [PATCH 09/32] re-arrange wording --- .../populate_end_product_sync_queue_spec.rb | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/spec/sql/triggers/populate_end_product_sync_queue_spec.rb b/spec/sql/triggers/populate_end_product_sync_queue_spec.rb index 79aca4179d1..5f4b87a0a58 100644 --- a/spec/sql/triggers/populate_end_product_sync_queue_spec.rb +++ b/spec/sql/triggers/populate_end_product_sync_queue_spec.rb @@ -16,7 +16,7 @@ let(:logged_epe1) { create(:end_product_establishment, :active, reference_id: 300_000) } let(:logged_ext_claim1) { create(:vbms_ext_claim, :cleared, :slc, id: 300_000) } - it "that are cleared, have a \"04%\" EP_CODE, + it "that have a \"04%\" EP_CODE, that are cleared, different sync status, and are not in pepsq table" do logged_epe1 logged_ext_claim1 @@ -27,7 +27,7 @@ let(:logged_epe2) { create(:end_product_establishment, synced_status: nil, reference_id: 300_000) } let(:logged_ext_claim2) { create(:vbms_ext_claim, :canceled, :hlr, id: 300_000) } - it "that are cancelled, have a \"03%\" EP_CODE, + it "that have a \"03%\" EP_CODE, that are cancelled, with out sync status, not in pepsq table " do logged_epe2 logged_ext_claim2 @@ -36,9 +36,9 @@ end let(:logged_epe3) { create(:end_product_establishment, synced_status: nil, reference_id: 300_000) } - let(:logged_ext_claim3) { create(:vbms_ext_claim, :canceled, :hlr, id: 300_000, ep_code: '930') } + let(:logged_ext_claim3) { create(:vbms_ext_claim, :canceled, :hlr, id: 300_000, ep_code: "930") } - it "that are cancelled, have a \"93%\" EP_CODE, + it "that have a \"93%\" EP_CODE, that are cancelled, with out sync status, not in pepsq table " do logged_epe3 logged_ext_claim3 @@ -47,9 +47,9 @@ end let(:logged_epe4) { create(:end_product_establishment, synced_status: nil, reference_id: 300_000) } - let(:logged_ext_claim4) { create(:vbms_ext_claim, :cleared, id: 300_000, ep_code: '680') } + let(:logged_ext_claim4) { create(:vbms_ext_claim, :cleared, id: 300_000, ep_code: "680") } - it "that are cleared, have a \"68%\" EP_CODE, + it "that have a \"68%\" EP_CODE, that are cleared, with out sync status, not in pepsq table " do logged_epe4 logged_ext_claim4 @@ -62,7 +62,7 @@ let(:logged_epe5) { create(:end_product_establishment, synced_status: nil, reference_id: 300_000) } let(:logged_ext_claim5) { create(:vbms_ext_claim, :rdc, :hlr, id: 300_000) } - it "that are rdc, have a \"03%\" EP_CODE, + it "that have a \"03%\" EP_CODE, that are rdc, with out sync status, not in pepsq table " do logged_epe5 logged_ext_claim5 @@ -72,7 +72,7 @@ let(:logged_epe6) { create(:end_product_establishment, synced_status: nil, reference_id: 300_000) } let(:logged_ext_claim6) { create(:vbms_ext_claim, :canceled, EP_CODE: "999", id: 300_000) } - it "that are canceled, have a wrong EP_CODE, + it "that have a wrong EP_CODE, that are canceled, with a nil sync status, not in pepsq table " do logged_epe6 logged_ext_claim6 @@ -82,7 +82,7 @@ let(:logged_epe7) { create(:end_product_establishment, synced_status: nil, reference_id: 300_000) } let(:logged_ext_claim7) { create(:vbms_ext_claim, :canceled, :slc, id: 300_000) } - it "that are canceled, have a wrong EP_CODE, + it "that have a wrong EP_CODE, that are canceled, with a nil sync status, already in the pepsq table " do logged_epe7 PriorityEndProductSyncQueue.create(end_product_establishment_id: logged_epe7.id) @@ -108,14 +108,14 @@ end context "we only log updated vbms_ext_claims" do - it "that are cleared, *have a \"04%\" EP_CODE, + it "that have a \"04%\" EP_CODE, that are cleared, different sync status, and are not in pepsq table" do @logged_ext_claim.update(LEVEL_STATUS_CODE: "CLR") expect(PriorityEndProductSyncQueue.count).to eq 1 expect(PriorityEndProductSyncQueue.first.end_product_establishment_id).to eq @logged_epe.id end - it "that are cancelled, *have a \"03%\" EP_CODE, + it "that have a \"03%\" EP_CODE, that are cancelled, with out sync status, not in pepsq table " do @logged_epe.update(synced_status: nil) @logged_ext_claim.update(LEVEL_STATUS_CODE: "CAN", EP_CODE: "030") @@ -123,14 +123,14 @@ expect(PriorityEndProductSyncQueue.first.end_product_establishment_id).to eq @logged_epe.id end - it "that are cleared, *have a \"93%\" EP_CODE, + it "that have a \"93%\" EP_CODE, that are cleared, different sync status, and are not in pepsq table" do @logged_ext_claim.update(LEVEL_STATUS_CODE: "CLR", EP_CODE: "930") expect(PriorityEndProductSyncQueue.count).to eq 1 expect(PriorityEndProductSyncQueue.first.end_product_establishment_id).to eq @logged_epe.id end - it "that are cancelled, *have a \"68%\" EP_CODE, + it "that have a \"68%\" EP_CODE, that are cancelled, with out sync status, not in pepsq table " do @logged_epe.update(synced_status: nil) @logged_ext_claim.update(LEVEL_STATUS_CODE: "CAN", EP_CODE: "680") @@ -140,21 +140,21 @@ end context "we do not log updated vbms_ext_claims" do - it "*that are rdc, have a \"03%\" EP_CODE, + it "that have a \"03%\" EP_CODE, that are rdc, with out sync status, not in pepsq table " do @logged_ext_claim.update(LEVEL_STATUS_CODE: "RDC", EP_CODE: "030") expect(PriorityEndProductSyncQueue.count).to eq 0 end - it "that are canceled, *have a wrong EP_CODE, + it "that have a wrong EP_CODE, that are canceled, with a nil sync status, not in pepsq table " do @logged_epe.update(synced_status: nil) @logged_ext_claim.update(LEVEL_STATUS_CODE: "CAN", EP_CODE: "999") expect(PriorityEndProductSyncQueue.count).to eq 0 end - it "that are canceled, have a wrong EP_CODE, - with a nil sync status, *already in the pepsq table " do + it "that have a wrong EP_CODE, that are canceled, + with a nil sync status, already in the pepsq table " do PriorityEndProductSyncQueue.create(end_product_establishment_id: @logged_epe.id) expect(PriorityEndProductSyncQueue.count).to eq 1 end From 05968173b7b804081520d5e4b5836a9dbf0f7fcc Mon Sep 17 00:00:00 2001 From: nkirby-va Date: Mon, 25 Sep 2023 14:59:49 -0400 Subject: [PATCH 10/32] move create trigger logic into create vbms ext claim file --- Makefile.example | 8 ++-- db/scripts/add_pepsq_populate_trigger.sql | 17 +++---- .../external/create_vbms_ext_claim_table.rb | 46 +++++++++++++++++++ 3 files changed, 57 insertions(+), 14 deletions(-) diff --git a/Makefile.example b/Makefile.example index f40f648393b..213390877dc 100644 --- a/Makefile.example +++ b/Makefile.example @@ -177,17 +177,14 @@ audit-remove: ## Remove caseflow_audit schema, tables and triggers in postgres # These tables should not be included as part of migrations external-db-create: ## Creates external_vbms_ext_claim table bundle exec rails r db/scripts/external/create_vbms_ext_claim_table.rb - make add-populate-pepsq-trigger external-db-remove: ## Remove external_vbms_ext_claim table bundle exec rails r db/scripts/external/remove_vbms_ext_claim_table.rb - make remove-populate-pepsq-trigger # This needs to be manually run after make reset/migrate in order for local tests involving external tables to pass. # Otherwise the caseflow_certification_test schema will not create these tables and will error out. external-db-create-test: ## Creates table in caseflow_certification_test DB for local RSPEC tests bundle exec rails r -e test db/scripts/external/create_vbms_ext_claim_table.rb - make add-populate-pepsq-trigger remove-vbms-ext-claim-seeds: ## Drops audit tables, removes all PriorityEndProductSyncQueue, BatchProcess, and seed-vbms-ext-claim records, then rebuilds audit tables make audit-remove @@ -201,11 +198,14 @@ reseed-vbms-ext-claim: remove-vbms-ext-claim-seeds seed-vbms-ext-claim ## Re-see add-populate-pepsq-trigger: bundle exec rails r db/scripts/add_pepsq_populate_trigger.rb +# Add trigger to vbms_ext_claim to populate pepsq table +add-populate-pepsq-trigger-test: + bundle exec rails r -e test db/scripts/add_pepsq_populate_trigger.rb + # Remove populate pepsq trigger from vbms_ext_claim table remove-populate-pepsq-trigger: bundle exec rails r db/scripts/drop_pepsq_populate_trigger.rb - c: ## Start rails console bundle exec rails console diff --git a/db/scripts/add_pepsq_populate_trigger.sql b/db/scripts/add_pepsq_populate_trigger.sql index 7ec71f2a6be..2f73819f60a 100644 --- a/db/scripts/add_pepsq_populate_trigger.sql +++ b/db/scripts/add_pepsq_populate_trigger.sql @@ -1,26 +1,23 @@ --- TODO: remove line 2 and 4 drop trigger if exists update_claim_status_trigger on vbms_ext_claim; -truncate table priority_end_product_sync_queue; - create or replace function public.update_claim_status_trigger_function() returns trigger as $$ declare string_claim_id varchar(25); epe_id integer; begin - if (NEW."EP_CODE" LIKE '04%' - OR NEW."EP_CODE" LIKE '03%' - OR NEW."EP_CODE" LIKE '93%' - OR NEW."EP_CODE" LIKE '68%') - and (NEW."LEVEL_STATUS_CODE" = 'CLR' OR NEW."LEVEL_STATUS_CODE" = 'CAN') then + if (NEW.\"EP_CODE\" LIKE '04%' + OR NEW.\"EP_CODE\" LIKE '03%' + OR NEW.\"EP_CODE\" LIKE '93%' + OR NEW.\"EP_CODE\" LIKE '68%') + and (NEW.\"LEVEL_STATUS_CODE\" = 'CLR' OR NEW.\"LEVEL_STATUS_CODE\" = 'CAN') then - string_claim_id := cast(NEW."CLAIM_ID" as varchar); + string_claim_id := cast(NEW.\"CLAIM_ID\" as varchar); select id into epe_id from end_product_establishments where (reference_id = string_claim_id - and (synced_status is null or synced_status <> NEW."LEVEL_STATUS_CODE")); + and (synced_status is null or synced_status <> NEW.\"LEVEL_STATUS_CODE\")); if epe_id > 0 then diff --git a/db/scripts/external/create_vbms_ext_claim_table.rb b/db/scripts/external/create_vbms_ext_claim_table.rb index 3a1a37e2470..8a00a10d6ed 100644 --- a/db/scripts/external/create_vbms_ext_claim_table.rb +++ b/db/scripts/external/create_vbms_ext_claim_table.rb @@ -44,4 +44,50 @@ conn.execute('CREATE INDEX IF NOT EXISTS claim_id_index ON public.vbms_ext_claim ("CLAIM_ID")') conn.execute('CREATE INDEX IF NOT EXISTS level_status_code_index ON public.vbms_ext_claim ("LEVEL_STATUS_CODE")') + +conn.execute(" + drop trigger if exists update_claim_status_trigger on vbms_ext_claim; + + create or replace function public.update_claim_status_trigger_function() + returns trigger as $$ + declare + string_claim_id varchar(25); + epe_id integer; + begin + if (NEW.\"EP_CODE\" LIKE '04%' + OR NEW.\"EP_CODE\" LIKE '03%' + OR NEW.\"EP_CODE\" LIKE '93%' + OR NEW.\"EP_CODE\" LIKE '68%') + and (NEW.\"LEVEL_STATUS_CODE\" = 'CLR' OR NEW.\"LEVEL_STATUS_CODE\" = 'CAN') then + + string_claim_id := cast(NEW.\"CLAIM_ID\" as varchar); + + select id into epe_id + from end_product_establishments + where (reference_id = string_claim_id + and (synced_status is null or synced_status <> NEW.\"LEVEL_STATUS_CODE\")); + + if epe_id > 0 + then + if not exists ( + select 1 + from priority_end_product_sync_queue + where end_product_establishment_id = (select id from end_product_establishments where reference_id = string_claim_id) -- can this sub query be replaced with 'epe_id'? + ) then + insert into priority_end_product_sync_queue (created_at, end_product_establishment_id, updated_at) + values (now(), epe_id, now()); + end if; + end if; + end if; + return null; + end; + $$ + language plpgsql; + + create trigger update_claim_status_trigger + after update or insert on vbms_ext_claim + for each row + execute procedure public.update_claim_status_trigger_function(); + ") + conn.close From 36e5490ce7ff0858254a7ba5ada3be3ac8f84214 Mon Sep 17 00:00:00 2001 From: nkirby-va Date: Mon, 25 Sep 2023 17:11:44 -0400 Subject: [PATCH 11/32] troubleshooting for eli --- spec/factories/end_product_establishment.rb | 2 +- spec/jobs/batch_processes/batch_process_rescue_job_spec.rb | 2 +- .../priority_ep_sync_batch_process_job_spec.rb | 2 +- .../batch_processes/priority_ep_sync_batch_process_spec.rb | 6 +++++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/spec/factories/end_product_establishment.rb b/spec/factories/end_product_establishment.rb index 2aee01d206e..4cdeea1104c 100644 --- a/spec/factories/end_product_establishment.rb +++ b/spec/factories/end_product_establishment.rb @@ -72,7 +72,7 @@ active_hlr modifier { "030" } code { "030HLRR" } - after(:build) do |end_product_establishment, _evaluator| + after(:create) do |end_product_establishment, _evaluator| create(:vbms_ext_claim, :hlr, :cleared, claim_id: end_product_establishment.reference_id) ep = end_product_establishment.result ep_store = Fakes::EndProductStore.new diff --git a/spec/jobs/batch_processes/batch_process_rescue_job_spec.rb b/spec/jobs/batch_processes/batch_process_rescue_job_spec.rb index 1cc6bc6c23f..d6b60031437 100644 --- a/spec/jobs/batch_processes/batch_process_rescue_job_spec.rb +++ b/spec/jobs/batch_processes/batch_process_rescue_job_spec.rb @@ -30,7 +30,7 @@ end let!(:pepsq_records_two) do - PopulateEndProductSyncQueueJob.perform_now + #PopulateEndProductSyncQueueJob.perform_now end let!(:second_batch_process) do diff --git a/spec/jobs/batch_processes/priority_ep_sync_batch_process_job_spec.rb b/spec/jobs/batch_processes/priority_ep_sync_batch_process_job_spec.rb index be010324dfe..9d427e58143 100644 --- a/spec/jobs/batch_processes/priority_ep_sync_batch_process_job_spec.rb +++ b/spec/jobs/batch_processes/priority_ep_sync_batch_process_job_spec.rb @@ -22,7 +22,7 @@ end let!(:pepsq_records) do - PopulateEndProductSyncQueueJob.perform_now + #PopulateEndProductSyncQueueJob.perform_now PriorityEndProductSyncQueue.all end diff --git a/spec/models/batch_processes/priority_ep_sync_batch_process_spec.rb b/spec/models/batch_processes/priority_ep_sync_batch_process_spec.rb index 23268ca59ad..f2707311f27 100644 --- a/spec/models/batch_processes/priority_ep_sync_batch_process_spec.rb +++ b/spec/models/batch_processes/priority_ep_sync_batch_process_spec.rb @@ -145,8 +145,9 @@ end let!(:pepsq_records) do - PopulateEndProductSyncQueueJob.perform_now + #PopulateEndProductSyncQueueJob.perform_now PriorityEndProductSyncQueue.all + byebug end let!(:original_pepsq_record_size) { pepsq_records.size } @@ -306,10 +307,13 @@ context "when priority_ep_sync_batch_process destroys synced pepsq records" do before do allow(Rails.logger).to receive(:info) + pepsq_records.reload subject + #byebug end it "should delete the synced_pepsq records from the pepsq table and log it" do + #byebug expect(batch_process.priority_end_product_sync_queue.count).to eq(0) expect(Rails.logger).to have_received(:info).with( "PriorityEpSyncBatchProcessJob #{pepsq_records.size} synced records deleted:"\ From a206f7f87f5a94491d5125d2a4302fea265eda80 Mon Sep 17 00:00:00 2001 From: nkirby-va Date: Tue, 26 Sep 2023 00:22:44 -0400 Subject: [PATCH 12/32] move traits to after create --- spec/factories/end_product_establishment.rb | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/spec/factories/end_product_establishment.rb b/spec/factories/end_product_establishment.rb index 4cdeea1104c..cfaa3bf5332 100644 --- a/spec/factories/end_product_establishment.rb +++ b/spec/factories/end_product_establishment.rb @@ -46,7 +46,7 @@ active_hlr modifier { "030" } code { "030HLRR" } - after(:build) do |end_product_establishment, _evaluator| + after(:create) do |end_product_establishment, _evaluator| create(:vbms_ext_claim, :hlr, :canceled, claim_id: end_product_establishment.reference_id) ep = end_product_establishment.result ep_store = Fakes::EndProductStore.new @@ -59,7 +59,7 @@ active_hlr modifier { "030" } code { "030HLRR" } - after(:build) do |end_product_establishment, _evaluator| + after(:create) do |end_product_establishment, _evaluator| create(:vbms_ext_claim, :hlr, :rdc, claim_id: end_product_establishment.reference_id) ep = end_product_establishment.result ep_store = Fakes::EndProductStore.new @@ -87,7 +87,7 @@ modifier { "030" } code { "030HLRR" } source { create(:higher_level_review, veteran_file_number: veteran_file_number) } - after(:build) do |end_product_establishment, _evaluator| + after(:create) do |end_product_establishment, _evaluator| create(:vbms_ext_claim, :hlr, :canceled, claim_id: end_product_establishment.reference_id) ep = end_product_establishment.result ep_store = Fakes::EndProductStore.new @@ -115,7 +115,7 @@ active_supp modifier { "040" } code { "040SCR" } - after(:build) do |end_product_establishment, _evaluator| + after(:create) do |end_product_establishment, _evaluator| create(:vbms_ext_claim, :slc, :canceled, claim_id: end_product_establishment.reference_id) ep = end_product_establishment.result ep_store = Fakes::EndProductStore.new @@ -128,7 +128,7 @@ active_supp modifier { "040" } code { "040SCR" } - after(:build) do |end_product_establishment, _evaluator| + after(:create) do |end_product_establishment, _evaluator| create(:vbms_ext_claim, :slc, :rdc, claim_id: end_product_establishment.reference_id) ep = end_product_establishment.result ep_store = Fakes::EndProductStore.new @@ -141,7 +141,7 @@ active_supp modifier { "040" } code { "040SCR" } - after(:build) do |end_product_establishment, _evaluator| + after(:create) do |end_product_establishment, _evaluator| create(:vbms_ext_claim, :slc, :cleared, claim_id: end_product_establishment.reference_id) ep = end_product_establishment.result ep_store = Fakes::EndProductStore.new @@ -156,7 +156,7 @@ modifier { "040" } code { "040SCR" } source { create(:supplemental_claim, veteran_file_number: veteran_file_number) } - after(:build) do |end_product_establishment, _evaluator| + after(:create) do |end_product_establishment, _evaluator| create(:vbms_ext_claim, :slc, :canceled, claim_id: end_product_establishment.reference_id) ep = end_product_establishment.result ep_store = Fakes::EndProductStore.new @@ -171,7 +171,7 @@ modifier { "040" } code { "040SCR" } source { create(:supplemental_claim, veteran_file_number: veteran_file_number) } - after(:build) do |end_product_establishment, _evaluator| + after(:create) do |end_product_establishment, _evaluator| create(:vbms_ext_claim, :slc, :cleared, claim_id: end_product_establishment.reference_id) ep = end_product_establishment.result ep_store = Fakes::EndProductStore.new @@ -186,7 +186,7 @@ modifier { "030" } code { "030HLRR" } source { create(:higher_level_review, veteran_file_number: veteran_file_number) } - after(:build) do |end_product_establishment, _evaluator| + after(:create) do |end_product_establishment, _evaluator| create(:vbms_ext_claim, :hlr, :cleared, claim_id: end_product_establishment.reference_id) ep = end_product_establishment.result ep_store = Fakes::EndProductStore.new From a86c5edba187634eacea1a838cf74a9f29627826 Mon Sep 17 00:00:00 2001 From: nkirby-va Date: Tue, 26 Sep 2023 00:28:48 -0400 Subject: [PATCH 13/32] remove call to PopulateEndProductSyncQueueJob --- .../jobs/batch_processes/batch_process_rescue_job_spec.rb | 8 -------- .../priority_ep_sync_batch_process_job_spec.rb | 1 - .../priority_ep_sync_batch_process_spec.rb | 6 +----- spec/models/caseflow_stuck_record_spec.rb | 1 - 4 files changed, 1 insertion(+), 15 deletions(-) diff --git a/spec/jobs/batch_processes/batch_process_rescue_job_spec.rb b/spec/jobs/batch_processes/batch_process_rescue_job_spec.rb index d6b60031437..3469ce51e2d 100644 --- a/spec/jobs/batch_processes/batch_process_rescue_job_spec.rb +++ b/spec/jobs/batch_processes/batch_process_rescue_job_spec.rb @@ -17,10 +17,6 @@ create_list(:end_product_establishment, 2, :active_hlr_with_cleared_vbms_ext_claim) end - let!(:pepsq_records_one) do - PopulateEndProductSyncQueueJob.perform_now - end - let!(:first_batch_process) do PriorityEpSyncBatchProcessJob.perform_now end @@ -29,10 +25,6 @@ create_list(:end_product_establishment, 2, :active_hlr_with_cleared_vbms_ext_claim) end - let!(:pepsq_records_two) do - #PopulateEndProductSyncQueueJob.perform_now - end - let!(:second_batch_process) do PriorityEpSyncBatchProcessJob.perform_now end diff --git a/spec/jobs/batch_processes/priority_ep_sync_batch_process_job_spec.rb b/spec/jobs/batch_processes/priority_ep_sync_batch_process_job_spec.rb index 9d427e58143..8d00d31b9d2 100644 --- a/spec/jobs/batch_processes/priority_ep_sync_batch_process_job_spec.rb +++ b/spec/jobs/batch_processes/priority_ep_sync_batch_process_job_spec.rb @@ -22,7 +22,6 @@ end let!(:pepsq_records) do - #PopulateEndProductSyncQueueJob.perform_now PriorityEndProductSyncQueue.all end diff --git a/spec/models/batch_processes/priority_ep_sync_batch_process_spec.rb b/spec/models/batch_processes/priority_ep_sync_batch_process_spec.rb index f2707311f27..216c8953d46 100644 --- a/spec/models/batch_processes/priority_ep_sync_batch_process_spec.rb +++ b/spec/models/batch_processes/priority_ep_sync_batch_process_spec.rb @@ -134,7 +134,7 @@ create(:end_product_establishment, :active_supp_with_active_vbms_ext_claim) end let!(:active_supp_epe_w_cleared_vbms_ext_claim) do - create(:end_product_establishment, :active_supp_with_canceled_vbms_ext_claim) + create(:end_product_establishment, :active_supp_with_cleared_vbms_ext_claim) end let!(:cleared_supp_epes_w_cleared_vbms_ext_claim) do create(:end_product_establishment, :cleared_supp_with_cleared_vbms_ext_claim) @@ -145,9 +145,7 @@ end let!(:pepsq_records) do - #PopulateEndProductSyncQueueJob.perform_now PriorityEndProductSyncQueue.all - byebug end let!(:original_pepsq_record_size) { pepsq_records.size } @@ -309,11 +307,9 @@ allow(Rails.logger).to receive(:info) pepsq_records.reload subject - #byebug end it "should delete the synced_pepsq records from the pepsq table and log it" do - #byebug expect(batch_process.priority_end_product_sync_queue.count).to eq(0) expect(Rails.logger).to have_received(:info).with( "PriorityEpSyncBatchProcessJob #{pepsq_records.size} synced records deleted:"\ diff --git a/spec/models/caseflow_stuck_record_spec.rb b/spec/models/caseflow_stuck_record_spec.rb index a6504482604..3747ef5f277 100644 --- a/spec/models/caseflow_stuck_record_spec.rb +++ b/spec/models/caseflow_stuck_record_spec.rb @@ -7,7 +7,6 @@ end let!(:caseflow_stuck_record) do - PopulateEndProductSyncQueueJob.perform_now 3.times do PriorityEndProductSyncQueue.first.update!(last_batched_at: nil) From d71035bda08a05c876337ca40f35ed68398129c1 Mon Sep 17 00:00:00 2001 From: nkirby-va Date: Tue, 26 Sep 2023 00:29:17 -0400 Subject: [PATCH 14/32] remove scheduled PopulateEndProductSyncQueueJob --- config/initializers/scheduled_jobs.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/config/initializers/scheduled_jobs.rb b/config/initializers/scheduled_jobs.rb index 241e6f6d6d3..7662c421fed 100644 --- a/config/initializers/scheduled_jobs.rb +++ b/config/initializers/scheduled_jobs.rb @@ -23,7 +23,6 @@ "monthly_metrics" => MonthlyMetricsReportJob, "nightly_syncs" => NightlySyncsJob, "out_of_service_reminder" => OutOfServiceReminderJob, - "populate_end_product_sync_queue" => PopulateEndProductSyncQueueJob, "prepare_establish_claim" => PrepareEstablishClaimTasksJob, "push_priority_appeals_to_judges" => PushPriorityAppealsToJudgesJob, "quarterly_metrics" => QuarterlyMetricsReportJob, From 5d508673afdf9652626d0244c4bd39b8fa2a22ce Mon Sep 17 00:00:00 2001 From: nkirby-va Date: Tue, 26 Sep 2023 00:31:38 -0400 Subject: [PATCH 15/32] remove PopulateEndProductSyncQueueJob comment --- app/models/batch_processes/priority_ep_sync_batch_process.rb | 1 - app/models/priority_queues/priority_end_product_sync_queue.rb | 1 - 2 files changed, 2 deletions(-) diff --git a/app/models/batch_processes/priority_ep_sync_batch_process.rb b/app/models/batch_processes/priority_ep_sync_batch_process.rb index 3b5b41196d4..3222f7d2b66 100644 --- a/app/models/batch_processes/priority_ep_sync_batch_process.rb +++ b/app/models/batch_processes/priority_ep_sync_batch_process.rb @@ -87,7 +87,6 @@ def assign_batch_to_queued_records!(records) private # Purpose: Destroys "SYNCED" PEPSQ records to limit the growing number of table records. - # This functionality is needed for the PopulateEndProductSyncQueueJob query to be performant. # # Params: None # diff --git a/app/models/priority_queues/priority_end_product_sync_queue.rb b/app/models/priority_queues/priority_end_product_sync_queue.rb index d8f68cd1adb..4f727a217d3 100644 --- a/app/models/priority_queues/priority_end_product_sync_queue.rb +++ b/app/models/priority_queues/priority_end_product_sync_queue.rb @@ -52,7 +52,6 @@ def declare_record_stuck! end # Purpose: Destroys "SYNCED" PEPSQ records to limit the growing number of table records. - # This functionality is needed for the PopulateEndProductSyncQueueJob query to be performant. # # Params: The batch process the synced records belong to # From fbf12ae80b1af7021af6fa3c6fffce6ec430d5fc Mon Sep 17 00:00:00 2001 From: nkirby-va Date: Tue, 26 Sep 2023 00:57:01 -0400 Subject: [PATCH 16/32] remove removal of vbms_ext_claim trigger --- spec/sql/triggers/populate_end_product_sync_queue_spec.rb | 4 ---- 1 file changed, 4 deletions(-) diff --git a/spec/sql/triggers/populate_end_product_sync_queue_spec.rb b/spec/sql/triggers/populate_end_product_sync_queue_spec.rb index 5f4b87a0a58..2724cc5abf8 100644 --- a/spec/sql/triggers/populate_end_product_sync_queue_spec.rb +++ b/spec/sql/triggers/populate_end_product_sync_queue_spec.rb @@ -8,9 +8,6 @@ before do PriorityEndProductSyncQueue.delete_all end - after(:all) do - system("remove add-populate-pepsq-trigger") - end context "we only log inserted vbms_ext_claims" do let(:logged_epe1) { create(:end_product_establishment, :active, reference_id: 300_000) } @@ -104,7 +101,6 @@ after(:all) do EndProductEstablishment.delete(@logged_epe) VbmsExtClaim.delete(@logged_ext_claim) - system("remove add-populate-pepsq-trigger") end context "we only log updated vbms_ext_claims" do From d189376ab044e73dbb205e288d37818093be00b7 Mon Sep 17 00:00:00 2001 From: nkirby-va Date: Tue, 26 Sep 2023 00:59:49 -0400 Subject: [PATCH 17/32] add trigger after testing removal --- spec/sql/triggers/populate_end_product_sync_queue_spec.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/spec/sql/triggers/populate_end_product_sync_queue_spec.rb b/spec/sql/triggers/populate_end_product_sync_queue_spec.rb index 2724cc5abf8..bde647057f5 100644 --- a/spec/sql/triggers/populate_end_product_sync_queue_spec.rb +++ b/spec/sql/triggers/populate_end_product_sync_queue_spec.rb @@ -161,6 +161,9 @@ before(:all) do system("make remove-populate-pepsq-trigger") end + after(:all) do + system("add remove-populate-pepsq-trigger") + end let(:logged_epe) { create(:end_product_establishment, :active, reference_id: 300_000) } let(:logged_ext_claim) { create(:vbms_ext_claim, :cleared, :slc, id: 300_000) } From cfcb4f95a1e2023969c2fef84dc0ece261e5f109 Mon Sep 17 00:00:00 2001 From: nkirby-va Date: Tue, 26 Sep 2023 01:00:52 -0400 Subject: [PATCH 18/32] remove PopulateEndProductSyncQueueJob and spec --- .../populate_end_product_sync_queue_job.rb | 102 -------- ...opulate_end_product_sync_queue_job_spec.rb | 226 ------------------ 2 files changed, 328 deletions(-) delete mode 100644 app/jobs/populate_end_product_sync_queue_job.rb delete mode 100644 spec/jobs/priority_queues/populate_end_product_sync_queue_job_spec.rb diff --git a/app/jobs/populate_end_product_sync_queue_job.rb b/app/jobs/populate_end_product_sync_queue_job.rb deleted file mode 100644 index 13d537fab01..00000000000 --- a/app/jobs/populate_end_product_sync_queue_job.rb +++ /dev/null @@ -1,102 +0,0 @@ -# frozen_string_literal: true - -# This job will find deltas between the end product establishment table and the VBMS ext claim table -# where VBMS ext claim level status code is CLR or CAN. If EP is already in the queue it will be skipped. -# Job will populate queue ENV["END_PRODUCT_QUEUE_BATCH_LIMIT"] records at a time. -# This job will run on a 50 minute loop, sleeping for 5 seconds between iterations. -class PopulateEndProductSyncQueueJob < CaseflowJob - queue_with_priority :low_priority - - JOB_DURATION ||= ENV["END_PRODUCT_QUEUE_JOB_DURATION"].to_i.minutes - SLEEP_DURATION ||= ENV["END_PRODUCT_QUEUE_SLEEP_DURATION"].to_i - BATCH_LIMIT ||= ENV["END_PRODUCT_QUEUE_BATCH_LIMIT"].to_i - - # rubocop:disable Metrics/CyclomaticComplexity - def perform - setup_job - loop do - break if job_running_past_expected_end_time? || should_stop_job - - begin - batch = ActiveRecord::Base.transaction do - priority_epes = find_priority_end_product_establishments_to_sync - next if priority_epes.empty? - - priority_epes - end - - batch ? insert_into_priority_sync_queue(batch) : stop_job(log_no_records_found: true) - - sleep(SLEEP_DURATION) - rescue StandardError => error - log_error(error, extra: { active_job_id: job_id.to_s, job_time: Time.zone.now.to_s }) - slack_msg = "Error running #{self.class.name}. Error: #{error.message}. Active Job ID: #{job_id}." - slack_msg += " See Sentry event #{Raven.last_event_id}." if Raven.last_event_id.present? - slack_service.send_notification("[ERROR] #{slack_msg}", self.class.to_s) - stop_job - end - end - end - # rubocop:enable Metrics/CyclomaticComplexity - - private - - attr_accessor :job_expected_end_time, :should_stop_job - - # rubocop:disable Metrics/MethodLength - def find_priority_end_product_establishments_to_sync - get_sql = <<-SQL - WITH priority_eps AS ( - SELECT vec."CLAIM_ID"::varchar, vec."LEVEL_STATUS_CODE" - FROM vbms_ext_claim vec - WHERE vec."LEVEL_STATUS_CODE" in ('CLR', 'CAN') - AND (vec."EP_CODE" LIKE '04%' OR vec."EP_CODE" LIKE '03%' OR vec."EP_CODE" LIKE '93%' OR vec."EP_CODE" LIKE '68%') - ), - priority_queued_epe_ids AS ( - SELECT end_product_establishment_id - FROM priority_end_product_sync_queue) - SELECT id - FROM end_product_establishments epe - INNER JOIN priority_eps - ON epe.reference_id = priority_eps."CLAIM_ID" - WHERE (epe.synced_status is null or epe.synced_status <> priority_eps."LEVEL_STATUS_CODE") - AND NOT EXISTS (SELECT end_product_establishment_id - FROM priority_queued_epe_ids - WHERE priority_queued_epe_ids.end_product_establishment_id = epe.id) - LIMIT #{BATCH_LIMIT}; - SQL - - ActiveRecord::Base.connection.exec_query(ActiveRecord::Base.sanitize_sql(get_sql)).rows.flatten - end - # rubocop:enable Metrics/MethodLength - - def insert_into_priority_sync_queue(batch) - priority_end_product_sync_queue_records = batch.map do |ep_id| - PriorityEndProductSyncQueue.new(end_product_establishment_id: ep_id) - end - - # Bulk insert PriorityEndProductSyncQueue records in a single SQL statement - PriorityEndProductSyncQueue.import(priority_end_product_sync_queue_records) - Rails.logger.info("PopulateEndProductSyncQueueJob EPEs processed: #{batch} - Time: #{Time.zone.now}") - end - - def setup_job - RequestStore.store[:current_user] = User.system_user - @should_stop_job = false - @job_expected_end_time = Time.zone.now + JOB_DURATION - end - - def job_running_past_expected_end_time? - Time.zone.now > job_expected_end_time - end - - # :reek:BooleanParameter - # :reek:ControlParameter - def stop_job(log_no_records_found: false) - self.should_stop_job = true - if log_no_records_found - Rails.logger.info("PopulateEndProductSyncQueueJob is not able to find any batchable EPE records."\ - " Active Job ID: #{job_id}. Time: #{Time.zone.now}") - end - end -end diff --git a/spec/jobs/priority_queues/populate_end_product_sync_queue_job_spec.rb b/spec/jobs/priority_queues/populate_end_product_sync_queue_job_spec.rb deleted file mode 100644 index cd06a95aa5f..00000000000 --- a/spec/jobs/priority_queues/populate_end_product_sync_queue_job_spec.rb +++ /dev/null @@ -1,226 +0,0 @@ -# frozen_string_literal: true - -describe PopulateEndProductSyncQueueJob, type: :job do - include ActiveJob::TestHelper - - let(:slack_service) { SlackService.new(url: "http://www.example.com") } - - let!(:epes_to_be_queued) do - create_list(:end_product_establishment, 2, :active_hlr_with_cleared_vbms_ext_claim) - end - - let!(:not_found_epe) do - create(:end_product_establishment, :active_hlr_with_active_vbms_ext_claim) - end - - before do - # Batch limit changes to 1 to test PopulateEndProductSyncQueueJob loop - stub_const("PopulateEndProductSyncQueueJob::BATCH_LIMIT", 1) - end - - subject do - PopulateEndProductSyncQueueJob.perform_later - end - - describe "#perform" do - context "when all records sync successfully" do - before do - allow(SlackService).to receive(:new).with(url: anything).and_return(slack_service) - allow(slack_service).to receive(:send_notification) { |_, first_arg| @slack_msg = first_arg } - perform_enqueued_jobs do - subject - end - end - - it "adds the 2 unsynced epes to the end product synce queue" do - expect(PriorityEndProductSyncQueue.count).to eq 2 - end - - it "the current user is set to a system user" do - expect(RequestStore.store[:current_user].id).to eq(User.system_user.id) - end - - it "adds the epes to the priority end product sync queue table" do - expect(PriorityEndProductSyncQueue.first.end_product_establishment_id).to eq epes_to_be_queued.first.id - expect(PriorityEndProductSyncQueue.second.end_product_establishment_id).to eq epes_to_be_queued.second.id - end - - it "the epes are associated with a vbms_ext_claim record" do - expect(EndProductEstablishment.find(PriorityEndProductSyncQueue.first.end_product_establishment_id) - .reference_id).to eq epes_to_be_queued.first.vbms_ext_claim.claim_id.to_s - expect(EndProductEstablishment.find(PriorityEndProductSyncQueue.second.end_product_establishment_id) - .reference_id).to eq epes_to_be_queued.second.vbms_ext_claim.claim_id.to_s - end - - it "the priority end product sync queue records have a status of 'NOT_PROCESSED'" do - expect(PriorityEndProductSyncQueue.first.status).to eq "NOT_PROCESSED" - expect(PriorityEndProductSyncQueue.second.status).to eq "NOT_PROCESSED" - end - - it "slack will NOT be notified when job runs successfully" do - expect(slack_service).to_not have_received(:send_notification) - end - end - - context "when the epe's reference id is a lettered string (i.e. only match on matching numbers)" do - before do - epes_to_be_queued.each { |epe| epe.update!(reference_id: "whaddup yooo") } - perform_enqueued_jobs do - subject - end - end - - it "doesn't add epe to the queue" do - expect(PriorityEndProductSyncQueue.count).to eq 0 - end - end - - context "when a priority end product sync queue record already exists with the epe id" do - before do - PriorityEndProductSyncQueue.create(end_product_establishment_id: epes_to_be_queued.first.id) - perform_enqueued_jobs do - subject - end - end - - it "will not add same epe more than once in the priorty end product sync queue table" do - expect(PriorityEndProductSyncQueue.count).to eq 2 - end - end - - context "when the epe records' synced_status value is nil" do - before do - epes_to_be_queued.each { |epe| epe.update!(synced_status: nil) } - perform_enqueued_jobs do - subject - end - end - - it "will add the epe if epe synced status is nil and other conditions are met" do - expect(PriorityEndProductSyncQueue.count).to eq 2 - end - end - - context "when the job duration ends before all PriorityEndProductSyncQueue records can be batched" do - before do - # Job duration of 0.001 seconds limits the job's loop to one iteration - stub_const("PopulateEndProductSyncQueueJob::JOB_DURATION", 0.001.seconds) - allow(SlackService).to receive(:new).with(url: anything).and_return(slack_service) - allow(slack_service).to receive(:send_notification) { |_, first_arg| @slack_msg = first_arg } - perform_enqueued_jobs do - subject - end - end - - it "there are 3 epe records" do - expect(EndProductEstablishment.count).to eq(3) - end - - it "creates 1 priority end product sync queue record" do - expect(PriorityEndProductSyncQueue.count).to eq(1) - end - - it "the current user is set to a system user" do - expect(RequestStore.store[:current_user].id).to eq(User.system_user.id) - end - - it "adds the epes to the priority end product sync queue table" do - expect(PriorityEndProductSyncQueue.first.end_product_establishment_id).to eq epes_to_be_queued.first.id - end - - it "the epes are associated with a vbms_ext_claim record" do - expect(EndProductEstablishment.find(PriorityEndProductSyncQueue.first.end_product_establishment_id) - .reference_id).to eq epes_to_be_queued.first.vbms_ext_claim.claim_id.to_s - end - - it "the priority end product sync queue record has a status of 'NOT_PROCESSED'" do - expect(PriorityEndProductSyncQueue.first.status).to eq "NOT_PROCESSED" - end - - it "slack will NOT be notified when job runs successfully" do - expect(slack_service).to_not have_received(:send_notification) - end - end - - context "when there are no records available to batch" do - before do - EndProductEstablishment.destroy_all - allow(Rails.logger).to receive(:info) - perform_enqueued_jobs do - subject - end - end - - it "doesn't add any epes to the batch" do - expect(PriorityEndProductSyncQueue.count).to eq 0 - end - - it "logs a message that says 'PopulateEndProductSyncQueueJob is not able to find any batchable EPE records'" do - expect(Rails.logger).to have_received(:info).with( - "PopulateEndProductSyncQueueJob is not able to find any batchable EPE records."\ - " Active Job ID: #{subject.job_id}."\ - " Time: #{Time.zone.now}" - ) - end - end - - context "when an error is raised during the job" do - let(:standard_error) { StandardError.new("Uh-Oh!") } - before do - allow(Rails.logger).to receive(:error) - allow(Raven).to receive(:capture_exception) - allow(Raven).to receive(:last_event_id) { "sentry_123" } - allow(SlackService).to receive(:new).with(url: anything).and_return(slack_service) - allow(slack_service).to receive(:send_notification) { |_, first_arg| @slack_msg = first_arg } - allow_any_instance_of(PopulateEndProductSyncQueueJob) - .to receive(:find_priority_end_product_establishments_to_sync).and_raise(standard_error) - perform_enqueued_jobs do - subject - end - end - - it "the error and the backtrace will be logged" do - expect(Rails.logger).to have_received(:error).with(an_instance_of(StandardError)) - end - - it "the error will be sent to Sentry" do - expect(Raven).to have_received(:capture_exception) - .with(instance_of(StandardError), - extra: { - active_job_id: subject.job_id, - job_time: Time.zone.now.to_s - }) - end - - it "slack will be notified when job fails" do - expect(slack_service).to have_received(:send_notification).with( - "[ERROR] Error running PopulateEndProductSyncQueueJob. Error: #{standard_error.message}."\ - " Active Job ID: #{subject.job_id}. See Sentry event sentry_123.", "PopulateEndProductSyncQueueJob" - ) - end - end - - context "when there are no records available to batch" do - before do - VbmsExtClaim.destroy_all - allow(Rails.logger).to receive(:info) - allow(SlackService).to receive(:new).with(url: anything).and_return(slack_service) - allow(slack_service).to receive(:send_notification) { |_, first_arg| @slack_msg = first_arg } - perform_enqueued_jobs do - subject - end - end - - it "a message that says 'Cannot Find Any Records to Batch' will be logged" do - expect(Rails.logger).to have_received(:info).with( - "PopulateEndProductSyncQueueJob is not able to find any batchable EPE records."\ - " Active Job ID: #{subject.job_id}. Time: #{Time.zone.now}" - ) - end - - it "slack will NOT be notified when job runs successfully" do - expect(slack_service).to_not have_received(:send_notification) - end - end - end -end From 3fbeb61236b2021c2d171c91524433cc1a9f39fc Mon Sep 17 00:00:00 2001 From: nkirby-va Date: Tue, 26 Sep 2023 01:14:18 -0400 Subject: [PATCH 19/32] fix cc issue --- spec/models/caseflow_stuck_record_spec.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/spec/models/caseflow_stuck_record_spec.rb b/spec/models/caseflow_stuck_record_spec.rb index 3747ef5f277..a642585ef26 100644 --- a/spec/models/caseflow_stuck_record_spec.rb +++ b/spec/models/caseflow_stuck_record_spec.rb @@ -7,7 +7,6 @@ end let!(:caseflow_stuck_record) do - 3.times do PriorityEndProductSyncQueue.first.update!(last_batched_at: nil) PriorityEpSyncBatchProcessJob.perform_now From 2337a6ec3485eaab445cb3c0343d225bd338dc37 Mon Sep 17 00:00:00 2001 From: nkirby-va Date: Tue, 26 Sep 2023 01:39:58 -0400 Subject: [PATCH 20/32] explicit trigger removal in test env --- Makefile.example | 4 ++-- spec/sql/triggers/populate_end_product_sync_queue_spec.rb | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile.example b/Makefile.example index 213390877dc..e9edbd2e8c3 100644 --- a/Makefile.example +++ b/Makefile.example @@ -203,8 +203,8 @@ add-populate-pepsq-trigger-test: bundle exec rails r -e test db/scripts/add_pepsq_populate_trigger.rb # Remove populate pepsq trigger from vbms_ext_claim table -remove-populate-pepsq-trigger: - bundle exec rails r db/scripts/drop_pepsq_populate_trigger.rb +remove-populate-pepsq-trigger-test: + bundle exec rails r -e test db/scripts/drop_pepsq_populate_trigger.rb c: ## Start rails console bundle exec rails console diff --git a/spec/sql/triggers/populate_end_product_sync_queue_spec.rb b/spec/sql/triggers/populate_end_product_sync_queue_spec.rb index bde647057f5..1e43f4d3b1c 100644 --- a/spec/sql/triggers/populate_end_product_sync_queue_spec.rb +++ b/spec/sql/triggers/populate_end_product_sync_queue_spec.rb @@ -159,10 +159,10 @@ context "when the trigger is removed from the vbms_ext_claim table" do before(:all) do - system("make remove-populate-pepsq-trigger") + system("make remove-populate-pepsq-trigger-test") end after(:all) do - system("add remove-populate-pepsq-trigger") + system("make add-populate-pepsq-trigger-test") end let(:logged_epe) { create(:end_product_establishment, :active, reference_id: 300_000) } From e1317e6cdc23899dfc917259f301d6de3fc1677a Mon Sep 17 00:00:00 2001 From: nkirby-va Date: Tue, 26 Sep 2023 08:54:20 -0400 Subject: [PATCH 21/32] change target name --- Makefile.example | 2 +- spec/sql/triggers/populate_end_product_sync_queue_spec.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile.example b/Makefile.example index e9edbd2e8c3..70f98b76e7e 100644 --- a/Makefile.example +++ b/Makefile.example @@ -203,7 +203,7 @@ add-populate-pepsq-trigger-test: bundle exec rails r -e test db/scripts/add_pepsq_populate_trigger.rb # Remove populate pepsq trigger from vbms_ext_claim table -remove-populate-pepsq-trigger-test: +drop-populate-pepsq-trigger-test: bundle exec rails r -e test db/scripts/drop_pepsq_populate_trigger.rb c: ## Start rails console diff --git a/spec/sql/triggers/populate_end_product_sync_queue_spec.rb b/spec/sql/triggers/populate_end_product_sync_queue_spec.rb index 1e43f4d3b1c..cc8e6036575 100644 --- a/spec/sql/triggers/populate_end_product_sync_queue_spec.rb +++ b/spec/sql/triggers/populate_end_product_sync_queue_spec.rb @@ -159,7 +159,7 @@ context "when the trigger is removed from the vbms_ext_claim table" do before(:all) do - system("make remove-populate-pepsq-trigger-test") + system("make drop-populate-pepsq-trigger-test") end after(:all) do system("make add-populate-pepsq-trigger-test") From 04b9d51b8a43f78b4185f568fc714f1c4940f203 Mon Sep 17 00:00:00 2001 From: nkirby-va Date: Tue, 26 Sep 2023 10:14:12 -0400 Subject: [PATCH 22/32] delete pepsq queue before each on removing trigger test --- spec/sql/triggers/populate_end_product_sync_queue_spec.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/spec/sql/triggers/populate_end_product_sync_queue_spec.rb b/spec/sql/triggers/populate_end_product_sync_queue_spec.rb index cc8e6036575..2eb98d99bd1 100644 --- a/spec/sql/triggers/populate_end_product_sync_queue_spec.rb +++ b/spec/sql/triggers/populate_end_product_sync_queue_spec.rb @@ -161,6 +161,9 @@ before(:all) do system("make drop-populate-pepsq-trigger-test") end + before do + PriorityEndProductSyncQueue.delete_all + end after(:all) do system("make add-populate-pepsq-trigger-test") end From 9fa4343340d6be6e4991fd6a10b1874bae91abb9 Mon Sep 17 00:00:00 2001 From: nkirby-va Date: Tue, 26 Sep 2023 10:34:52 -0400 Subject: [PATCH 23/32] add make target --- Makefile.example | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Makefile.example b/Makefile.example index 70f98b76e7e..c69e89b11fd 100644 --- a/Makefile.example +++ b/Makefile.example @@ -202,6 +202,10 @@ add-populate-pepsq-trigger: add-populate-pepsq-trigger-test: bundle exec rails r -e test db/scripts/add_pepsq_populate_trigger.rb +# Remove populate pepsq trigger from vbms_ext_claim table +drop-populate-pepsq-trigger: + bundle exec rails r db/scripts/drop_pepsq_populate_trigger.rb + # Remove populate pepsq trigger from vbms_ext_claim table drop-populate-pepsq-trigger-test: bundle exec rails r -e test db/scripts/drop_pepsq_populate_trigger.rb From 4dc302ac38875dc68a30547762964efdd52e64b9 Mon Sep 17 00:00:00 2001 From: nkirby-va Date: Tue, 26 Sep 2023 11:11:03 -0400 Subject: [PATCH 24/32] switch to explicit bundle command --- .../sql/triggers/populate_end_product_sync_queue_spec.rb | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/spec/sql/triggers/populate_end_product_sync_queue_spec.rb b/spec/sql/triggers/populate_end_product_sync_queue_spec.rb index 2eb98d99bd1..06ea5dd189d 100644 --- a/spec/sql/triggers/populate_end_product_sync_queue_spec.rb +++ b/spec/sql/triggers/populate_end_product_sync_queue_spec.rb @@ -3,7 +3,8 @@ describe "vbms_ext_claim trigger to populate end_product_sync_que table", :postgres do context "when the trigger is added to the vbms_ext_claim table before the creation new records" do before(:all) do - system("make add-populate-pepsq-trigger") + system("bundle exec rails r -e test db/scripts/drop_pepsq_populate_trigger.rb") + system("bundle exec rails r -e test db/scripts/add_pepsq_populate_trigger.rb") end before do PriorityEndProductSyncQueue.delete_all @@ -93,7 +94,7 @@ before(:all) do @logged_epe = create(:end_product_establishment, :active, reference_id: 300_000) @logged_ext_claim = create(:vbms_ext_claim, :rdc, :slc, id: 300_000) - system("make add-populate-pepsq-trigger") + system("bundle exec rails r -e test db/scripts/add_pepsq_populate_trigger.rb") end before do PriorityEndProductSyncQueue.delete_all @@ -159,13 +160,13 @@ context "when the trigger is removed from the vbms_ext_claim table" do before(:all) do - system("make drop-populate-pepsq-trigger-test") + system("bundle exec rails r -e test db/scripts/drop_pepsq_populate_trigger.rb") end before do PriorityEndProductSyncQueue.delete_all end after(:all) do - system("make add-populate-pepsq-trigger-test") + system("bundle exec rails r -e test db/scripts/add_pepsq_populate_trigger.rb") end let(:logged_epe) { create(:end_product_establishment, :active, reference_id: 300_000) } From 0de7639cd962b67ed6a1c8f3e1f11732713c62c9 Mon Sep 17 00:00:00 2001 From: nkirby-va Date: Tue, 26 Sep 2023 16:54:32 -0400 Subject: [PATCH 25/32] use epe_id and add trigger sql to create table --- db/scripts/add_pepsq_populate_trigger.rb | 2 +- db/scripts/add_pepsq_populate_trigger.sql | 2 +- .../external/create_vbms_ext_claim_table.rb | 2 +- .../external/create_vbms_ext_claim_table.sql | 43 +++++++++++++++++++ 4 files changed, 46 insertions(+), 3 deletions(-) diff --git a/db/scripts/add_pepsq_populate_trigger.rb b/db/scripts/add_pepsq_populate_trigger.rb index fced0251748..e68c03822b2 100644 --- a/db/scripts/add_pepsq_populate_trigger.rb +++ b/db/scripts/add_pepsq_populate_trigger.rb @@ -30,7 +30,7 @@ if not exists ( select 1 from priority_end_product_sync_queue - where end_product_establishment_id = (select id from end_product_establishments where reference_id = string_claim_id) -- can this sub query be replaced with 'epe_id'? + where end_product_establishment_id = epe_id ) then insert into priority_end_product_sync_queue (created_at, end_product_establishment_id, updated_at) values (now(), epe_id, now()); diff --git a/db/scripts/add_pepsq_populate_trigger.sql b/db/scripts/add_pepsq_populate_trigger.sql index 2f73819f60a..3319f7f57e3 100644 --- a/db/scripts/add_pepsq_populate_trigger.sql +++ b/db/scripts/add_pepsq_populate_trigger.sql @@ -24,7 +24,7 @@ returns trigger as $$ if not exists ( select 1 from priority_end_product_sync_queue - where end_product_establishment_id = (select id from end_product_establishments where reference_id = string_claim_id) -- can this sub query be replaced with 'epe_id'? + where end_product_establishment_id = epe_id ) then insert into priority_end_product_sync_queue (created_at, end_product_establishment_id, updated_at) values (now(), epe_id, now()); diff --git a/db/scripts/external/create_vbms_ext_claim_table.rb b/db/scripts/external/create_vbms_ext_claim_table.rb index 8a00a10d6ed..c327b4c3c2b 100644 --- a/db/scripts/external/create_vbms_ext_claim_table.rb +++ b/db/scripts/external/create_vbms_ext_claim_table.rb @@ -72,7 +72,7 @@ if not exists ( select 1 from priority_end_product_sync_queue - where end_product_establishment_id = (select id from end_product_establishments where reference_id = string_claim_id) -- can this sub query be replaced with 'epe_id'? + where end_product_establishment_id = epe_id ) then insert into priority_end_product_sync_queue (created_at, end_product_establishment_id, updated_at) values (now(), epe_id, now()); diff --git a/db/scripts/external/create_vbms_ext_claim_table.sql b/db/scripts/external/create_vbms_ext_claim_table.sql index 02a6e0f356e..017b6b9b8cf 100644 --- a/db/scripts/external/create_vbms_ext_claim_table.sql +++ b/db/scripts/external/create_vbms_ext_claim_table.sql @@ -40,3 +40,46 @@ CREATE TABLE IF NOT EXISTS PUBLIC.VBMS_EXT_CLAIM ( CREATE INDEX IF NOT EXISTS CLAIM_ID_INDEX ON PUBLIC.VBMS_EXT_CLAIM ("CLAIM_ID"); CREATE INDEX IF NOT EXISTS LEVEL_STATUS_CODE_INDEX ON PUBLIC.VBMS_EXT_CLAIM ("LEVEL_STATUS_CODE"); + +drop trigger if exists update_claim_status_trigger on vbms_ext_claim; + +create or replace function public.update_claim_status_trigger_function() +returns trigger as $$ + declare + string_claim_id varchar(25); + epe_id integer; + begin + if (NEW.\"EP_CODE\" LIKE '04%' + OR NEW.\"EP_CODE\" LIKE '03%' + OR NEW.\"EP_CODE\" LIKE '93%' + OR NEW.\"EP_CODE\" LIKE '68%') + and (NEW.\"LEVEL_STATUS_CODE\" = 'CLR' OR NEW.\"LEVEL_STATUS_CODE\" = 'CAN') then + + string_claim_id := cast(NEW.\"CLAIM_ID\" as varchar); + + select id into epe_id + from end_product_establishments + where (reference_id = string_claim_id + and (synced_status is null or synced_status <> NEW.\"LEVEL_STATUS_CODE\")); + + if epe_id > 0 + then + if not exists ( + select 1 + from priority_end_product_sync_queue + where end_product_establishment_id = epe_id + ) then + insert into priority_end_product_sync_queue (created_at, end_product_establishment_id, updated_at) + values (now(), epe_id, now()); + end if; + end if; + end if; + return null; + end; +$$ +language plpgsql; + +create trigger update_claim_status_trigger +after update or insert on vbms_ext_claim +for each row +execute procedure public.update_claim_status_trigger_function(); From dc82b9f29c1e0f9877648530a7664d6ed5d203ad Mon Sep 17 00:00:00 2001 From: nkirby-va Date: Tue, 26 Sep 2023 17:45:40 -0400 Subject: [PATCH 26/32] rename files - make comment pointing to trigger --- Makefile.example | 8 ++++---- .../priority_queues/priority_end_product_sync_queue.rb | 5 +++++ ...rb => add_pepsq_populate_trigger_to_vbms_ext_claim.rb} | 0 ...l => add_pepsq_populate_trigger_to_vbms_ext_claim.sql} | 0 ...b => drop_pepsq_populate_trigger_to_vbms_ext_claim.rb} | 0 ... => drop_pepsq_populate_trigger_to_vbms_ext_claim.sql} | 0 spec/sql/triggers/populate_end_product_sync_queue_spec.rb | 4 ++++ 7 files changed, 13 insertions(+), 4 deletions(-) rename db/scripts/{add_pepsq_populate_trigger.rb => add_pepsq_populate_trigger_to_vbms_ext_claim.rb} (100%) rename db/scripts/{add_pepsq_populate_trigger.sql => add_pepsq_populate_trigger_to_vbms_ext_claim.sql} (100%) rename db/scripts/{drop_pepsq_populate_trigger.rb => drop_pepsq_populate_trigger_to_vbms_ext_claim.rb} (100%) rename db/scripts/{drop_pepsq_populate_trigger.sql => drop_pepsq_populate_trigger_to_vbms_ext_claim.sql} (100%) diff --git a/Makefile.example b/Makefile.example index c69e89b11fd..fa4fcce7792 100644 --- a/Makefile.example +++ b/Makefile.example @@ -196,19 +196,19 @@ reseed-vbms-ext-claim: remove-vbms-ext-claim-seeds seed-vbms-ext-claim ## Re-see # Add trigger to vbms_ext_claim to populate pepsq table add-populate-pepsq-trigger: - bundle exec rails r db/scripts/add_pepsq_populate_trigger.rb + bundle exec rails r db/scripts/add_pepsq_populate_trigger_to_vbms_ext_claim.rb # Add trigger to vbms_ext_claim to populate pepsq table add-populate-pepsq-trigger-test: - bundle exec rails r -e test db/scripts/add_pepsq_populate_trigger.rb + bundle exec rails r -e test db/scripts/add_pepsq_populate_trigger_to_vbms_ext_claim.rb # Remove populate pepsq trigger from vbms_ext_claim table drop-populate-pepsq-trigger: - bundle exec rails r db/scripts/drop_pepsq_populate_trigger.rb + bundle exec rails r db/scripts/drop_pepsq_populate_trigger_to_vbms_ext_claim.rb # Remove populate pepsq trigger from vbms_ext_claim table drop-populate-pepsq-trigger-test: - bundle exec rails r -e test db/scripts/drop_pepsq_populate_trigger.rb + bundle exec rails r -e test db/scripts/drop_pepsq_populate_trigger_to_vbms_ext_claim.rb c: ## Start rails console bundle exec rails console diff --git a/app/models/priority_queues/priority_end_product_sync_queue.rb b/app/models/priority_queues/priority_end_product_sync_queue.rb index 4f727a217d3..b15259373db 100644 --- a/app/models/priority_queues/priority_end_product_sync_queue.rb +++ b/app/models/priority_queues/priority_end_product_sync_queue.rb @@ -2,6 +2,11 @@ # Model for Priority End Product Sync Queue table. # This table consists of records of End Product Establishment IDs that need to be synced with VBMS. + +# These are populated via the trigger that is created on creation of the vbms_ext_claim table +# The trigger is located in: +# db/scripts/external/create_vbms_ext_claim_table.rb +# db/scripts/ class PriorityEndProductSyncQueue < CaseflowRecord self.table_name = "priority_end_product_sync_queue" diff --git a/db/scripts/add_pepsq_populate_trigger.rb b/db/scripts/add_pepsq_populate_trigger_to_vbms_ext_claim.rb similarity index 100% rename from db/scripts/add_pepsq_populate_trigger.rb rename to db/scripts/add_pepsq_populate_trigger_to_vbms_ext_claim.rb diff --git a/db/scripts/add_pepsq_populate_trigger.sql b/db/scripts/add_pepsq_populate_trigger_to_vbms_ext_claim.sql similarity index 100% rename from db/scripts/add_pepsq_populate_trigger.sql rename to db/scripts/add_pepsq_populate_trigger_to_vbms_ext_claim.sql diff --git a/db/scripts/drop_pepsq_populate_trigger.rb b/db/scripts/drop_pepsq_populate_trigger_to_vbms_ext_claim.rb similarity index 100% rename from db/scripts/drop_pepsq_populate_trigger.rb rename to db/scripts/drop_pepsq_populate_trigger_to_vbms_ext_claim.rb diff --git a/db/scripts/drop_pepsq_populate_trigger.sql b/db/scripts/drop_pepsq_populate_trigger_to_vbms_ext_claim.sql similarity index 100% rename from db/scripts/drop_pepsq_populate_trigger.sql rename to db/scripts/drop_pepsq_populate_trigger_to_vbms_ext_claim.sql diff --git a/spec/sql/triggers/populate_end_product_sync_queue_spec.rb b/spec/sql/triggers/populate_end_product_sync_queue_spec.rb index 06ea5dd189d..38be1d931bd 100644 --- a/spec/sql/triggers/populate_end_product_sync_queue_spec.rb +++ b/spec/sql/triggers/populate_end_product_sync_queue_spec.rb @@ -1,5 +1,9 @@ # frozen_string_literal: true +# The PriorityEndProductSyncQue is populated via the trigger that is created on creation of the vbms_ext_claim table +# The trigger is located in: +# db/scripts/external/create_vbms_ext_claim_table.rb +# db/scripts/ describe "vbms_ext_claim trigger to populate end_product_sync_que table", :postgres do context "when the trigger is added to the vbms_ext_claim table before the creation new records" do before(:all) do From d3bdb6d223c252bd7b53fdbd37f944e05f471539 Mon Sep 17 00:00:00 2001 From: nkirby-va Date: Tue, 26 Sep 2023 17:54:44 -0400 Subject: [PATCH 27/32] modifty names and bundle target --- Makefile.example | 4 ++-- ...=> drop_pepsq_populate_trigger_from_vbms_ext_claim.rb} | 0 ...> drop_pepsq_populate_trigger_from_vbms_ext_claim.sql} | 0 spec/sql/triggers/populate_end_product_sync_queue_spec.rb | 8 ++++---- 4 files changed, 6 insertions(+), 6 deletions(-) rename db/scripts/{drop_pepsq_populate_trigger_to_vbms_ext_claim.rb => drop_pepsq_populate_trigger_from_vbms_ext_claim.rb} (100%) rename db/scripts/{drop_pepsq_populate_trigger_to_vbms_ext_claim.sql => drop_pepsq_populate_trigger_from_vbms_ext_claim.sql} (100%) diff --git a/Makefile.example b/Makefile.example index fa4fcce7792..45cc6e80d09 100644 --- a/Makefile.example +++ b/Makefile.example @@ -204,11 +204,11 @@ add-populate-pepsq-trigger-test: # Remove populate pepsq trigger from vbms_ext_claim table drop-populate-pepsq-trigger: - bundle exec rails r db/scripts/drop_pepsq_populate_trigger_to_vbms_ext_claim.rb + bundle exec rails r db/scripts/drop_pepsq_populate_trigger_from_vbms_ext_claim.rb # Remove populate pepsq trigger from vbms_ext_claim table drop-populate-pepsq-trigger-test: - bundle exec rails r -e test db/scripts/drop_pepsq_populate_trigger_to_vbms_ext_claim.rb + bundle exec rails r -e test db/scripts/drop_pepsq_populate_trigger_from_vbms_ext_claim.rb c: ## Start rails console bundle exec rails console diff --git a/db/scripts/drop_pepsq_populate_trigger_to_vbms_ext_claim.rb b/db/scripts/drop_pepsq_populate_trigger_from_vbms_ext_claim.rb similarity index 100% rename from db/scripts/drop_pepsq_populate_trigger_to_vbms_ext_claim.rb rename to db/scripts/drop_pepsq_populate_trigger_from_vbms_ext_claim.rb diff --git a/db/scripts/drop_pepsq_populate_trigger_to_vbms_ext_claim.sql b/db/scripts/drop_pepsq_populate_trigger_from_vbms_ext_claim.sql similarity index 100% rename from db/scripts/drop_pepsq_populate_trigger_to_vbms_ext_claim.sql rename to db/scripts/drop_pepsq_populate_trigger_from_vbms_ext_claim.sql diff --git a/spec/sql/triggers/populate_end_product_sync_queue_spec.rb b/spec/sql/triggers/populate_end_product_sync_queue_spec.rb index 38be1d931bd..274ee5df1a5 100644 --- a/spec/sql/triggers/populate_end_product_sync_queue_spec.rb +++ b/spec/sql/triggers/populate_end_product_sync_queue_spec.rb @@ -7,8 +7,8 @@ describe "vbms_ext_claim trigger to populate end_product_sync_que table", :postgres do context "when the trigger is added to the vbms_ext_claim table before the creation new records" do before(:all) do - system("bundle exec rails r -e test db/scripts/drop_pepsq_populate_trigger.rb") - system("bundle exec rails r -e test db/scripts/add_pepsq_populate_trigger.rb") + system("bundle exec rails r -e test db/scripts/drop_pepsq_populate_trigger_from_vbms_ext_claim.rb") + system("bundle exec rails r -e test db/scripts/add_pepsq_populate_trigger_to_vbms_ext_claim.rb") end before do PriorityEndProductSyncQueue.delete_all @@ -164,13 +164,13 @@ context "when the trigger is removed from the vbms_ext_claim table" do before(:all) do - system("bundle exec rails r -e test db/scripts/drop_pepsq_populate_trigger.rb") + system("bundle exec rails r -e test db/scripts/drop_pepsq_populate_trigger_from_vbms_ext_claim.rb") end before do PriorityEndProductSyncQueue.delete_all end after(:all) do - system("bundle exec rails r -e test db/scripts/add_pepsq_populate_trigger.rb") + system("bundle exec rails r -e test db/scripts/add_pepsq_populate_trigger_to_vbms_ext_claim.rb") end let(:logged_epe) { create(:end_product_establishment, :active, reference_id: 300_000) } From b385b533c7e69b0b0ac8bba2621f33cefebef44f Mon Sep 17 00:00:00 2001 From: nkirby-va Date: Wed, 27 Sep 2023 12:07:09 -0400 Subject: [PATCH 28/32] remove extra trigger code and point to trigger rb file --- .../external/create_vbms_ext_claim_table.rb | 48 +------------------ 1 file changed, 2 insertions(+), 46 deletions(-) diff --git a/db/scripts/external/create_vbms_ext_claim_table.rb b/db/scripts/external/create_vbms_ext_claim_table.rb index c327b4c3c2b..ca113bda18e 100644 --- a/db/scripts/external/create_vbms_ext_claim_table.rb +++ b/db/scripts/external/create_vbms_ext_claim_table.rb @@ -44,50 +44,6 @@ conn.execute('CREATE INDEX IF NOT EXISTS claim_id_index ON public.vbms_ext_claim ("CLAIM_ID")') conn.execute('CREATE INDEX IF NOT EXISTS level_status_code_index ON public.vbms_ext_claim ("LEVEL_STATUS_CODE")') - -conn.execute(" - drop trigger if exists update_claim_status_trigger on vbms_ext_claim; - - create or replace function public.update_claim_status_trigger_function() - returns trigger as $$ - declare - string_claim_id varchar(25); - epe_id integer; - begin - if (NEW.\"EP_CODE\" LIKE '04%' - OR NEW.\"EP_CODE\" LIKE '03%' - OR NEW.\"EP_CODE\" LIKE '93%' - OR NEW.\"EP_CODE\" LIKE '68%') - and (NEW.\"LEVEL_STATUS_CODE\" = 'CLR' OR NEW.\"LEVEL_STATUS_CODE\" = 'CAN') then - - string_claim_id := cast(NEW.\"CLAIM_ID\" as varchar); - - select id into epe_id - from end_product_establishments - where (reference_id = string_claim_id - and (synced_status is null or synced_status <> NEW.\"LEVEL_STATUS_CODE\")); - - if epe_id > 0 - then - if not exists ( - select 1 - from priority_end_product_sync_queue - where end_product_establishment_id = epe_id - ) then - insert into priority_end_product_sync_queue (created_at, end_product_establishment_id, updated_at) - values (now(), epe_id, now()); - end if; - end if; - end if; - return null; - end; - $$ - language plpgsql; - - create trigger update_claim_status_trigger - after update or insert on vbms_ext_claim - for each row - execute procedure public.update_claim_status_trigger_function(); - ") - conn.close + +system("bundle exec rails r db/scripts/add_pepsq_populate_trigger_to_vbms_ext_claim.rb") From f22a9faf70a5cee4ce5feaf6c421595f51c1f55e Mon Sep 17 00:00:00 2001 From: nkirby-va Date: Wed, 27 Sep 2023 12:39:41 -0400 Subject: [PATCH 29/32] add bundle call for removing trigger and function --- db/scripts/external/remove_vbms_ext_claim_table.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/db/scripts/external/remove_vbms_ext_claim_table.rb b/db/scripts/external/remove_vbms_ext_claim_table.rb index 192de6a0f15..b954f1889bf 100644 --- a/db/scripts/external/remove_vbms_ext_claim_table.rb +++ b/db/scripts/external/remove_vbms_ext_claim_table.rb @@ -6,3 +6,6 @@ conn.execute( "drop table IF EXISTS public.vbms_ext_claim;" ) +conn.close + +system("bundle exec rails r db/scripts/drop_pepsq_populate_trigger_from_vbms_ext_claim.rb") From 024fce925fc37ecdd7b46beedae9f0140ccf59a1 Mon Sep 17 00:00:00 2001 From: nkirby-va Date: Thu, 28 Sep 2023 11:20:50 -0400 Subject: [PATCH 30/32] remove escaped quotes on sql script --- ...dd_pepsq_populate_trigger_to_vbms_ext_claim.sql | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/db/scripts/add_pepsq_populate_trigger_to_vbms_ext_claim.sql b/db/scripts/add_pepsq_populate_trigger_to_vbms_ext_claim.sql index 3319f7f57e3..aa0908df172 100644 --- a/db/scripts/add_pepsq_populate_trigger_to_vbms_ext_claim.sql +++ b/db/scripts/add_pepsq_populate_trigger_to_vbms_ext_claim.sql @@ -6,18 +6,18 @@ returns trigger as $$ string_claim_id varchar(25); epe_id integer; begin - if (NEW.\"EP_CODE\" LIKE '04%' - OR NEW.\"EP_CODE\" LIKE '03%' - OR NEW.\"EP_CODE\" LIKE '93%' - OR NEW.\"EP_CODE\" LIKE '68%') - and (NEW.\"LEVEL_STATUS_CODE\" = 'CLR' OR NEW.\"LEVEL_STATUS_CODE\" = 'CAN') then + if (NEW."EP_CODE" LIKE '04%' + OR NEW."EP_CODE" LIKE '03%' + OR NEW."EP_CODE" LIKE '93%' + OR NEW."EP_CODE" LIKE '68%') + and (NEW."LEVEL_STATUS_CODE" = 'CLR' OR NEW."LEVEL_STATUS_CODE" = 'CAN') then - string_claim_id := cast(NEW.\"CLAIM_ID\" as varchar); + string_claim_id := cast(NEW."CLAIM_ID" as varchar); select id into epe_id from end_product_establishments where (reference_id = string_claim_id - and (synced_status is null or synced_status <> NEW.\"LEVEL_STATUS_CODE\")); + and (synced_status is null or synced_status <> NEW."LEVEL_STATUS_CODE")); if epe_id > 0 then From c7a0e0fb73ddd9761b614d7feff990e3627fb5cd Mon Sep 17 00:00:00 2001 From: nkirby-va Date: Thu, 28 Sep 2023 11:23:20 -0400 Subject: [PATCH 31/32] remove escaped quotes on sql script --- .../external/create_vbms_ext_claim_table.sql | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/db/scripts/external/create_vbms_ext_claim_table.sql b/db/scripts/external/create_vbms_ext_claim_table.sql index 017b6b9b8cf..fd8306dc9a6 100644 --- a/db/scripts/external/create_vbms_ext_claim_table.sql +++ b/db/scripts/external/create_vbms_ext_claim_table.sql @@ -49,18 +49,18 @@ returns trigger as $$ string_claim_id varchar(25); epe_id integer; begin - if (NEW.\"EP_CODE\" LIKE '04%' - OR NEW.\"EP_CODE\" LIKE '03%' - OR NEW.\"EP_CODE\" LIKE '93%' - OR NEW.\"EP_CODE\" LIKE '68%') - and (NEW.\"LEVEL_STATUS_CODE\" = 'CLR' OR NEW.\"LEVEL_STATUS_CODE\" = 'CAN') then + if (NEW."EP_CODE" LIKE '04%' + OR NEW."EP_CODE" LIKE '03%' + OR NEW."EP_CODE" LIKE '93%' + OR NEW."EP_CODE" LIKE '68%') + and (NEW."LEVEL_STATUS_CODE" = 'CLR' OR NEW."LEVEL_STATUS_CODE" = 'CAN') then - string_claim_id := cast(NEW.\"CLAIM_ID\" as varchar); + string_claim_id := cast(NEW."CLAIM_ID" as varchar); select id into epe_id from end_product_establishments where (reference_id = string_claim_id - and (synced_status is null or synced_status <> NEW.\"LEVEL_STATUS_CODE\")); + and (synced_status is null or synced_status <> NEW."LEVEL_STATUS_CODE")); if epe_id > 0 then From 9b64aa2aeab87ce716c0f82bbbc8eb5ab3850c96 Mon Sep 17 00:00:00 2001 From: nkirby-va Date: Thu, 28 Sep 2023 11:29:59 -0400 Subject: [PATCH 32/32] add proper path and add explicit removal --- spec/sql/triggers/populate_end_product_sync_queue_spec.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/spec/sql/triggers/populate_end_product_sync_queue_spec.rb b/spec/sql/triggers/populate_end_product_sync_queue_spec.rb index 274ee5df1a5..8569f31d90a 100644 --- a/spec/sql/triggers/populate_end_product_sync_queue_spec.rb +++ b/spec/sql/triggers/populate_end_product_sync_queue_spec.rb @@ -98,7 +98,8 @@ before(:all) do @logged_epe = create(:end_product_establishment, :active, reference_id: 300_000) @logged_ext_claim = create(:vbms_ext_claim, :rdc, :slc, id: 300_000) - system("bundle exec rails r -e test db/scripts/add_pepsq_populate_trigger.rb") + system("bundle exec rails r -e test db/scripts/drop_pepsq_populate_trigger_from_vbms_ext_claim.rb") + system("bundle exec rails r -e test db/scripts/add_pepsq_populate_trigger_to_vbms_ext_claim.rb") end before do PriorityEndProductSyncQueue.delete_all