diff --git a/Makefile.example b/Makefile.example index 3d187a5428f..45cc6e80d09 100644 --- a/Makefile.example +++ b/Makefile.example @@ -194,6 +194,22 @@ remove-vbms-ext-claim-seeds: ## Drops audit tables, removes all PriorityEndProdu reseed-vbms-ext-claim: remove-vbms-ext-claim-seeds seed-vbms-ext-claim ## Re-seeds database with records created from seed-vbms-ext-claim +# Add trigger to vbms_ext_claim to populate pepsq table +add-populate-pepsq-trigger: + bundle exec rails r db/scripts/add_pepsq_populate_trigger_to_vbms_ext_claim.rb + +# Add trigger to vbms_ext_claim to populate pepsq table +add-populate-pepsq-trigger-test: + bundle exec rails r -e test db/scripts/add_pepsq_populate_trigger_to_vbms_ext_claim.rb + +# Remove populate pepsq trigger from vbms_ext_claim table +drop-populate-pepsq-trigger: + bundle exec rails r db/scripts/drop_pepsq_populate_trigger_from_vbms_ext_claim.rb + +# Remove populate pepsq trigger from vbms_ext_claim table +drop-populate-pepsq-trigger-test: + bundle exec rails r -e test db/scripts/drop_pepsq_populate_trigger_from_vbms_ext_claim.rb + c: ## Start rails console bundle exec rails console diff --git a/app/jobs/populate_end_product_sync_queue_job.rb b/app/jobs/populate_end_product_sync_queue_job.rb deleted file mode 100644 index 13d537fab01..00000000000 --- a/app/jobs/populate_end_product_sync_queue_job.rb +++ /dev/null @@ -1,102 +0,0 @@ -# frozen_string_literal: true - -# This job will find deltas between the end product establishment table and the VBMS ext claim table -# where VBMS ext claim level status code is CLR or CAN. If EP is already in the queue it will be skipped. -# Job will populate queue ENV["END_PRODUCT_QUEUE_BATCH_LIMIT"] records at a time. -# This job will run on a 50 minute loop, sleeping for 5 seconds between iterations. -class PopulateEndProductSyncQueueJob < CaseflowJob - queue_with_priority :low_priority - - JOB_DURATION ||= ENV["END_PRODUCT_QUEUE_JOB_DURATION"].to_i.minutes - SLEEP_DURATION ||= ENV["END_PRODUCT_QUEUE_SLEEP_DURATION"].to_i - BATCH_LIMIT ||= ENV["END_PRODUCT_QUEUE_BATCH_LIMIT"].to_i - - # rubocop:disable Metrics/CyclomaticComplexity - def perform - setup_job - loop do - break if job_running_past_expected_end_time? || should_stop_job - - begin - batch = ActiveRecord::Base.transaction do - priority_epes = find_priority_end_product_establishments_to_sync - next if priority_epes.empty? - - priority_epes - end - - batch ? insert_into_priority_sync_queue(batch) : stop_job(log_no_records_found: true) - - sleep(SLEEP_DURATION) - rescue StandardError => error - log_error(error, extra: { active_job_id: job_id.to_s, job_time: Time.zone.now.to_s }) - slack_msg = "Error running #{self.class.name}. Error: #{error.message}. Active Job ID: #{job_id}." - slack_msg += " See Sentry event #{Raven.last_event_id}." if Raven.last_event_id.present? - slack_service.send_notification("[ERROR] #{slack_msg}", self.class.to_s) - stop_job - end - end - end - # rubocop:enable Metrics/CyclomaticComplexity - - private - - attr_accessor :job_expected_end_time, :should_stop_job - - # rubocop:disable Metrics/MethodLength - def find_priority_end_product_establishments_to_sync - get_sql = <<-SQL - WITH priority_eps AS ( - SELECT vec."CLAIM_ID"::varchar, vec."LEVEL_STATUS_CODE" - FROM vbms_ext_claim vec - WHERE vec."LEVEL_STATUS_CODE" in ('CLR', 'CAN') - AND (vec."EP_CODE" LIKE '04%' OR vec."EP_CODE" LIKE '03%' OR vec."EP_CODE" LIKE '93%' OR vec."EP_CODE" LIKE '68%') - ), - priority_queued_epe_ids AS ( - SELECT end_product_establishment_id - FROM priority_end_product_sync_queue) - SELECT id - FROM end_product_establishments epe - INNER JOIN priority_eps - ON epe.reference_id = priority_eps."CLAIM_ID" - WHERE (epe.synced_status is null or epe.synced_status <> priority_eps."LEVEL_STATUS_CODE") - AND NOT EXISTS (SELECT end_product_establishment_id - FROM priority_queued_epe_ids - WHERE priority_queued_epe_ids.end_product_establishment_id = epe.id) - LIMIT #{BATCH_LIMIT}; - SQL - - ActiveRecord::Base.connection.exec_query(ActiveRecord::Base.sanitize_sql(get_sql)).rows.flatten - end - # rubocop:enable Metrics/MethodLength - - def insert_into_priority_sync_queue(batch) - priority_end_product_sync_queue_records = batch.map do |ep_id| - PriorityEndProductSyncQueue.new(end_product_establishment_id: ep_id) - end - - # Bulk insert PriorityEndProductSyncQueue records in a single SQL statement - PriorityEndProductSyncQueue.import(priority_end_product_sync_queue_records) - Rails.logger.info("PopulateEndProductSyncQueueJob EPEs processed: #{batch} - Time: #{Time.zone.now}") - end - - def setup_job - RequestStore.store[:current_user] = User.system_user - @should_stop_job = false - @job_expected_end_time = Time.zone.now + JOB_DURATION - end - - def job_running_past_expected_end_time? - Time.zone.now > job_expected_end_time - end - - # :reek:BooleanParameter - # :reek:ControlParameter - def stop_job(log_no_records_found: false) - self.should_stop_job = true - if log_no_records_found - Rails.logger.info("PopulateEndProductSyncQueueJob is not able to find any batchable EPE records."\ - " Active Job ID: #{job_id}. Time: #{Time.zone.now}") - end - end -end diff --git a/app/models/batch_processes/priority_ep_sync_batch_process.rb b/app/models/batch_processes/priority_ep_sync_batch_process.rb index 3b5b41196d4..3222f7d2b66 100644 --- a/app/models/batch_processes/priority_ep_sync_batch_process.rb +++ b/app/models/batch_processes/priority_ep_sync_batch_process.rb @@ -87,7 +87,6 @@ def assign_batch_to_queued_records!(records) private # Purpose: Destroys "SYNCED" PEPSQ records to limit the growing number of table records. - # This functionality is needed for the PopulateEndProductSyncQueueJob query to be performant. # # Params: None # diff --git a/app/models/priority_queues/priority_end_product_sync_queue.rb b/app/models/priority_queues/priority_end_product_sync_queue.rb index d8f68cd1adb..b15259373db 100644 --- a/app/models/priority_queues/priority_end_product_sync_queue.rb +++ b/app/models/priority_queues/priority_end_product_sync_queue.rb @@ -2,6 +2,11 @@ # Model for Priority End Product Sync Queue table. # This table consists of records of End Product Establishment IDs that need to be synced with VBMS. + +# These are populated via the trigger that is created on creation of the vbms_ext_claim table +# The trigger is located in: +# db/scripts/external/create_vbms_ext_claim_table.rb +# db/scripts/ class PriorityEndProductSyncQueue < CaseflowRecord self.table_name = "priority_end_product_sync_queue" @@ -52,7 +57,6 @@ def declare_record_stuck! end # Purpose: Destroys "SYNCED" PEPSQ records to limit the growing number of table records. - # This functionality is needed for the PopulateEndProductSyncQueueJob query to be performant. # # Params: The batch process the synced records belong to # diff --git a/config/environments/demo.rb b/config/environments/demo.rb index 1815b62083b..2215958bce8 100644 --- a/config/environments/demo.rb +++ b/config/environments/demo.rb @@ -93,11 +93,6 @@ ENV["BATCH_PROCESS_ERROR_DELAY"] ||= "12" # In number of hours ENV["BATCH_PROCESS_MAX_ERRORS_BEFORE_STUCK"] ||= "3" # When record errors for X time, it's declared stuck - # Populate End Product Sync Queue ENVs - ENV["END_PRODUCT_QUEUE_JOB_DURATION"] ||= "1" # Number of hours the job will run for - ENV["END_PRODUCT_QUEUE_SLEEP_DURATION"] ||= "5" # Number of seconds between loop iterations - ENV["END_PRODUCT_QUEUE_BATCH_LIMIT"] ||= "500" # Max number of records in a batch - # Setup S3 config.s3_enabled = ENV["AWS_BUCKET_NAME"].present? config.s3_bucket_name = ENV["AWS_BUCKET_NAME"] diff --git a/config/environments/development.rb b/config/environments/development.rb index 9822cd7b692..6c2dfb02f7f 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -100,11 +100,6 @@ # Quarterly Notifications Batch Sizes ENV["QUARTERLY_NOTIFICATIONS_JOB_BATCH_SIZE"] ||= "1000" - # Populate End Product Sync Queue ENVs - ENV["END_PRODUCT_QUEUE_JOB_DURATION"] ||= "50" # Number of minutes the job will run for - ENV["END_PRODUCT_QUEUE_SLEEP_DURATION"] ||= "5" # Number of seconds between loop iterations - ENV["END_PRODUCT_QUEUE_BATCH_LIMIT"] ||= "500" # Max number of records in a batch - # Travel Board Sync Batch Size ENV["TRAVEL_BOARD_HEARING_SYNC_BATCH_LIMIT"] ||= "250" diff --git a/config/environments/test.rb b/config/environments/test.rb index fff81c6c7a2..426c6f8fb64 100644 --- a/config/environments/test.rb +++ b/config/environments/test.rb @@ -115,11 +115,6 @@ # Quarterly Notifications Batch Sizes ENV["QUARTERLY_NOTIFICATIONS_JOB_BATCH_SIZE"] ||= "1000" - # Populate End Product Sync Queue ENVs - ENV["END_PRODUCT_QUEUE_JOB_DURATION"] ||= "50" # Number of minutes the job will run for - ENV["END_PRODUCT_QUEUE_SLEEP_DURATION"] ||= "0" # Number of seconds between loop iterations - ENV["END_PRODUCT_QUEUE_BATCH_LIMIT"] ||= "250" # Max number of records in a batch - # Travel Board Sync Batch Size ENV["TRAVEL_BOARD_HEARING_SYNC_BATCH_LIMIT"] ||= "250" diff --git a/config/initializers/scheduled_jobs.rb b/config/initializers/scheduled_jobs.rb index 241e6f6d6d3..7662c421fed 100644 --- a/config/initializers/scheduled_jobs.rb +++ b/config/initializers/scheduled_jobs.rb @@ -23,7 +23,6 @@ "monthly_metrics" => MonthlyMetricsReportJob, "nightly_syncs" => NightlySyncsJob, "out_of_service_reminder" => OutOfServiceReminderJob, - "populate_end_product_sync_queue" => PopulateEndProductSyncQueueJob, "prepare_establish_claim" => PrepareEstablishClaimTasksJob, "push_priority_appeals_to_judges" => PushPriorityAppealsToJudgesJob, "quarterly_metrics" => QuarterlyMetricsReportJob, diff --git a/db/scripts/add_pepsq_populate_trigger_to_vbms_ext_claim.rb b/db/scripts/add_pepsq_populate_trigger_to_vbms_ext_claim.rb new file mode 100644 index 00000000000..e68c03822b2 --- /dev/null +++ b/db/scripts/add_pepsq_populate_trigger_to_vbms_ext_claim.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +require "pg" + +conn = CaseflowRecord.connection +conn.execute(" + drop trigger if exists update_claim_status_trigger on vbms_ext_claim; + + create or replace function public.update_claim_status_trigger_function() + returns trigger as $$ + declare + string_claim_id varchar(25); + epe_id integer; + begin + if (NEW.\"EP_CODE\" LIKE '04%' + OR NEW.\"EP_CODE\" LIKE '03%' + OR NEW.\"EP_CODE\" LIKE '93%' + OR NEW.\"EP_CODE\" LIKE '68%') + and (NEW.\"LEVEL_STATUS_CODE\" = 'CLR' OR NEW.\"LEVEL_STATUS_CODE\" = 'CAN') then + + string_claim_id := cast(NEW.\"CLAIM_ID\" as varchar); + + select id into epe_id + from end_product_establishments + where (reference_id = string_claim_id + and (synced_status is null or synced_status <> NEW.\"LEVEL_STATUS_CODE\")); + + if epe_id > 0 + then + if not exists ( + select 1 + from priority_end_product_sync_queue + where end_product_establishment_id = epe_id + ) then + insert into priority_end_product_sync_queue (created_at, end_product_establishment_id, updated_at) + values (now(), epe_id, now()); + end if; + end if; + end if; + return null; + end; + $$ + language plpgsql; + + create trigger update_claim_status_trigger + after update or insert on vbms_ext_claim + for each row + execute procedure public.update_claim_status_trigger_function(); + ") + +conn.close diff --git a/db/scripts/add_pepsq_populate_trigger_to_vbms_ext_claim.sql b/db/scripts/add_pepsq_populate_trigger_to_vbms_ext_claim.sql new file mode 100644 index 00000000000..aa0908df172 --- /dev/null +++ b/db/scripts/add_pepsq_populate_trigger_to_vbms_ext_claim.sql @@ -0,0 +1,42 @@ +drop trigger if exists update_claim_status_trigger on vbms_ext_claim; + +create or replace function public.update_claim_status_trigger_function() +returns trigger as $$ + declare + string_claim_id varchar(25); + epe_id integer; + begin + if (NEW."EP_CODE" LIKE '04%' + OR NEW."EP_CODE" LIKE '03%' + OR NEW."EP_CODE" LIKE '93%' + OR NEW."EP_CODE" LIKE '68%') + and (NEW."LEVEL_STATUS_CODE" = 'CLR' OR NEW."LEVEL_STATUS_CODE" = 'CAN') then + + string_claim_id := cast(NEW."CLAIM_ID" as varchar); + + select id into epe_id + from end_product_establishments + where (reference_id = string_claim_id + and (synced_status is null or synced_status <> NEW."LEVEL_STATUS_CODE")); + + if epe_id > 0 + then + if not exists ( + select 1 + from priority_end_product_sync_queue + where end_product_establishment_id = epe_id + ) then + insert into priority_end_product_sync_queue (created_at, end_product_establishment_id, updated_at) + values (now(), epe_id, now()); + end if; + end if; + end if; + return null; + end; +$$ +language plpgsql; + +create trigger update_claim_status_trigger +after update or insert on vbms_ext_claim +for each row +execute procedure public.update_claim_status_trigger_function(); diff --git a/db/scripts/drop_pepsq_populate_trigger_from_vbms_ext_claim.rb b/db/scripts/drop_pepsq_populate_trigger_from_vbms_ext_claim.rb new file mode 100644 index 00000000000..8173564ce36 --- /dev/null +++ b/db/scripts/drop_pepsq_populate_trigger_from_vbms_ext_claim.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +require "pg" + +conn = CaseflowRecord.connection +conn.execute(" + drop trigger if exists update_claim_status_trigger on vbms_ext_claim; + drop function if exists public.update_claim_status_trigger_function(); + ") + +conn.close diff --git a/db/scripts/drop_pepsq_populate_trigger_from_vbms_ext_claim.sql b/db/scripts/drop_pepsq_populate_trigger_from_vbms_ext_claim.sql new file mode 100644 index 00000000000..97b50402069 --- /dev/null +++ b/db/scripts/drop_pepsq_populate_trigger_from_vbms_ext_claim.sql @@ -0,0 +1,2 @@ +drop trigger if exists update_claim_status_trigger on vbms_ext_claim; +drop function if exists public.update_claim_status_trigger_function(); diff --git a/db/scripts/external/create_vbms_ext_claim_table.rb b/db/scripts/external/create_vbms_ext_claim_table.rb index 3a1a37e2470..ca113bda18e 100644 --- a/db/scripts/external/create_vbms_ext_claim_table.rb +++ b/db/scripts/external/create_vbms_ext_claim_table.rb @@ -45,3 +45,5 @@ conn.execute('CREATE INDEX IF NOT EXISTS claim_id_index ON public.vbms_ext_claim ("CLAIM_ID")') conn.execute('CREATE INDEX IF NOT EXISTS level_status_code_index ON public.vbms_ext_claim ("LEVEL_STATUS_CODE")') conn.close + +system("bundle exec rails r db/scripts/add_pepsq_populate_trigger_to_vbms_ext_claim.rb") diff --git a/db/scripts/external/create_vbms_ext_claim_table.sql b/db/scripts/external/create_vbms_ext_claim_table.sql index 02a6e0f356e..fd8306dc9a6 100644 --- a/db/scripts/external/create_vbms_ext_claim_table.sql +++ b/db/scripts/external/create_vbms_ext_claim_table.sql @@ -40,3 +40,46 @@ CREATE TABLE IF NOT EXISTS PUBLIC.VBMS_EXT_CLAIM ( CREATE INDEX IF NOT EXISTS CLAIM_ID_INDEX ON PUBLIC.VBMS_EXT_CLAIM ("CLAIM_ID"); CREATE INDEX IF NOT EXISTS LEVEL_STATUS_CODE_INDEX ON PUBLIC.VBMS_EXT_CLAIM ("LEVEL_STATUS_CODE"); + +drop trigger if exists update_claim_status_trigger on vbms_ext_claim; + +create or replace function public.update_claim_status_trigger_function() +returns trigger as $$ + declare + string_claim_id varchar(25); + epe_id integer; + begin + if (NEW."EP_CODE" LIKE '04%' + OR NEW."EP_CODE" LIKE '03%' + OR NEW."EP_CODE" LIKE '93%' + OR NEW."EP_CODE" LIKE '68%') + and (NEW."LEVEL_STATUS_CODE" = 'CLR' OR NEW."LEVEL_STATUS_CODE" = 'CAN') then + + string_claim_id := cast(NEW."CLAIM_ID" as varchar); + + select id into epe_id + from end_product_establishments + where (reference_id = string_claim_id + and (synced_status is null or synced_status <> NEW."LEVEL_STATUS_CODE")); + + if epe_id > 0 + then + if not exists ( + select 1 + from priority_end_product_sync_queue + where end_product_establishment_id = epe_id + ) then + insert into priority_end_product_sync_queue (created_at, end_product_establishment_id, updated_at) + values (now(), epe_id, now()); + end if; + end if; + end if; + return null; + end; +$$ +language plpgsql; + +create trigger update_claim_status_trigger +after update or insert on vbms_ext_claim +for each row +execute procedure public.update_claim_status_trigger_function(); diff --git a/db/scripts/external/remove_vbms_ext_claim_table.rb b/db/scripts/external/remove_vbms_ext_claim_table.rb index 192de6a0f15..b954f1889bf 100644 --- a/db/scripts/external/remove_vbms_ext_claim_table.rb +++ b/db/scripts/external/remove_vbms_ext_claim_table.rb @@ -6,3 +6,6 @@ conn.execute( "drop table IF EXISTS public.vbms_ext_claim;" ) +conn.close + +system("bundle exec rails r db/scripts/drop_pepsq_populate_trigger_from_vbms_ext_claim.rb") diff --git a/spec/factories/end_product_establishment.rb b/spec/factories/end_product_establishment.rb index 2aee01d206e..cfaa3bf5332 100644 --- a/spec/factories/end_product_establishment.rb +++ b/spec/factories/end_product_establishment.rb @@ -46,7 +46,7 @@ active_hlr modifier { "030" } code { "030HLRR" } - after(:build) do |end_product_establishment, _evaluator| + after(:create) do |end_product_establishment, _evaluator| create(:vbms_ext_claim, :hlr, :canceled, claim_id: end_product_establishment.reference_id) ep = end_product_establishment.result ep_store = Fakes::EndProductStore.new @@ -59,7 +59,7 @@ active_hlr modifier { "030" } code { "030HLRR" } - after(:build) do |end_product_establishment, _evaluator| + after(:create) do |end_product_establishment, _evaluator| create(:vbms_ext_claim, :hlr, :rdc, claim_id: end_product_establishment.reference_id) ep = end_product_establishment.result ep_store = Fakes::EndProductStore.new @@ -72,7 +72,7 @@ active_hlr modifier { "030" } code { "030HLRR" } - after(:build) do |end_product_establishment, _evaluator| + after(:create) do |end_product_establishment, _evaluator| create(:vbms_ext_claim, :hlr, :cleared, claim_id: end_product_establishment.reference_id) ep = end_product_establishment.result ep_store = Fakes::EndProductStore.new @@ -87,7 +87,7 @@ modifier { "030" } code { "030HLRR" } source { create(:higher_level_review, veteran_file_number: veteran_file_number) } - after(:build) do |end_product_establishment, _evaluator| + after(:create) do |end_product_establishment, _evaluator| create(:vbms_ext_claim, :hlr, :canceled, claim_id: end_product_establishment.reference_id) ep = end_product_establishment.result ep_store = Fakes::EndProductStore.new @@ -115,7 +115,7 @@ active_supp modifier { "040" } code { "040SCR" } - after(:build) do |end_product_establishment, _evaluator| + after(:create) do |end_product_establishment, _evaluator| create(:vbms_ext_claim, :slc, :canceled, claim_id: end_product_establishment.reference_id) ep = end_product_establishment.result ep_store = Fakes::EndProductStore.new @@ -128,7 +128,7 @@ active_supp modifier { "040" } code { "040SCR" } - after(:build) do |end_product_establishment, _evaluator| + after(:create) do |end_product_establishment, _evaluator| create(:vbms_ext_claim, :slc, :rdc, claim_id: end_product_establishment.reference_id) ep = end_product_establishment.result ep_store = Fakes::EndProductStore.new @@ -141,7 +141,7 @@ active_supp modifier { "040" } code { "040SCR" } - after(:build) do |end_product_establishment, _evaluator| + after(:create) do |end_product_establishment, _evaluator| create(:vbms_ext_claim, :slc, :cleared, claim_id: end_product_establishment.reference_id) ep = end_product_establishment.result ep_store = Fakes::EndProductStore.new @@ -156,7 +156,7 @@ modifier { "040" } code { "040SCR" } source { create(:supplemental_claim, veteran_file_number: veteran_file_number) } - after(:build) do |end_product_establishment, _evaluator| + after(:create) do |end_product_establishment, _evaluator| create(:vbms_ext_claim, :slc, :canceled, claim_id: end_product_establishment.reference_id) ep = end_product_establishment.result ep_store = Fakes::EndProductStore.new @@ -171,7 +171,7 @@ modifier { "040" } code { "040SCR" } source { create(:supplemental_claim, veteran_file_number: veteran_file_number) } - after(:build) do |end_product_establishment, _evaluator| + after(:create) do |end_product_establishment, _evaluator| create(:vbms_ext_claim, :slc, :cleared, claim_id: end_product_establishment.reference_id) ep = end_product_establishment.result ep_store = Fakes::EndProductStore.new @@ -186,7 +186,7 @@ modifier { "030" } code { "030HLRR" } source { create(:higher_level_review, veteran_file_number: veteran_file_number) } - after(:build) do |end_product_establishment, _evaluator| + after(:create) do |end_product_establishment, _evaluator| create(:vbms_ext_claim, :hlr, :cleared, claim_id: end_product_establishment.reference_id) ep = end_product_establishment.result ep_store = Fakes::EndProductStore.new diff --git a/spec/jobs/batch_processes/batch_process_rescue_job_spec.rb b/spec/jobs/batch_processes/batch_process_rescue_job_spec.rb index 1cc6bc6c23f..3469ce51e2d 100644 --- a/spec/jobs/batch_processes/batch_process_rescue_job_spec.rb +++ b/spec/jobs/batch_processes/batch_process_rescue_job_spec.rb @@ -17,10 +17,6 @@ create_list(:end_product_establishment, 2, :active_hlr_with_cleared_vbms_ext_claim) end - let!(:pepsq_records_one) do - PopulateEndProductSyncQueueJob.perform_now - end - let!(:first_batch_process) do PriorityEpSyncBatchProcessJob.perform_now end @@ -29,10 +25,6 @@ create_list(:end_product_establishment, 2, :active_hlr_with_cleared_vbms_ext_claim) end - let!(:pepsq_records_two) do - PopulateEndProductSyncQueueJob.perform_now - end - let!(:second_batch_process) do PriorityEpSyncBatchProcessJob.perform_now end diff --git a/spec/jobs/batch_processes/priority_ep_sync_batch_process_job_spec.rb b/spec/jobs/batch_processes/priority_ep_sync_batch_process_job_spec.rb index be010324dfe..8d00d31b9d2 100644 --- a/spec/jobs/batch_processes/priority_ep_sync_batch_process_job_spec.rb +++ b/spec/jobs/batch_processes/priority_ep_sync_batch_process_job_spec.rb @@ -22,7 +22,6 @@ end let!(:pepsq_records) do - PopulateEndProductSyncQueueJob.perform_now PriorityEndProductSyncQueue.all end diff --git a/spec/jobs/priority_queues/populate_end_product_sync_queue_job_spec.rb b/spec/jobs/priority_queues/populate_end_product_sync_queue_job_spec.rb deleted file mode 100644 index cd06a95aa5f..00000000000 --- a/spec/jobs/priority_queues/populate_end_product_sync_queue_job_spec.rb +++ /dev/null @@ -1,226 +0,0 @@ -# frozen_string_literal: true - -describe PopulateEndProductSyncQueueJob, type: :job do - include ActiveJob::TestHelper - - let(:slack_service) { SlackService.new(url: "http://www.example.com") } - - let!(:epes_to_be_queued) do - create_list(:end_product_establishment, 2, :active_hlr_with_cleared_vbms_ext_claim) - end - - let!(:not_found_epe) do - create(:end_product_establishment, :active_hlr_with_active_vbms_ext_claim) - end - - before do - # Batch limit changes to 1 to test PopulateEndProductSyncQueueJob loop - stub_const("PopulateEndProductSyncQueueJob::BATCH_LIMIT", 1) - end - - subject do - PopulateEndProductSyncQueueJob.perform_later - end - - describe "#perform" do - context "when all records sync successfully" do - before do - allow(SlackService).to receive(:new).with(url: anything).and_return(slack_service) - allow(slack_service).to receive(:send_notification) { |_, first_arg| @slack_msg = first_arg } - perform_enqueued_jobs do - subject - end - end - - it "adds the 2 unsynced epes to the end product synce queue" do - expect(PriorityEndProductSyncQueue.count).to eq 2 - end - - it "the current user is set to a system user" do - expect(RequestStore.store[:current_user].id).to eq(User.system_user.id) - end - - it "adds the epes to the priority end product sync queue table" do - expect(PriorityEndProductSyncQueue.first.end_product_establishment_id).to eq epes_to_be_queued.first.id - expect(PriorityEndProductSyncQueue.second.end_product_establishment_id).to eq epes_to_be_queued.second.id - end - - it "the epes are associated with a vbms_ext_claim record" do - expect(EndProductEstablishment.find(PriorityEndProductSyncQueue.first.end_product_establishment_id) - .reference_id).to eq epes_to_be_queued.first.vbms_ext_claim.claim_id.to_s - expect(EndProductEstablishment.find(PriorityEndProductSyncQueue.second.end_product_establishment_id) - .reference_id).to eq epes_to_be_queued.second.vbms_ext_claim.claim_id.to_s - end - - it "the priority end product sync queue records have a status of 'NOT_PROCESSED'" do - expect(PriorityEndProductSyncQueue.first.status).to eq "NOT_PROCESSED" - expect(PriorityEndProductSyncQueue.second.status).to eq "NOT_PROCESSED" - end - - it "slack will NOT be notified when job runs successfully" do - expect(slack_service).to_not have_received(:send_notification) - end - end - - context "when the epe's reference id is a lettered string (i.e. only match on matching numbers)" do - before do - epes_to_be_queued.each { |epe| epe.update!(reference_id: "whaddup yooo") } - perform_enqueued_jobs do - subject - end - end - - it "doesn't add epe to the queue" do - expect(PriorityEndProductSyncQueue.count).to eq 0 - end - end - - context "when a priority end product sync queue record already exists with the epe id" do - before do - PriorityEndProductSyncQueue.create(end_product_establishment_id: epes_to_be_queued.first.id) - perform_enqueued_jobs do - subject - end - end - - it "will not add same epe more than once in the priorty end product sync queue table" do - expect(PriorityEndProductSyncQueue.count).to eq 2 - end - end - - context "when the epe records' synced_status value is nil" do - before do - epes_to_be_queued.each { |epe| epe.update!(synced_status: nil) } - perform_enqueued_jobs do - subject - end - end - - it "will add the epe if epe synced status is nil and other conditions are met" do - expect(PriorityEndProductSyncQueue.count).to eq 2 - end - end - - context "when the job duration ends before all PriorityEndProductSyncQueue records can be batched" do - before do - # Job duration of 0.001 seconds limits the job's loop to one iteration - stub_const("PopulateEndProductSyncQueueJob::JOB_DURATION", 0.001.seconds) - allow(SlackService).to receive(:new).with(url: anything).and_return(slack_service) - allow(slack_service).to receive(:send_notification) { |_, first_arg| @slack_msg = first_arg } - perform_enqueued_jobs do - subject - end - end - - it "there are 3 epe records" do - expect(EndProductEstablishment.count).to eq(3) - end - - it "creates 1 priority end product sync queue record" do - expect(PriorityEndProductSyncQueue.count).to eq(1) - end - - it "the current user is set to a system user" do - expect(RequestStore.store[:current_user].id).to eq(User.system_user.id) - end - - it "adds the epes to the priority end product sync queue table" do - expect(PriorityEndProductSyncQueue.first.end_product_establishment_id).to eq epes_to_be_queued.first.id - end - - it "the epes are associated with a vbms_ext_claim record" do - expect(EndProductEstablishment.find(PriorityEndProductSyncQueue.first.end_product_establishment_id) - .reference_id).to eq epes_to_be_queued.first.vbms_ext_claim.claim_id.to_s - end - - it "the priority end product sync queue record has a status of 'NOT_PROCESSED'" do - expect(PriorityEndProductSyncQueue.first.status).to eq "NOT_PROCESSED" - end - - it "slack will NOT be notified when job runs successfully" do - expect(slack_service).to_not have_received(:send_notification) - end - end - - context "when there are no records available to batch" do - before do - EndProductEstablishment.destroy_all - allow(Rails.logger).to receive(:info) - perform_enqueued_jobs do - subject - end - end - - it "doesn't add any epes to the batch" do - expect(PriorityEndProductSyncQueue.count).to eq 0 - end - - it "logs a message that says 'PopulateEndProductSyncQueueJob is not able to find any batchable EPE records'" do - expect(Rails.logger).to have_received(:info).with( - "PopulateEndProductSyncQueueJob is not able to find any batchable EPE records."\ - " Active Job ID: #{subject.job_id}."\ - " Time: #{Time.zone.now}" - ) - end - end - - context "when an error is raised during the job" do - let(:standard_error) { StandardError.new("Uh-Oh!") } - before do - allow(Rails.logger).to receive(:error) - allow(Raven).to receive(:capture_exception) - allow(Raven).to receive(:last_event_id) { "sentry_123" } - allow(SlackService).to receive(:new).with(url: anything).and_return(slack_service) - allow(slack_service).to receive(:send_notification) { |_, first_arg| @slack_msg = first_arg } - allow_any_instance_of(PopulateEndProductSyncQueueJob) - .to receive(:find_priority_end_product_establishments_to_sync).and_raise(standard_error) - perform_enqueued_jobs do - subject - end - end - - it "the error and the backtrace will be logged" do - expect(Rails.logger).to have_received(:error).with(an_instance_of(StandardError)) - end - - it "the error will be sent to Sentry" do - expect(Raven).to have_received(:capture_exception) - .with(instance_of(StandardError), - extra: { - active_job_id: subject.job_id, - job_time: Time.zone.now.to_s - }) - end - - it "slack will be notified when job fails" do - expect(slack_service).to have_received(:send_notification).with( - "[ERROR] Error running PopulateEndProductSyncQueueJob. Error: #{standard_error.message}."\ - " Active Job ID: #{subject.job_id}. See Sentry event sentry_123.", "PopulateEndProductSyncQueueJob" - ) - end - end - - context "when there are no records available to batch" do - before do - VbmsExtClaim.destroy_all - allow(Rails.logger).to receive(:info) - allow(SlackService).to receive(:new).with(url: anything).and_return(slack_service) - allow(slack_service).to receive(:send_notification) { |_, first_arg| @slack_msg = first_arg } - perform_enqueued_jobs do - subject - end - end - - it "a message that says 'Cannot Find Any Records to Batch' will be logged" do - expect(Rails.logger).to have_received(:info).with( - "PopulateEndProductSyncQueueJob is not able to find any batchable EPE records."\ - " Active Job ID: #{subject.job_id}. Time: #{Time.zone.now}" - ) - end - - it "slack will NOT be notified when job runs successfully" do - expect(slack_service).to_not have_received(:send_notification) - end - end - end -end diff --git a/spec/models/batch_processes/priority_ep_sync_batch_process_spec.rb b/spec/models/batch_processes/priority_ep_sync_batch_process_spec.rb index 23268ca59ad..216c8953d46 100644 --- a/spec/models/batch_processes/priority_ep_sync_batch_process_spec.rb +++ b/spec/models/batch_processes/priority_ep_sync_batch_process_spec.rb @@ -134,7 +134,7 @@ create(:end_product_establishment, :active_supp_with_active_vbms_ext_claim) end let!(:active_supp_epe_w_cleared_vbms_ext_claim) do - create(:end_product_establishment, :active_supp_with_canceled_vbms_ext_claim) + create(:end_product_establishment, :active_supp_with_cleared_vbms_ext_claim) end let!(:cleared_supp_epes_w_cleared_vbms_ext_claim) do create(:end_product_establishment, :cleared_supp_with_cleared_vbms_ext_claim) @@ -145,7 +145,6 @@ end let!(:pepsq_records) do - PopulateEndProductSyncQueueJob.perform_now PriorityEndProductSyncQueue.all end @@ -306,6 +305,7 @@ context "when priority_ep_sync_batch_process destroys synced pepsq records" do before do allow(Rails.logger).to receive(:info) + pepsq_records.reload subject end diff --git a/spec/models/caseflow_stuck_record_spec.rb b/spec/models/caseflow_stuck_record_spec.rb index a6504482604..a642585ef26 100644 --- a/spec/models/caseflow_stuck_record_spec.rb +++ b/spec/models/caseflow_stuck_record_spec.rb @@ -7,8 +7,6 @@ end let!(:caseflow_stuck_record) do - PopulateEndProductSyncQueueJob.perform_now - 3.times do PriorityEndProductSyncQueue.first.update!(last_batched_at: nil) PriorityEpSyncBatchProcessJob.perform_now diff --git a/spec/sql/triggers/populate_end_product_sync_queue_spec.rb b/spec/sql/triggers/populate_end_product_sync_queue_spec.rb new file mode 100644 index 00000000000..8569f31d90a --- /dev/null +++ b/spec/sql/triggers/populate_end_product_sync_queue_spec.rb @@ -0,0 +1,194 @@ +# frozen_string_literal: true + +# The PriorityEndProductSyncQue is populated via the trigger that is created on creation of the vbms_ext_claim table +# The trigger is located in: +# db/scripts/external/create_vbms_ext_claim_table.rb +# db/scripts/ +describe "vbms_ext_claim trigger to populate end_product_sync_que table", :postgres do + context "when the trigger is added to the vbms_ext_claim table before the creation new records" do + before(:all) do + system("bundle exec rails r -e test db/scripts/drop_pepsq_populate_trigger_from_vbms_ext_claim.rb") + system("bundle exec rails r -e test db/scripts/add_pepsq_populate_trigger_to_vbms_ext_claim.rb") + end + before do + PriorityEndProductSyncQueue.delete_all + end + + context "we only log inserted vbms_ext_claims" do + let(:logged_epe1) { create(:end_product_establishment, :active, reference_id: 300_000) } + let(:logged_ext_claim1) { create(:vbms_ext_claim, :cleared, :slc, id: 300_000) } + + it "that have a \"04%\" EP_CODE, that are cleared, + different sync status, and are not in pepsq table" do + logged_epe1 + logged_ext_claim1 + expect(PriorityEndProductSyncQueue.count).to eq 1 + expect(PriorityEndProductSyncQueue.first.end_product_establishment_id).to eq logged_epe1.id + end + + let(:logged_epe2) { create(:end_product_establishment, synced_status: nil, reference_id: 300_000) } + let(:logged_ext_claim2) { create(:vbms_ext_claim, :canceled, :hlr, id: 300_000) } + + it "that have a \"03%\" EP_CODE, that are cancelled, + with out sync status, not in pepsq table " do + logged_epe2 + logged_ext_claim2 + expect(PriorityEndProductSyncQueue.count).to eq 1 + expect(PriorityEndProductSyncQueue.first.end_product_establishment_id).to eq logged_epe2.id + end + + let(:logged_epe3) { create(:end_product_establishment, synced_status: nil, reference_id: 300_000) } + let(:logged_ext_claim3) { create(:vbms_ext_claim, :canceled, :hlr, id: 300_000, ep_code: "930") } + + it "that have a \"93%\" EP_CODE, that are cancelled, + with out sync status, not in pepsq table " do + logged_epe3 + logged_ext_claim3 + expect(PriorityEndProductSyncQueue.count).to eq 1 + expect(PriorityEndProductSyncQueue.first.end_product_establishment_id).to eq logged_epe3.id + end + + let(:logged_epe4) { create(:end_product_establishment, synced_status: nil, reference_id: 300_000) } + let(:logged_ext_claim4) { create(:vbms_ext_claim, :cleared, id: 300_000, ep_code: "680") } + + it "that have a \"68%\" EP_CODE, that are cleared, + with out sync status, not in pepsq table " do + logged_epe4 + logged_ext_claim4 + expect(PriorityEndProductSyncQueue.count).to eq 1 + expect(PriorityEndProductSyncQueue.first.end_product_establishment_id).to eq logged_epe4.id + end + end + + context "we do not log inserted (on creation) vbms_ext_claims" do + let(:logged_epe5) { create(:end_product_establishment, synced_status: nil, reference_id: 300_000) } + let(:logged_ext_claim5) { create(:vbms_ext_claim, :rdc, :hlr, id: 300_000) } + + it "that have a \"03%\" EP_CODE, that are rdc, + with out sync status, not in pepsq table " do + logged_epe5 + logged_ext_claim5 + expect(PriorityEndProductSyncQueue.count).to eq 0 + end + + let(:logged_epe6) { create(:end_product_establishment, synced_status: nil, reference_id: 300_000) } + let(:logged_ext_claim6) { create(:vbms_ext_claim, :canceled, EP_CODE: "999", id: 300_000) } + + it "that have a wrong EP_CODE, that are canceled, + with a nil sync status, not in pepsq table " do + logged_epe6 + logged_ext_claim6 + expect(PriorityEndProductSyncQueue.count).to eq 0 + end + + let(:logged_epe7) { create(:end_product_establishment, synced_status: nil, reference_id: 300_000) } + let(:logged_ext_claim7) { create(:vbms_ext_claim, :canceled, :slc, id: 300_000) } + + it "that have a wrong EP_CODE, that are canceled, + with a nil sync status, already in the pepsq table " do + logged_epe7 + PriorityEndProductSyncQueue.create(end_product_establishment_id: logged_epe7.id) + logged_ext_claim7 + expect(PriorityEndProductSyncQueue.count).to eq 1 + end + end + end + + context "when the trigger is added and records already exist in the vbms_ext_claim table" do + before(:all) do + @logged_epe = create(:end_product_establishment, :active, reference_id: 300_000) + @logged_ext_claim = create(:vbms_ext_claim, :rdc, :slc, id: 300_000) + system("bundle exec rails r -e test db/scripts/drop_pepsq_populate_trigger_from_vbms_ext_claim.rb") + system("bundle exec rails r -e test db/scripts/add_pepsq_populate_trigger_to_vbms_ext_claim.rb") + end + before do + PriorityEndProductSyncQueue.delete_all + end + after(:all) do + EndProductEstablishment.delete(@logged_epe) + VbmsExtClaim.delete(@logged_ext_claim) + end + + context "we only log updated vbms_ext_claims" do + it "that have a \"04%\" EP_CODE, that are cleared, + different sync status, and are not in pepsq table" do + @logged_ext_claim.update(LEVEL_STATUS_CODE: "CLR") + expect(PriorityEndProductSyncQueue.count).to eq 1 + expect(PriorityEndProductSyncQueue.first.end_product_establishment_id).to eq @logged_epe.id + end + + it "that have a \"03%\" EP_CODE, that are cancelled, + with out sync status, not in pepsq table " do + @logged_epe.update(synced_status: nil) + @logged_ext_claim.update(LEVEL_STATUS_CODE: "CAN", EP_CODE: "030") + expect(PriorityEndProductSyncQueue.count).to eq 1 + expect(PriorityEndProductSyncQueue.first.end_product_establishment_id).to eq @logged_epe.id + end + + it "that have a \"93%\" EP_CODE, that are cleared, + different sync status, and are not in pepsq table" do + @logged_ext_claim.update(LEVEL_STATUS_CODE: "CLR", EP_CODE: "930") + expect(PriorityEndProductSyncQueue.count).to eq 1 + expect(PriorityEndProductSyncQueue.first.end_product_establishment_id).to eq @logged_epe.id + end + + it "that have a \"68%\" EP_CODE, that are cancelled, + with out sync status, not in pepsq table " do + @logged_epe.update(synced_status: nil) + @logged_ext_claim.update(LEVEL_STATUS_CODE: "CAN", EP_CODE: "680") + expect(PriorityEndProductSyncQueue.count).to eq 1 + expect(PriorityEndProductSyncQueue.first.end_product_establishment_id).to eq @logged_epe.id + end + end + + context "we do not log updated vbms_ext_claims" do + it "that have a \"03%\" EP_CODE, that are rdc, + with out sync status, not in pepsq table " do + @logged_ext_claim.update(LEVEL_STATUS_CODE: "RDC", EP_CODE: "030") + expect(PriorityEndProductSyncQueue.count).to eq 0 + end + + it "that have a wrong EP_CODE, that are canceled, + with a nil sync status, not in pepsq table " do + @logged_epe.update(synced_status: nil) + @logged_ext_claim.update(LEVEL_STATUS_CODE: "CAN", EP_CODE: "999") + expect(PriorityEndProductSyncQueue.count).to eq 0 + end + + it "that have a wrong EP_CODE, that are canceled, + with a nil sync status, already in the pepsq table " do + PriorityEndProductSyncQueue.create(end_product_establishment_id: @logged_epe.id) + expect(PriorityEndProductSyncQueue.count).to eq 1 + end + end + end + + context "when the trigger is removed from the vbms_ext_claim table" do + before(:all) do + system("bundle exec rails r -e test db/scripts/drop_pepsq_populate_trigger_from_vbms_ext_claim.rb") + end + before do + PriorityEndProductSyncQueue.delete_all + end + after(:all) do + system("bundle exec rails r -e test db/scripts/add_pepsq_populate_trigger_to_vbms_ext_claim.rb") + end + + let(:logged_epe) { create(:end_product_establishment, :active, reference_id: 300_000) } + let(:logged_ext_claim) { create(:vbms_ext_claim, :cleared, :slc, id: 300_000) } + + it "no records should be inserted into pepsq on creation of new vbms_ext_claim records" do + logged_epe + logged_ext_claim + expect(PriorityEndProductSyncQueue.count).to eq 0 + end + + it "no records should be inserted into pepsq on update of existing vbms_ext_claim records" do + logged_epe + logged_ext_claim + logged_epe.update(synced_status: nil) + logged_ext_claim.update(LEVEL_STATUS_CODE: "CAN", EP_CODE: "030") + expect(PriorityEndProductSyncQueue.count).to eq 0 + end + end +end