Merge branch 'master' into feature/APPEALS-15376

.github/workflows/workflow.yml

@@ -189,6 +189,8 @@ jobs:
         run: |
           ./ci-bin/capture-log "DB=etl bundle exec rake db:create db:schema:load db:migrate"
           ./ci-bin/capture-log "bundle exec rake db:create db:schema:load db:migrate"
+          ./ci-bin/capture-log "make -f Makefile.example external-db-create"
+        # added line to create external table(s) that are needed for tests
 
       # We don't want to seed DBs here because DatabaseCleaner just truncates it anyway. The setup_vacols
       # rake task needs to be run because it adds data to two tables that are ignored by DBCleaner

Gemfile

@@ -47,8 +47,8 @@ gem "paranoia", "~> 2.2"
 gem "pdf-forms"
 # Used in Caseflow Dispatch
 gem "pdfjs_viewer-rails", git: "https://github.com/senny/pdfjs_viewer-rails.git", ref: "a4249eacbf70175db63b57e9f364d0a9a79e2b43"
-#Used to build out PDF files on the backend
-#https://github.com/pdfkit/pdfkit
+# Used to build out PDF files on the backend
+# https://github.com/pdfkit/pdfkit
 gem "pdfkit"
 gem "pg", platforms: :ruby
 # Application server: Puma
@@ -61,6 +61,7 @@ gem "rails", "5.2.4.6"
 gem "rainbow"
 # React
 gem "react_on_rails", "11.3.0"
+gem "redis-mutex"
 gem "redis-namespace"
 gem "redis-rails", "~> 5.0.2"
 gem "request_store"

Gemfile.lock

@@ -523,6 +523,10 @@ GEM
     redis-activesupport (5.0.4)
       activesupport (>= 3, < 6)
       redis-store (>= 1.3, < 2)
+    redis-classy (2.4.1)
+      redis-namespace (~> 1.0)
+    redis-mutex (4.0.2)
+      redis-classy (~> 2.0)
     redis-namespace (1.6.0)
       redis (>= 3.0.4)
     redis-rack (2.0.4)
@@ -783,6 +787,7 @@ DEPENDENCIES
   rainbow
   rb-readline
   react_on_rails (= 11.3.0)
+  redis-mutex
   redis-namespace
   redis-rails (~> 5.0.2)
   request_store

Makefile.example

@@ -156,20 +156,44 @@ audit: ## Create caseflow_audit schema, tables, and triggers in postgres
 	bundle exec rails r db/scripts/audit/tables/create_vbms_distributions_audit.rb
 	bundle exec rails r db/scripts/audit/tables/create_vbms_distribution_destinations_audit.rb
 	bundle exec rails r db/scripts/audit/tables/create_vbms_uploaded_documents_audit.rb
+	bundle exec rails r db/scripts/audit/tables/create_priority_end_product_sync_queue_audit.rb
 	bundle exec rails r db/scripts/audit/functions/add_row_to_appeal_states_audit_table_function.rb
 	bundle exec rails r db/scripts/audit/functions/add_row_to_vbms_communication_packages_audit_table_function.rb
 	bundle exec rails r db/scripts/audit/functions/add_row_to_vbms_distributions_audit_table_function.rb
 	bundle exec rails r db/scripts/audit/functions/add_row_to_vbms_distribution_destinations_audit_table_function.rb
 	bundle exec rails r db/scripts/audit/functions/add_row_to_vbms_uploaded_documents_audit_table_function.rb
+	bundle exec rails r db/scripts/audit/functions/add_row_to_priority_end_product_sync_queue_audit_table_function.rb
 	bundle exec rails r db/scripts/audit/triggers/create_appeal_states_audit_trigger.rb
 	bundle exec rails r db/scripts/audit/triggers/create_vbms_communication_packages_audit_trigger.rb
 	bundle exec rails r db/scripts/audit/triggers/create_vbms_distributions_audit_trigger.rb
 	bundle exec rails r db/scripts/audit/triggers/create_vbms_distribution_destinations_audit_trigger.rb
 	bundle exec rails r db/scripts/audit/triggers/create_vbms_uploaded_documents_audit_trigger.rb
+	bundle exec rails r db/scripts/audit/triggers/create_priority_end_product_sync_queue_audit_trigger.rb
 
 audit-remove: ## Remove caseflow_audit schema, tables and triggers in postgres
 	bundle exec rails r db/scripts/audit/remove_caseflow_audit_schema.rb
 
+# The external-db make commands create/remove replicas (for local environment only) of external db tables that exist in Prod
+# These tables should not be included as part of migrations
+external-db-create: ## Creates external_vbms_ext_claim table
+	bundle exec rails r db/scripts/external/create_vbms_ext_claim_table.rb
+
+external-db-remove:  ## Remove external_vbms_ext_claim table
+	bundle exec rails r db/scripts/external/remove_vbms_ext_claim_table.rb
+
+# This needs to be manually run after make reset/migrate in order for local tests involving external tables to pass.
+# Otherwise the caseflow_certification_test schema will not create these tables and will error out.
+external-db-create-test: ## Creates table in caseflow_certification_test DB for local RSPEC tests
+	bundle exec rails r -e test db/scripts/external/create_vbms_ext_claim_table.rb
+
+remove-vbms-ext-claim-seeds: ## Drops audit tables, removes all PriorityEndProductSyncQueue, BatchProcess, and seed-vbms-ext-claim records, then rebuilds audit tables
+	make audit-remove
+	make external-db-create
+	bundle exec rails r db/scripts/external/remove_vbms_ext_claim_seeds.rb
+	make audit
+
+reseed-vbms-ext-claim: remove-vbms-ext-claim-seeds seed-vbms-ext-claim ## Re-seeds database with records created from seed-vbms-ext-claim
+
 c:  ## Start rails console
 	bundle exec rails console
 
@@ -188,7 +212,7 @@ db-migrate:	## Migrate main Caseflow db
 db-rollback:	## Rollback main Caseflow db
 	bundle exec rake db:rollback
 
-migrate: etl-migrate etl-test-prepare db-migrate ## Migrate all Rails databases
+migrate: external-db-remove etl-migrate etl-test-prepare db-migrate ## Migrate all non-external Rails databases
 
 rollback: etl-rollback db-rollback	## Rollback all Rails databases
 
@@ -199,9 +223,14 @@ reset: reset-dbs seed-dbs enable-feature-flags ## Resets databases and enable fe
 
 reset-dbs: ## Resets Caseflow and ETL database schemas
 	make audit-remove
+	make external-db-remove
 	DB=etl bundle exec rake db:drop db:create db:schema:load
 	bundle exec rake db:drop db:create db:schema:load
 	make audit
+	make external-db-create
+
+seed-vbms-ext-claim: ## Seed only vbms_ext_claim
+	bundle exec rake db:seed:vbms_ext_claim
 
 seed-dbs:	## Seed all databases
 	bundle exec rake local:vacols:seed

app/jobs/batch_processes/batch_process_rescue_job.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+# This job will search for and reprocess unfinished Batch Processes nightly.
+# Search Criteria is for Batch Processes that are in an unfinished state ('PRE_PROCESSING', 'PROCESSING') &
+# have a created_at date/time that is greater than the ERROR_DELAY defined within batch_process.rb
+class BatchProcessRescueJob < CaseflowJob
+  queue_with_priority :low_priority
+
+  # :reek:FeatureEnvy
+  def perform
+    batches = BatchProcess.needs_reprocessing
+    if batches.any?
+      batches.each do |batch|
+        begin
+          batch.process_batch!
+        rescue StandardError => error
+          log_error(error, extra: { active_job_id: job_id.to_s, job_time: Time.zone.now.to_s })
+          slack_msg = "Error running #{self.class.name}.  Error: #{error.message}.  Active Job ID: #{job_id}."
+          slack_msg += "  See Sentry event #{Raven.last_event_id}." if Raven.last_event_id.present?
+          slack_service.send_notification("[ERROR] #{slack_msg}", self.class.to_s)
+          next
+        end
+      end
+    else
+      Rails.logger.info("No Unfinished Batches Could Be Identified.  Time: #{Time.zone.now}.")
+    end
+  end
+end

app/jobs/batch_processes/priority_ep_sync_batch_process_job.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+class PriorityEpSyncBatchProcessJob < CaseflowJob
+  queue_with_priority :low_priority
+
+  # Using macro-style definition. The locking scope will be TheClass#method and only one method can run at any
+  # given time.
+  include RedisMutex::Macro
+
+  # Default options for RedisMutex#with_lock
+  # :block  => 1    # Specify in seconds how long you want to wait for the lock to be released.
+  #                 # Specify 0 if you need non-blocking sematics and return false immediately. (default: 1)
+  # :sleep  => 0.1  # Specify in seconds how long the polling interval should be when :block is given.
+  #                 # It is NOT recommended to go below 0.01. (default: 0.1)
+  # :expire => 10   # Specify in seconds when the lock should be considered stale when something went wrong
+  #                 # with the one who held the lock and failed to unlock. (default: 10)
+  #
+  # RedisMutex.with_lock("PriorityEpSyncBatchProcessJob", block: 60, expire: 100)
+  # Key => "PriorityEpSyncBatchProcessJob"
+
+  JOB_DURATION ||= ENV["BATCH_PROCESS_JOB_DURATION"].to_i.minutes
+  SLEEP_DURATION ||= ENV["BATCH_PROCESS_SLEEP_DURATION"].to_i
+
+  # Attempts to create & process batches for 50 minutes
+  # There will be a 5 second rest between each iteration
+  # Job will end if there are no records are left to batch
+
+  # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/CyclomaticComplexity
+  def perform
+    setup_job
+    loop do
+      break if job_running_past_expected_end_time? || should_stop_job
+
+      begin
+        batch = nil
+        RedisMutex.with_lock("PriorityEpSyncBatchProcessJob", block: 60, expire: 100) do
+          batch = ActiveRecord::Base.transaction do
+            records_to_batch = PriorityEpSyncBatchProcess.find_records_to_batch
+            next if records_to_batch.empty?
+
+            PriorityEpSyncBatchProcess.create_batch!(records_to_batch)
+          end
+        end
+
+        batch ? batch.process_batch! : stop_job(log_no_records_found: true)
+
+        sleep(SLEEP_DURATION)
+      rescue StandardError => error
+        log_error(error, extra: { job_id: job_id.to_s, job_time: Time.zone.now.to_s })
+        slack_msg = "Error running #{self.class.name}.  Error: #{error.message}.  Active Job ID: #{job_id}."
+        slack_msg += "  See Sentry event #{Raven.last_event_id}." if Raven.last_event_id.present?
+        slack_service.send_notification("[ERROR] #{slack_msg}", self.class.to_s)
+        stop_job
+      end
+    end
+  end
+  # rubocop:enable Metrics/MethodLength, Metrics/AbcSize, Metrics/CyclomaticComplexity
+
+  private
+
+  attr_accessor :job_expected_end_time, :should_stop_job
+
+  def setup_job
+    RequestStore.store[:current_user] = User.system_user
+    @should_stop_job = false
+    @job_expected_end_time = Time.zone.now + JOB_DURATION
+  end
+
+  def job_running_past_expected_end_time?
+    Time.zone.now > job_expected_end_time
+  end
+
+  # :reek:BooleanParameter
+  # :reek:ControlParameter
+  def stop_job(log_no_records_found: false)
+    self.should_stop_job = true
+    if log_no_records_found
+      Rails.logger.info("#{self.class} Cannot Find Any Records to Batch."\
+        "  Job will be enqueued again at the top of the hour.  Active Job ID: #{job_id}.  Time: #{Time.zone.now}")
+    end
+  end
+end

app/jobs/populate_end_product_sync_queue_job.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+# This job will find deltas between the end product establishment table and the VBMS ext claim table
+# where VBMS ext claim level status code is CLR or CAN. If EP is already in the queue it will be skipped.
+# Job will populate queue ENV["END_PRODUCT_QUEUE_BATCH_LIMIT"] records at a time.
+# This job will run on a 50 minute loop, sleeping for 5 seconds between iterations.
+class PopulateEndProductSyncQueueJob < CaseflowJob
+  queue_with_priority :low_priority
+
+  JOB_DURATION ||= ENV["END_PRODUCT_QUEUE_JOB_DURATION"].to_i.minutes
+  SLEEP_DURATION ||= ENV["END_PRODUCT_QUEUE_SLEEP_DURATION"].to_i
+  BATCH_LIMIT ||= ENV["END_PRODUCT_QUEUE_BATCH_LIMIT"].to_i
+
+  # rubocop:disable Metrics/CyclomaticComplexity
+  def perform
+    setup_job
+    loop do
+      break if job_running_past_expected_end_time? || should_stop_job
+
+      begin
+        batch = ActiveRecord::Base.transaction do
+          priority_epes = find_priority_end_product_establishments_to_sync
+          next if priority_epes.empty?
+
+          priority_epes
+        end
+
+        batch ? insert_into_priority_sync_queue(batch) : stop_job(log_no_records_found: true)
+
+        sleep(SLEEP_DURATION)
+      rescue StandardError => error
+        log_error(error, extra: { active_job_id: job_id.to_s, job_time: Time.zone.now.to_s })
+        slack_msg = "Error running #{self.class.name}.  Error: #{error.message}.  Active Job ID: #{job_id}."
+        slack_msg += "  See Sentry event #{Raven.last_event_id}." if Raven.last_event_id.present?
+        slack_service.send_notification("[ERROR] #{slack_msg}", self.class.to_s)
+        stop_job
+      end
+    end
+  end
+  # rubocop:enable Metrics/CyclomaticComplexity
+
+  private
+
+  attr_accessor :job_expected_end_time, :should_stop_job
+
+  def find_priority_end_product_establishments_to_sync
+    get_batch = <<-SQL
+    select id
+      from end_product_establishments
+      inner join vbms_ext_claim
+      on end_product_establishments.reference_id = vbms_ext_claim."CLAIM_ID"::varchar
+      where (end_product_establishments.synced_status <> vbms_ext_claim."LEVEL_STATUS_CODE" or end_product_establishments.synced_status is null)
+        and vbms_ext_claim."LEVEL_STATUS_CODE" in ('CLR','CAN')
+        and end_product_establishments.id not in (select end_product_establishment_id from priority_end_product_sync_queue)
+      limit #{BATCH_LIMIT};
+    SQL
+
+    ActiveRecord::Base.connection.exec_query(ActiveRecord::Base.sanitize_sql(get_batch)).rows.flatten
+  end
+
+  def insert_into_priority_sync_queue(batch)
+    batch.each do |ep_id|
+      PriorityEndProductSyncQueue.create!(
+        end_product_establishment_id: ep_id
+      )
+    end
+    Rails.logger.info("PopulateEndProductSyncQueueJob EPEs processed: #{batch} - Time: #{Time.zone.now}")
+  end
+
+  def setup_job
+    RequestStore.store[:current_user] = User.system_user
+    @should_stop_job = false
+    @job_expected_end_time = Time.zone.now + JOB_DURATION
+  end
+
+  def job_running_past_expected_end_time?
+    Time.zone.now > job_expected_end_time
+  end
+
+  # :reek:BooleanParameter
+  # :reek:ControlParameter
+  def stop_job(log_no_records_found: false)
+    self.should_stop_job = true
+    if log_no_records_found
+      Rails.logger.info("PopulateEndProductSyncQueueJob is not able to find any batchable EPE records."\
+        "  Active Job ID: #{job_id}.  Time: #{Time.zone.now}")
+    end
+  end
+end