Merge pull request #1546 from department-of-veterans-affairs/release/…

…FY24Q2.6.2

Release/fy24 q2.6.2

Gemfile

@@ -13,6 +13,7 @@ gem "connect_vbms", git: "https://github.com/department-of-veterans-affairs/conn
 gem "connect_vva", git: "https://github.com/department-of-veterans-affairs/connect_vva.git", ref: "dfd1aeb2605c1f237f520bcdc41b059202e8944d"
 gem "distribute_reads"
 gem "dogstatsd-ruby"
+gem "statsd-instrument"
 gem "httpclient"
 gem "jbuilder", "~> 2.0"
 gem "jquery-rails", ">= 4.3.4"

Gemfile.lock

@@ -485,6 +485,7 @@ GEM
       actionpack (>= 5.2)
       activesupport (>= 5.2)
       sprockets (>= 3.0.0)
+    statsd-instrument (3.7.0)
     systemu (2.6.5)
     therubyracer (0.12.3)
       libv8 (~> 3.16.14.15)
@@ -600,6 +601,7 @@ DEPENDENCIES
   sinatra (= 2.2.0)
   single_cov
   sniffybara!
+  statsd-instrument
   therubyracer
   timecop
   turbolinks

app/controllers/concerns/collect_custom_metrics.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module CollectCustomMetrics
+  extend ActiveSupport::Concern
+
+  included do
+    before_action :collect_custom_metrics
+  end
+
+  def collect_custom_metrics
+    collect_postgres_metrics
+  end
+
+  def collect_postgres_metrics
+    conns = ActiveRecord::Base.connection_pool.connections
+
+    active = conns.count { |conn| conn.in_use? && conn.owner.alive? }
+    dead = conns.count { |conn| conn.in_use? && !conn.owner.alive? }
+    idle = conns.count { |conn| !conn.in_use? }
+
+    emit_metrics_service_point("postgres", "active", active)
+    emit_metrics_service_point("postgres", "dead", dead)
+    emit_metrics_service_point("postgres", "idle", idle)
+  end
+
+  def emit_metrics_service_point(db_name, type, count)
+    MetricsService.emit_gauge(
+      metric_group: "database",
+      metric_name: "#{type}_connections",
+      metric_value: count,
+      app_name: "efolder",
+      attrs: {
+        database: db_name
+      }
+    )
+  end
+end

app/controllers/health_checks_controller.rb

@@ -1,5 +1,5 @@
 class HealthChecksController < ApplicationController
-  include CollectDataDogMetrics
+  include CollectCustomMetrics
   skip_before_action :authenticate
   skip_before_action :check_out_of_service
   newrelic_ignore_apdex

app/jobs/application_job.rb

@@ -8,18 +8,18 @@ class ApplicationJob < ActiveJob::Base
     job.start_time = Time.zone.now
   end
 
-  def datadog_report_runtime(metric_group_name:)
-    DataDogService.record_runtime(
+  def metrics_service_report_runtime(metric_group_name:)
+    MetricsService.record_runtime(
       app_name: "efolder_job",
       metric_group: metric_group_name,
       start_time: start_time
     )
   end
 
-  def datadog_report_time_segment(segment:, start_time:)
+  def metrics_service_report_time_segment(segment:, start_time:)
     job_duration_seconds = Time.zone.now - start_time
 
-    DataDogService.emit_gauge(
+    MetricsService.emit_gauge(
       app_name: "efolder_job_segment",
       metric_group: segment,
       metric_name: "runtime",

app/jobs/middleware/job_data_dog_metric_middleware.rb → app/jobs/middleware/job_metrics_service_metric_middleware.rb

@@ -1,4 +1,4 @@
-class JobDataDogMetricMiddleware
+class JobMetricsServiceMetricMiddleware
   def call(_worker, queue, _msg, body)
     job_class = body["job_class"]
 
@@ -7,7 +7,7 @@ def call(_worker, queue, _msg, body)
     end
 
     begin
-      DataDogService.emit_gauge(
+      MetricsService.emit_gauge(
         metric_group: "job",
         metric_name: "elapsed_time",
         metric_value: stopwatch.real,

app/services/metrics_service.rb

@@ -1,57 +1,124 @@
+# frozen_string_literal: true
+
 require "benchmark"
+require "datadog/statsd"
+require "statsd-instrument"
 
 # see https://dropwizard.github.io/metrics/3.1.0/getting-started/ for abstractions on metric types
-# :nocov:
 class MetricsService
-  # rubocop:disable Metrics/MethodLength
-  @app = "eFolder"
-  def self.record(description, service: nil, name: "unknown")
+  @statsd = Datadog::Statsd.new
+
+  # :reek:LongParameterList
+  def self.increment_counter(metric_group:, metric_name:, app_name:, attrs: {}, by: 1)
+    tags = get_tags(app_name, attrs)
+    stat_name = get_stat_name(metric_group, metric_name)
+    @statsd.increment(stat_name, tags: tags, by: by)
+
+    # Dynatrace statD implementation
+    StatsD.increment(stat_name, tags: tags)
+  end
+
+  def self.record_runtime(metric_group:, app_name:, start_time: Time.zone.now)
+    metric_name = "runtime"
+    job_duration_seconds = Time.zone.now - start_time
+
+    emit_gauge(
+      app_name: app_name,
+      metric_group: metric_group,
+      metric_name: metric_name,
+      metric_value: job_duration_seconds
+    )
+  end
+
+  # :reek:LongParameterList
+  def self.emit_gauge(metric_group:, metric_name:, metric_value:, app_name:, attrs: {})
+    tags = get_tags(app_name, attrs)
+    stat_name = get_stat_name(metric_group, metric_name)
+    @statsd.gauge(stat_name, metric_value, tags: tags)
+
+    # Dynatrace statD implementation
+    StatsD.gauge(stat_name, metric_value, tags: tags)
+  end
+
+  # :nocov:
+  # :reek:LongParameterList
+  def self.histogram(metric_group:, metric_name:, metric_value:, app_name:, attrs: {})
+    tags = get_tags(app_name, attrs)
+    stat_name = get_stat_name(metric_group, metric_name)
+    @statsd.histogram(stat_name, metric_value, tags: tags)
+
+    # Dynatrace statD implementation
+    StatsD.histogram(stat_name, metric_value, tags: tags)
+  end
+  # :nocov:
+
+  private_class_method def self.get_stat_name(metric_group, metric_name)
+    "dsva-appeals.#{metric_group}.#{metric_name}"
+  end
+
+  private_class_method def self.get_tags(app_name, attrs)
+    extra_tags = attrs.reduce([]) do |tags, (key, val)|
+      tags + ["#{key}:#{val}"]
+    end
+    [
+      "app:#{app_name}",
+      "env:#{Rails.current_env}"
+    ] + extra_tags
+  end
+
+  # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+  # :reek:LongParameterList
+  def self.record(description, service: nil, name: "unknown", caller: nil)
     return_value = nil
+    app = RequestStore[:application] || "other"
 
     Rails.logger.info("STARTED #{description}")
     stopwatch = Benchmark.measure do
       return_value = yield
     end
 
-    if service && Rails.env.production?
+    if service
       latency = stopwatch.real
-      DataDogService.emit_gauge(
+      sent_to_info = {
         metric_group: "service",
         metric_name: "request_latency",
         metric_value: latency,
-        app_name: @app,
+        app_name: app,
         attrs: {
-          service: service,
-          endpoint: name
+          service: service ||= app,
+          endpoint: name,
+          uuid: SecureRandom.uuid
         }
-      )
+      }
+      MetricsService.emit_gauge(sent_to_info)
     end
 
     Rails.logger.info("FINISHED #{description}: #{stopwatch}")
+
     return_value
-  rescue StandardError
-    increment_datadog_counter("request_error", service, name) if service
+  rescue StandardError => error
+    Rails.logger.error("#{error.message}\n#{error.backtrace.join("\n")}")
+    Raven.capture_exception(error, extra: { type: "request_error", service: service, name: name, app: app })
 
-    Rails.logger.info("RESCUED #{description}")
+    increment_metrics_service_counter("request_error", service, name, app) if service
 
     # Re-raise the same error. We don't want to interfere at all in normal error handling.
     # This is just to capture the metric.
     raise
   ensure
-    increment_datadog_counter("request_attempt", service, name) if service
+    increment_metrics_service_counter("request_attempt", service, name, app) if service
   end
-  # rubocop:enable Metrics/MethodLength
+  # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
 
-  private_class_method def self.increment_datadog_counter(metric_name, service, endpoint_name)
-    DataDogService.increment_counter(
+  private_class_method def self.increment_metrics_service_counter(metric_name, service, endpoint_name, app_name)
+    increment_counter(
       metric_group: "service",
       metric_name: metric_name,
-      app_name: @app,
+      app_name: app_name,
       attrs: {
         service: service,
         endpoint: endpoint_name
       }
     )
   end
-end
-# :nocov:
+end

config.ru

@@ -24,8 +24,8 @@ module PumaThreadLogger
           waiting = @waiting
         }
 
-        emit_datadog_point("idle", waiting)
-        emit_datadog_point("active", thread_count - waiting)
+        emit_metrics_service_point("idle", waiting)
+        emit_metrics_service_point("active", thread_count - waiting)
 
         # For some reason, even a single Puma server (not clustered) has two booted ThreadPools.
         # One of them is empty, and the other is actually doing work.
@@ -49,8 +49,8 @@ module PumaThreadLogger
     super *args
   end
 
-  def emit_datadog_point(type, count)
-    DataDogService.emit_gauge(
+  def emit_metrics_service_point(type, count)
+    MetricsService.emit_gauge(
       metric_group: "puma",
       metric_name: "#{type}_threads",
       metric_value: count,

config/environments/development.rb

@@ -74,4 +74,7 @@
 
   config.sqs_create_queues = true
   config.sqs_endpoint = 'http://localhost:4566'
-end
+
+  # Dynatrace variables
+  ENV["STATSD_ENV"] = "development"
+end

config/environments/test.rb

@@ -54,4 +54,7 @@
   config.s3_enabled = false
 
   config.api_key = "token"
-end
+
+  # Dynatrace variables
+  ENV["STATSD_ENV"] = "test"
+end

config/initializers/shoryuken.rb

@@ -1,4 +1,4 @@
-require "#{Rails.root}/app/jobs/middleware/job_data_dog_metric_middleware"
+require "#{Rails.root}/app/jobs/middleware/job_metrics_service_metric_middleware"
 
 # set up default exponential backoff parameters
 ActiveJob::QueueAdapters::ShoryukenAdapter::JobWrapper
@@ -23,6 +23,6 @@
 
   # register all shoryuken middleware
   config.server_middleware do |chain|
-    chain.add JobDataDogMetricMiddleware
+    chain.add JobMetricsServiceMetricMiddleware
   end
 end

spec/features/backend_error_flows_spec.rb

@@ -39,7 +39,7 @@
   end
 
   before do
-    allow(DataDogService).to receive(:emit_gauge) { true } # mock DD during tests
+    allow(MetricsService).to receive(:emit_gauge) { true } # mock DD during tests
 
     @user = User.create(css_id: "123123", station_id: "116")