feature/APPEALS-50882

.codeclimate.yml

@@ -52,6 +52,7 @@ plugins:
           mass_threshold: 81
     exclude_patterns:
     - 'db/migrate/*'
+    - 'db/etl_migrate/*'
     - 'app/controllers/idt/api/v2/appeals_controller.rb'
     - 'spec/controllers/idt/api/v2/appeals_controller_spec.rb'
     - 'spec/controllers/idt/api/appeals_controller_spec.rb'
@@ -79,6 +80,7 @@ plugins:
         enabled: false
     exclude_patterns:
     - 'db/migrate/*'
+    - 'db/etl_migrate/*'
     - 'app/jobs/dispatch_email_job.rb'
     - 'app/mailers/dispatch_mailer.rb'
   rubocop:
@@ -115,6 +117,7 @@ plugins:
 
 exclude_patterns:
   - 'db/schema.rb'
+  - 'db/etl_schema.rb'
   - 'db/seeds.rb'
   - 'db/scripts/*'
   - 'node_modules/**/*'

.github/workflows/make-docs.yml

@@ -40,50 +40,24 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
 
-      # Since installing a client to Oracle DB is a pain and setting up the VACOLS DB is not necessary, let's skip it.
-      - name: Remove references to VACOLS Oracle DB
-        run: |
-          sed '/ruby-oci8/d' -i Gemfile
-          sed '/activerecord-oracle_enhanced-adapter/d' -i Gemfile
-
-          # Would like to run `bundle lock --update` to update Gemfile.lock without installing gems (to be done in next step)
-          # but bundler is not installed yet, so manually modify it:
-          sed '/ruby-oci8/d' -i Gemfile.lock
-          sed '/^    activerecord-oracle_enhanced-adapter/,/^    [a-z]/{/^    activerecord-oracle_enhanced-adapter/!{/^    [a-z]/!d}}' -i Gemfile.lock
-          sed '/activerecord-oracle_enhanced-adapter/d' -i Gemfile.lock
-          sed '/ruby-plsql/d' -i Gemfile.lock
-
-          # Remove VACOLS database configuration
-          # https://stackoverflow.com/questions/6287755/using-sed-to-delete-all-lines-between-two-matching-patterns
-          sed '/_vacols:$/,/^$/{/^_vacols:$/!{/^$/!d}}' -i config/database.yml
-
-          # Even though it won't be used in this script, Rails expects a 'test_vacols' configuration due to "#{Rails.env}_vacols"
-          echo '# Copied from demo_vacols configuration
-          test_vacols:
-            adapter: postgresql
-            pool: 1
-            timeout: 5000
-            database: test-vacols
-          ' >> config/database.yml
-
       - name: Setup Ruby and install gems
         uses: ruby/setup-ruby@v1
         with:
           bundler-cache: true
 
       - name: Setup test database
         env:
-          RAILS_ENV: test
+          RAILS_ENV: make_docs
           POSTGRES_HOST: localhost
           POSTGRES_PASSWORD: postgres
           POSTGRES_USER: postgres
         run: |
           echo "::group::Set up Caseflow DB"
-          bin/rails db:create && bin/rails db:schema:load
+          bin/rails db:create:primary && bin/rails db:schema:load:primary
           echo "::endgroup::"
 
           echo "::group::Set up Caseflow ETL DB"
-          DB=etl bundle exec rake db:create db:schema:load
+          bundle exec rake db:create:etl db:schema:load:etl
           echo "::endgroup::"
 
           # Skipping VACOLS since Oracle DB is not set up
@@ -95,7 +69,7 @@ jobs:
 
       - name: Create DB schema documentation
         env:
-          RAILS_ENV: test
+          RAILS_ENV: make_docs
           POSTGRES_HOST: localhost
           POSTGRES_PASSWORD: postgres
           POSTGRES_USER: postgres

.github/workflows/workflow.yml

@@ -193,8 +193,8 @@ jobs:
 
       - name: Database setup
         run: |
-          ./ci-bin/capture-log "DB=etl bundle exec rake db:create db:schema:load db:migrate"
-          ./ci-bin/capture-log "bundle exec rake db:create db:schema:load db:migrate"
+          ./ci-bin/capture-log "bundle exec rake db:create:etl db:schema:load:etl"
+          ./ci-bin/capture-log "bundle exec rake db:create:primary db:schema:load:primary"
           ./ci-bin/capture-log "make -f Makefile.example external-db-create"
         # added line to create external table(s) that are needed for tests
 

.reek.yml

@@ -291,5 +291,5 @@ directories:
 # Directories and files below will not be scanned at all
 exclude_paths:
   - db/migrate
-  - db/etl/migrate
+  - db/etl_migrate
   - spec

.rubocop.yml

@@ -6,6 +6,7 @@ AllCops:
   Exclude:
     - 'bin/**/*'
     - 'db/**/*'
+    - 'config.ru'
     - 'config/**/*'
     - 'script/**/*'
     - 'vendor/**/*'

Dangerfile

@@ -35,19 +35,19 @@ if git.modified_files.grep(/db\/schema.rb/).any?
   warn("This PR changes the schema. Please use the PR template checklist.")
 end
 
-if git.modified_files.grep(/db\/etl\/schema.rb/).any?
+if git.modified_files.grep(/db\/etl_schema.rb/).any?
   warn("This PR changes the etl schema. Please use the PR template checklist.")
 end
 
 new_db_migrations = git.modified_files.grep(/db\/migrate\//).any?
-new_etl_migrations = git.modified_files.grep(/db\/etl\/migrate\//).any?
+new_etl_migrations = git.modified_files.grep(/db\/etl_migrate\//).any?
 
 # migration without migrating
 if new_db_migrations && git.modified_files.grep(/db\/schema.rb/).none?
   warn("This PR contains db migrations, but the schema.rb is not modified. Did you forget to run 'make migrate'?")
 end
 
-if new_etl_migrations && git.modified_files.grep(/db\/etl\/schema.rb/).none?
+if new_etl_migrations && git.modified_files.grep(/db\/etl_schema.rb/).none?
   warn("This PR contains etl migrations, but the etl schema.rb is not modified. Did you forget to run 'make migrate'?")
 end
 

Dockerfile

@@ -66,10 +66,6 @@ RUN apt install -y ${CASEFLOW} &&  \
 # install jemalloc
 RUN apt install -y --no-install-recommends libjemalloc-dev
 
-
-# install datadog agent
-RUN DD_INSTALL_ONLY=true DD_AGENT_MAJOR_VERSION=7 DD_API_KEY=$(cat config/datadog.key) bash -c "$(curl -L https://raw.githubusercontent.com/DataDog/datadog-agent/master/cmd/agent/install_script.sh)"
-
 RUN rm -rf /var/lib/apt/lists/*
 
 # Installing the version of bundler that corresponds to the Gemfile.lock

Gemfile

@@ -5,7 +5,7 @@ source ENV["GEM_SERVER_URL"] || "https://rubygems.org"
 
 # State machine
 gem "aasm", "4.11.0"
-gem "activerecord-import"
+gem "activerecord-import", "1.0.3"
 gem "acts_as_tree"
 
 # amoeba gem for cloning appeals
@@ -17,13 +17,11 @@ gem "bgs", git: "https://github.com/department-of-veterans-affairs/ruby-bgs.git"
 gem "bootsnap", require: false
 gem "browser"
 gem "business_time", "~> 0.9.3"
-gem "caseflow", git: "https://github.com/department-of-veterans-affairs/caseflow-commons", ref: "716b58caf2116da5fca21c3b3aeea6c9712f3b9d"
+gem "caseflow", git: "https://github.com/department-of-veterans-affairs/caseflow-commons", ref: "9bd3635fbd8094d25160669f38d8699e2f1d7a98"
 gem "connect_mpi", git: "https://github.com/department-of-veterans-affairs/connect-mpi.git", ref: "a3a58c64f85b980a8b5ea6347430dd73a99ea74c"
 gem "connect_vbms", git: "https://github.com/department-of-veterans-affairs/connect_vbms.git", ref: "9807d9c9f0f3e3494a60b6693dc4f455c1e3e922"
 gem "console_tree_renderer", git: "https://github.com/department-of-veterans-affairs/console-tree-renderer.git", tag: "v0.1.1"
 gem "countries"
-gem "ddtrace"
-gem "dogstatsd-ruby"
 gem "dry-schema", "~> 1.4"
 gem "fast_jsonapi"
 gem "fuzzy_match"
@@ -36,8 +34,28 @@ gem "moment_timezone-rails"
 # Rails 6 has native support for multiple dbs, so prefer that over multiverse after upgrade.
 # https://github.com/ankane/multiverse#upgrading-to-rails-6
 gem "multiverse"
-gem "newrelic_rpm"
 gem "nokogiri", ">= 1.11.0.rc4"
+
+gem "opentelemetry-exporter-otlp", require: false
+gem "opentelemetry-sdk", require: false
+
+gem "opentelemetry-instrumentation-action_pack", require: false
+gem "opentelemetry-instrumentation-action_view", require: false
+gem "opentelemetry-instrumentation-active_job", require: false
+gem "opentelemetry-instrumentation-active_model_serializers", require: false
+gem "opentelemetry-instrumentation-active_record", require: false
+gem "opentelemetry-instrumentation-aws_sdk", require: false
+gem "opentelemetry-instrumentation-concurrent_ruby", require: false
+gem "opentelemetry-instrumentation-faraday", require: false
+gem "opentelemetry-instrumentation-http", require: false
+gem "opentelemetry-instrumentation-http_client", require: false
+gem "opentelemetry-instrumentation-net_http", require: false
+gem "opentelemetry-instrumentation-pg", require: false
+gem "opentelemetry-instrumentation-rack", require: false
+gem "opentelemetry-instrumentation-rails", require: false
+gem "opentelemetry-instrumentation-rake", require: false
+gem "opentelemetry-instrumentation-redis", require: false
+
 gem "paper_trail", "~> 12.0"
 # Used to speed up reporting
 gem "parallel"
@@ -56,13 +74,13 @@ gem "pg", platforms: :ruby
 # Discussion: https://github.com/18F/college-choice/issues/597#issuecomment-139034834
 gem "puma", "5.6.4"
 gem "rack", "~> 2.2.6.2"
-gem "rails", "6.0.6.1"
+gem "rails", "6.1.7.7"
 # Used to colorize output for rake tasks
 gem "rainbow"
 # React
 gem "react_on_rails", "11.3.0"
 gem "redis-mutex"
-gem "redis-namespace"
+gem "redis-namespace", "~> 1.11.0"
 gem "redis-rails", "~> 5.0.2"
 gem "request_store"
 gem "roo", "~> 2.7"
@@ -80,22 +98,22 @@ gem "stringex", require: false
 gem "strong_migrations"
 # print trees
 gem "tty-tree"
-gem "tzinfo", "1.2.10"
+gem "tzinfo", "~> 2.0"
 # Use Uglifier as compressor for JavaScript assets
 gem "uglifier", ">= 1.3.0"
 gem "validates_email_format_of"
 gem "ziptz"
 
 group :production, :staging, :ssh_forwarding, :development, :test do
   # Oracle DB
-  gem "activerecord-oracle_enhanced-adapter", "~> 6.0.0"
+  gem "activerecord-oracle_enhanced-adapter", "~> 6.1.0"
   gem "ruby-oci8", "~> 2.2"
 end
 
-group :test, :development, :demo do
+group :test, :development, :demo, :make_docs do
   # Security scanners
   gem "brakeman"
-  gem "bullet"
+  gem "bullet", "~> 6.1.0"
   gem "bundler-audit"
   # Testing tools
   gem "capybara"
@@ -122,7 +140,8 @@ group :test, :development, :demo do
   gem "rubocop-performance"
   gem "rubocop-rails"
   gem "scss_lint", require: false
-  gem "simplecov", git: "https://github.com/colszowka/simplecov.git", require: false
+  gem "simplecov", require: false
+  gem "simplecov-lcov", require: false
   gem "single_cov"
   gem "sniffybara", git: "https://github.com/department-of-veterans-affairs/sniffybara.git"
   gem "sql_tracker"