Remove Hush from test_helper.exs

.github/workflows/test.yml

@@ -126,7 +126,7 @@ jobs:
           POSTGRES_USER: docker
           POSTGRES_PASSWORD: d0ck3r
         options: >-
-          --health-cmd "pg_isready -U postgres"
+          --health-cmd "pg_isready -U docker"
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5

.gitignore

@@ -49,6 +49,7 @@ npm-debug.log
 *.tfvars
 
 /infrastructure/**/.terraform/
+/infrastructure/**/*.tfstate
 /infrastructure/**/_build/
 /infrastructure/**/build/
 /infrastructure/**/builds/

app/config/config.exs

@@ -115,11 +115,6 @@ config :authoritex,
     NUL.Authority
   ]
 
-config :ex_aws,
-  access_key_id: [:instance_role],
-  secret_access_key: [:instance_role],
-  region: System.get_env("AWS_REGION", "us-east-1")
-
 config :httpoison_retry, wait: 50
 
 config :meadow, :extra_mime_types, %{

app/config/dev.exs

@@ -1,5 +1,10 @@
 import Config
 
+config :ex_aws,
+  access_key_id: [:instance_role],
+  secret_access_key: [:instance_role],
+  region: System.get_env("AWS_REGION", "us-east-1")
+
 if System.get_env("AWS_DEV_ENVIRONMENT") |> is_nil() do
   # Configures lambda scripts
   config :meadow, :lambda,

app/config/prod.exs

@@ -0,0 +1,6 @@
+import Config
+
+config :ex_aws,
+  access_key_id: [:instance_role],
+  secret_access_key: [:instance_role],
+  region: System.get_env("AWS_REGION", "us-east-1")

app/config/test.exs

@@ -1,50 +1,19 @@
 import Config
 
-# We don't run a server during test. If one is required,
-# you can enable the server option below.
-config :meadow, MeadowWeb.Endpoint,
-  http: [port: 4002],
-  server: false
+config :ex_aws,
+  access_key_id: "fake",
+  secret_access_key: "fake",
+  region: "us-east-1"
 
-config :meadow,
-  index_interval: 1234,
-  mediaconvert_client: MediaConvert.Mock,
-  streaming_url: "https://test-streaming-url/",
-  iiif_server_url: "http://localhost:8184/iiif/3/",
-  iiif_manifest_url_deprecated: "http://test-pyramids.s3.localhost.localstack.cloud:4566/public/",
-  digital_collections_url: "https://fen.rdc-staging.library.northwestern.edu/"
-
-# Configures lambda scripts
-config :meadow, :lambda,
-  digester: {:local, {Path.expand("../lambdas/digester/index.js"), "handler"}},
-  exif: {:local, {Path.expand("../lambdas/exif/index.js"), "handler"}},
-  frame_extractor: {:local, {Path.expand("../lambdas/frame-extractor/index.js"), "handler"}},
-  mediainfo: {:local, {Path.expand("../lambdas/mediainfo/index.js"), "handler"}},
-  mime_type: {:local, {Path.expand("../lambdas/mime-type/index.js"), "handler"}},
-  tiff: {:local, {Path.expand("../lambdas/pyramid-tiff/index.js"), "handler"}}
-
-config :meadow,
-  checksum_notification: %{
-    arn: "arn:aws:lambda:us-east-1:000000000000:function:digest-tag",
-    buckets: ["test-ingest", "test-uploads"]
-  },
-  required_checksum_tags: ["computed-md5"],
-  checksum_wait_timeout: 15_000
-
-config :meadow,
-  ark: %{
-    default_shoulder: "ark:/12345/nu2",
-    user: "mockuser",
-    password: "mockpassword",
-    target_url: "https://devbox.library.northwestern.edu:3333/items/",
-    url: "http://localhost:3944/"
-  }
-
-config :meadow, :elasticsearch_retry,
-  interval: 100,
-  max_retries: 3
-
-config :authoritex, authorities: [Authoritex.Mock, NUL.Authority]
+if System.get_env("AWS_DEV_ENVIRONMENT") |> is_nil() do
+  [:mediaconvert, :s3, :secretsmanager, :sns, :sqs]
+  |> Enum.each(fn service ->
+    config :ex_aws, service,
+      scheme: "http://",
+      host: "localhost.localstack.cloud",
+      port: 4566
+  end)
+end
 
 config :ueberauth, Ueberauth,
   providers: [
@@ -58,51 +27,8 @@ config :ueberauth, Ueberauth,
        ]}
   ]
 
-config :meadow, Meadow.Repo,
-  show_sensitive_data_on_connection_error: true,
-  timeout: 60_000,
-  connect_timeout: 60_000,
-  handshake_timeout: 60_000,
-  pool: Ecto.Adapters.SQL.Sandbox,
-  queue_target: 5000,
-  pool_size: 50
-
-config :meadow,
-  dc_api: [
-    v2: %{
-      "api_token_secret" => "TEST_SECRET",
-      "api_token_ttl" => 300,
-      "base_url" => "http://dcapi-test.northwestern.edu"
-    }
-  ],
-  iiif_distribution_id: nil
-
-if System.get_env("AWS_DEV_ENVIRONMENT") |> is_nil() do
-  [:mediaconvert, :s3, :secretsmanager, :sns, :sqs]
-  |> Enum.each(fn service ->
-    config :ex_aws, service,
-      scheme: "http://",
-      host: "localhost.localstack.cloud",
-      port: 4566,
-      access_key_id: "fake",
-      secret_access_key: "fake",
-      region: "us-east-1"
-  end)
-end
-
-config :exldap, :settings, base: "OU=test,DC=library,DC=northwestern,DC=edu"
-
 # Print only warnings and errors during test
 config :logger, level: :info
 config :logger, :console, format: {Meadow.TestLogHandler, :format}
 
-config :ex_unit,
-  assert_receive_timeout: 500
-
-config :honeybadger,
-  environment_name: :test,
-  exclude_envs: [:dev, :test],
-  api_key: "abc123",
-  origin: "http://localhost:4444"
-
 config :elixir, :ansi_enabled, true

app/lib/meadow/config/runtime.ex

@@ -6,12 +6,13 @@ defmodule Meadow.Config.Runtime do
   alias Meadow.Config.Pipeline
 
   @config_map %{
-    meadow: "config/meadow",
-    inference: "infrastructure/inference",
+    ezid: "infrastructure/ezid",
+    honeybadger: "infrastructure/honeybadger",
+    iiif: "infrastructure/iiif",
     index: "infrastructure/index",
+    inference: "infrastructure/inference",
     ldap: "infrastructure/ldap",
-    iiif: "infrastructure/iiif",
-    honeybadger: "infrastructure/honeybadger",
+    meadow: "config/meadow",
     nusso: "infrastructure/nusso",
     wildcard_ssl: "config/wildcard_ssl"
   }

app/lib/meadow/config/runtime/dev.ex

@@ -5,12 +5,17 @@ defmodule Meadow.Config.Runtime.Dev do
 
   import Meadow.Config.Runtime
 
+  defp fetch_cert do
+    cert_path = :code.priv_dir(:meadow) |> Path.join("cert")
+    File.mkdir_p!(cert_path)
+    Path.join(cert_path, "cert.pem") |> File.write!(get_secret(:wildcard_ssl, ["certificate"]))
+    Path.join(cert_path, "key.pem") |> File.write!(get_secret(:wildcard_ssl, ["key"]))
+  end
+
   def configure! do
     import Config
 
-    File.mkdir_p!("priv/cert")
-    File.write!("priv/cert/cert.pem", get_secret(:wildcard_ssl, ["certificate"]))
-    File.write!("priv/cert/key.pem", get_secret(:wildcard_ssl, ["key"]))
+    fetch_cert()
 
     config :meadow, Meadow.Repo,
       show_sensitive_data_on_connection_error: true,

app/lib/meadow/config/runtime/test.ex

@@ -3,22 +3,90 @@ defmodule Meadow.Config.Runtime.Test do
   Load and apply Meadow's runtime configuration for the test environment
   """
 
-  import Meadow.Config.Runtime
-
   def configure! do
     import Config
 
-    config :meadow, Meadow.Search.Cluster,
-      url: get_secret(:meadow, ["search", "cluster_endpoint"], "http://localhost:9200"),
-      bulk_page_size: 3,
-      bulk_wait_interval: 2,
-      embedding_model_id: nil
-
-    config :meadow, :sitemaps,
-      gzip: true,
-      store: Sitemapper.S3Store,
-      base_url: "http://localhost:3333/",
-      sitemap_url: "http://localhost:3333/",
-      store_config: [bucket: prefix("uploads"), path: ""]
+    # We don't run a server during test. If one is required,
+    # you can enable the server option below.
+    config :meadow, MeadowWeb.Endpoint,
+      http: [port: 4002],
+      server: false
+
+    config :meadow,
+      index_interval: 1234,
+      mediaconvert_client: MediaConvert.Mock,
+      streaming_url: "https://test-streaming-url/",
+      iiif_server_url: "http://localhost:8184/iiif/3/",
+      iiif_manifest_url_deprecated:
+        "http://test-pyramids.s3.localhost.localstack.cloud:4566/public/",
+      digital_collections_url: "https://fen.rdc-staging.library.northwestern.edu/"
+
+    # Configures lambda scripts
+    config :meadow, :lambda,
+      digester: {:local, {Path.expand("../lambdas/digester/index.js"), "handler"}},
+      exif: {:local, {Path.expand("../lambdas/exif/index.js"), "handler"}},
+      frame_extractor: {:local, {Path.expand("../lambdas/frame-extractor/index.js"), "handler"}},
+      mediainfo: {:local, {Path.expand("../lambdas/mediainfo/index.js"), "handler"}},
+      mime_type: {:local, {Path.expand("../lambdas/mime-type/index.js"), "handler"}},
+      tiff: {:local, {Path.expand("../lambdas/pyramid-tiff/index.js"), "handler"}}
+
+    config :meadow,
+      checksum_notification: %{
+        arn: "arn:aws:lambda:us-east-1:000000000000:function:digest-tag",
+        buckets: ["test-ingest", "test-uploads"]
+      },
+      required_checksum_tags: ["computed-md5"],
+      checksum_wait_timeout: 15_000
+
+    config :meadow,
+      ark: %{
+        default_shoulder: "ark:/12345/nu2",
+        user: "mockuser",
+        password: "mockpassword",
+        target_url: "https://devbox.library.northwestern.edu:3333/items/",
+        url: "http://localhost:3944/"
+      }
+
+    config :meadow, :elasticsearch_retry,
+      interval: 100,
+      max_retries: 3
+
+    config :authoritex, authorities: [Authoritex.Mock, NUL.Authority]
+
+    config :meadow, Meadow.Repo,
+      show_sensitive_data_on_connection_error: true,
+      timeout: 60_000,
+      connect_timeout: 60_000,
+      handshake_timeout: 60_000,
+      pool: Ecto.Adapters.SQL.Sandbox,
+      queue_target: 5000,
+      pool_size: 50
+
+    config :meadow,
+      dc_api: [
+        v2: %{
+          "api_token_secret" => "TEST_SECRET",
+          "api_token_ttl" => 300,
+          "base_url" => "http://dcapi-test.northwestern.edu"
+        }
+      ],
+      iiif_distribution_id: nil
+
+    config :exldap, :settings,
+      base: "OU=test,DC=library,DC=northwestern,DC=edu",
+      server: "localhost",
+      port: 389,
+      user_dn: "cn=Administrator,cn=Users,dc=library,dc=northwestern,dc=edu",
+      password: "d0ck3rAdm1n!",
+      ssl: false
+
+    config :ex_unit,
+      assert_receive_timeout: 500
+
+    config :honeybadger,
+      environment_name: :test,
+      exclude_envs: [:dev, :test],
+      api_key: "abc123",
+      origin: "http://localhost:4444"
   end
 end

app/test/test_helper.exs

@@ -1,4 +1,38 @@
-Hush.resolve!()
+alias Meadow.Config.Runtime
+
+case :ets.info(:secret_cache, :name) do
+  :secret_cache -> :ets.delete_all_objects(:secret_cache)
+  :undefined -> :ets.new(:secret_cache, [:set, :protected, :named_table])
+end
+
+cluster_config =
+  Application.get_env(:meadow, Meadow.Search.Cluster)
+  |> Keyword.merge(
+    url:
+      Runtime.get_secret(
+        :meadow,
+        ["search", "cluster_endpoint"],
+        "http://localhost:9200"
+      ),
+    bulk_page_size: 3,
+    bulk_wait_interval: 2,
+    embedding_model_id: nil
+  )
+
+Application.put_env(:meadow, Meadow.Search.Cluster, cluster_config)
+
+sitemap_config =
+  Application.get_env(:meadow, :sitemaps)
+  |> Keyword.merge(
+    gzip: true,
+    store: Sitemapper.S3Store,
+    base_url: "http://localhost:3333/",
+    sitemap_url: "http://localhost:3333/",
+    store_config: [bucket: Runtime.prefix("uploads"), path: ""]
+  )
+
+Application.put_env(:meadow, :sitemaps, sitemap_config)
+
 Meadow.Repo.wait_for_connection()
 
 Mix.Task.run("ecto.setup")

infrastructure/docker-compose.yml

@@ -0,0 +1,38 @@
+---
+services:
+  db:
+    image: ghcr.io/nulib/postgres:10-alpine
+    environment:
+      POSTGRES_USER: docker
+      POSTGRES_PASSWORD: d0ck3r
+    healthcheck:
+      test: ["CMD", "pg_isready", "-U", "postgres"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    ports:
+      - 5432:5432
+  ldap:
+    image: ghcr.io/nulib/ldap-alpine
+    ports:
+      - 389:389
+      - 636:636
+  opensearch:
+    image: opensearchproject/opensearch:2.11.1
+    environment:
+      bootstrap.memory_lock: true
+      OPENSEARCH_JAVA_OPTS: "-Xms256m -Xmx256m"
+      DISABLE_INSTALL_DEMO_CONFIG: true
+      DISABLE_SECURITY_PLUGIN: true
+      discovery.type: single-node
+    ports:
+      - 9200:9200
+  localstack:
+    image: localstack/localstack
+    environment:
+      DOCKER_HOST: unix:///var/run/docker.sock
+      GATEWAY_LISTEN: 0.0.0.0:4566
+    ports:
+      - 4566:4566
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock:ro