Skip to content

Commit

Permalink
Current (finished?) state with hexkit issues
Browse files Browse the repository at this point in the history
  • Loading branch information
mephenor committed Jul 17, 2024
1 parent a8ee177 commit 60ae524
Show file tree
Hide file tree
Showing 10 changed files with 366 additions and 18 deletions.
6 changes: 2 additions & 4 deletions services/fins/src/fins/adapters/inbound/event_sub.py
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ async def _consume_file_internally_registered(self, *, payload: JsonObject):
)

await self._information_service.register_information(
file_information=validated_payload
file_registered=validated_payload
)


Expand Down Expand Up @@ -126,9 +126,7 @@ def __init__(
async def changed(
self, resource_id: str, update: event_schemas.FileDeletionRequested
) -> None:
"""Consume change event for File Deletion Requests.
Idempotence is handled by the core, so no intermediary is required.
"""
"""Consume change event for File Deletion Requests."""
await self.information_service.deletion_requested(file_id=update.file_id)

async def deleted(self, resource_id: str) -> None:
Expand Down
23 changes: 16 additions & 7 deletions services/fins/src/fins/core/information_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""TODO"""
"""Contains logic for public file information storage, retrieval and deletion."""

import logging

Expand Down Expand Up @@ -40,7 +40,8 @@ async def deletion_requested(self, file_id: str):
await self._file_information_dao.get_by_id(id_=file_id)
except ResourceNotFoundError:
log.info(
f"Information for file with id '{file_id}' has already been deleted."
f"Information for file with id '{
file_id}' has already been deleted."
)
return

Expand All @@ -56,19 +57,27 @@ async def register_information(
size=file_registered.decrypted_size,
sha256_hash=file_registered.decrypted_sha256,
)
file_id = file_information.file_id

try:
await self._file_information_dao.insert(file_information)
log.debug(f"Sucessfully inserted information for file {file_id} ")
except ResourceAlreadyExistsError:
information_exists = self.InformationAlreadyRegistered(
file_id=file_information.file_id
)
log.warning(information_exists)
existing_information = self._file_information_dao.get_by_id(id_=file_id)
log.debug(f"Found existing information for file {file_id}")
# Only log if information to be inserted is a mismatch
if existing_information != file_information:
information_exists = self.MismatchingInformationAlreadyRegistered(
file_id=file_id
)
log.error(information_exists)

async def serve_information(self, file_id: str) -> FileInformation:
"""Retrieve stored public information for the five file ID."""
"""Retrieve stored public information for the five file ID to be served by API."""
try:
file_information = await self._file_information_dao.get_by_id(file_id)
log.debug(f"Information for file {
file_information.file_id} has been served.")
except ResourceNotFoundError as error:
information_not_found = self.InformationNotFoundError(file_id=file_id)
log.warning(information_not_found)
Expand Down
2 changes: 1 addition & 1 deletion services/fins/src/fins/core/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,4 +36,4 @@ class FileInformation(BaseModel):


class FileDeletionRequested(event_schemas.FileDeletionRequested):
"""TODO"""
"""Internal event_schema alias for DAO/DTO modelling purposes."""
14 changes: 8 additions & 6 deletions services/fins/src/fins/ports/inbound/information_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""TODO"""
"""Contains interfaces for public file information storage, retrieval and deletion."""

from abc import ABC, abstractmethod

Expand All @@ -26,18 +26,20 @@ class InformationServicePort(ABC):
metadata for files registered with the Internal File Registry service.
"""

class InformationAlreadyRegistered(RuntimeError):
class MismatchingInformationAlreadyRegistered(RuntimeError):
"""Raised when information for a given file ID is not registered."""

def __init__(self, *, file_id: str):
message = f"Information for the file with ID {file_id} has already been registered."
message = f"Mismatching information for the file with ID {
file_id} has already been registered."
super().__init__(message)

class InformationNotFoundError(RuntimeError):
"""Raised when information for a given file ID is not registered."""

def __init__(self, *, file_id: str):
message = f"Information for the file with ID {file_id} is not registered."
message = f"Information for the file with ID {
file_id} is not registered."
super().__init__(message)

@abstractmethod
Expand All @@ -46,10 +48,10 @@ async def deletion_requested(self, file_id: str):

@abstractmethod
async def register_information(
self, file_information: event_schemas.FileInternallyRegistered
self, file_registered: event_schemas.FileInternallyRegistered
):
"""Store information for a file newly registered with the Internal File Registry."""

@abstractmethod
async def serve_information(self, file_id: str) -> FileInformation:
"""Retrieve stored public information for the five file ID."""
"""Retrieve stored public information for the five file ID to be served by API."""
1 change: 1 addition & 0 deletions services/fins/src/fins/py.typed
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
# Marker file for PEP 561. This package uses inline types.
41 changes: 41 additions & 0 deletions services/fins/tests_fins/fixtures/config.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
# Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
# for the German Human Genome-Phenome Archive (GHGA)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Test config"""

from pathlib import Path

from pydantic_settings import BaseSettings

from fins.config import Config
from tests_fins.fixtures.utils import BASE_DIR

TEST_CONFIG_YAML = BASE_DIR / "test_config.yaml"


def get_config(
sources: list[BaseSettings] | None = None,
default_config_yaml: Path = TEST_CONFIG_YAML,
) -> Config:
"""Merges parameters from the default TEST_CONFIG_YAML with params inferred
from testcontainers.
"""
sources_dict: dict[str, object] = {}

if sources is not None:
for source in sources:
sources_dict.update(**source.model_dump())

return Config(config_yaml=default_config_yaml, **sources_dict)
148 changes: 148 additions & 0 deletions services/fins/tests_fins/fixtures/joint.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,148 @@
# Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
# for the German Human Genome-Phenome Archive (GHGA)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Join the functionality of all fixtures for API-level integration testing."""

__all__ = [
"FILE_INFORMATION_MOCK",
"INCOMING_PAYLOAD_MOCK",
"joint_fixture",
"JointFixture",
"kafka_container_fixture",
"kafka_fixture",
"mongodb_container_fixture",
"mongodb_fixture",
]
from collections.abc import AsyncGenerator
from dataclasses import dataclass

import ghga_event_schemas.pydantic_ as event_schemas
import httpx
import pytest_asyncio
from ghga_service_commons.api.testing import AsyncTestClient
from ghga_service_commons.utils.utc_dates import now_as_utc
from hexkit.providers.akafka import KafkaEventSubscriber, KafkaOutboxSubscriber
from hexkit.providers.akafka.testutils import (
KafkaFixture,
kafka_container_fixture,
kafka_fixture,
)
from hexkit.providers.mongodb import MongoDbDaoFactory
from hexkit.providers.mongodb.testutils import (
MongoDbFixture,
mongodb_container_fixture,
mongodb_fixture,
)

from fins.adapters.inbound.dao import (
get_file_deletion_requested_dao,
get_file_information_dao,
)
from fins.config import Config
from fins.core import models
from fins.inject import (
prepare_core,
prepare_event_subscriber,
prepare_outbox_subscriber,
prepare_rest_app,
)
from fins.ports.inbound.dao import FileDeletionRequestedDaoPort, FileInformationDaoPort
from fins.ports.inbound.information_service import InformationServicePort
from tests_fins.fixtures.config import get_config

FILE_ID = "test-file"
DECRYPTED_SHA256 = "fake-checksum"
DECRYPTED_SIZE = 12345678

INCOMING_PAYLOAD_MOCK = event_schemas.FileInternallyRegistered(
s3_endpoint_alias="test-node",
file_id=FILE_ID,
object_id="test-object",
bucket_id="test-bucket",
upload_date=now_as_utc().isoformat(),
decrypted_size=DECRYPTED_SIZE,
decrypted_sha256=DECRYPTED_SHA256,
encrypted_part_size=1,
encrypted_parts_md5=["some", "checksum"],
encrypted_parts_sha256=["some", "checksum"],
content_offset=1234,
decryption_secret_id="some-secret",
)

FILE_INFORMATION_MOCK = models.FileInformation(
file_id=FILE_ID, sha256_hash=DECRYPTED_SHA256, size=DECRYPTED_SIZE
)


@dataclass
class JointFixture:
"""Returned by the `joint_fixture`."""

config: Config
information_service: InformationServicePort
file_information_dao: FileInformationDaoPort
file_deletion_requested_dao: FileDeletionRequestedDaoPort
rest_client: httpx.AsyncClient
event_subscriber: KafkaEventSubscriber
outbox_subscriber: KafkaOutboxSubscriber
mongodb: MongoDbFixture
kafka: KafkaFixture


@pytest_asyncio.fixture
async def joint_fixture(
mongodb: MongoDbFixture,
kafka: KafkaFixture,
) -> AsyncGenerator[JointFixture, None]:
"""A fixture that embeds all other fixtures for API-level integration testing"""
config = get_config(
sources=[
mongodb.config,
kafka.config,
]
)

dao_factory = MongoDbDaoFactory(config=config)
file_information_dao = await get_file_information_dao(dao_factory=dao_factory)
file_deletion_requested_dao = await get_file_deletion_requested_dao(
dao_factory=dao_factory
)

# prepare everything except the outbox subscriber
async with (
prepare_core(config=config) as information_service,
prepare_rest_app(
config=config, information_service_override=information_service
) as app,
prepare_event_subscriber(
config=config, information_service_override=information_service
) as event_subscriber,
prepare_outbox_subscriber(
config=config,
information_service_override=information_service,
) as outbox_subscriber,
AsyncTestClient(app=app) as rest_client,
):
yield JointFixture(
config=config,
information_service=information_service,
file_deletion_requested_dao=file_deletion_requested_dao,
file_information_dao=file_information_dao,
rest_client=rest_client,
event_subscriber=event_subscriber,
outbox_subscriber=outbox_subscriber,
mongodb=mongodb,
kafka=kafka,
)
11 changes: 11 additions & 0 deletions services/fins/tests_fins/fixtures/test_config.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
service_name: fins

service_instance_id: '1'
kafka_servers: ["kafka:9092"]

db_connection_str: "mongodb://mongodb:27017"
db_name: "dev_db"

files_to_delete_topic: file_deletions
file_registered_event_topic: internal_file_registry
file_registered_event_type: file_registered
20 changes: 20 additions & 0 deletions services/fins/tests_fins/fixtures/utils.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
# Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
# for the German Human Genome-Phenome Archive (GHGA)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""General testing utilities"""

from pathlib import Path

BASE_DIR = Path(__file__).parent.resolve()
Loading

0 comments on commit 60ae524

Please sign in to comment.