From 27cdb46e6982c348c4213d588ec362d15a2b4de4 Mon Sep 17 00:00:00 2001 From: "Thomas J. Zajac" Date: Mon, 15 Jul 2024 12:31:08 +0000 Subject: [PATCH] Update scripts --- .devcontainer/docker-compose.yml | 1 + README.md | 1 + scripts/update_config_docs.py | 6 +- .../fins/.readme_generation/description.md | 3 +- services/fins/README.md | 182 +++--------- services/fins/config_schema.json | 273 ++++++------------ services/fins/dev_config.yaml | 18 +- services/fins/example_config.yaml | 20 +- services/fins/openapi.yaml | 205 +++++-------- services/fins/pyproject.toml | 2 +- .../adapters/inbound/fastapi_/configure.py | 6 +- .../fins/adapters/inbound/fastapi_/routes.py | 3 +- 12 files changed, 222 insertions(+), 498 deletions(-) diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index 34b800f4..3178b8c2 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -36,3 +36,4 @@ services: DCS_CONFIG_YAML: /workspace/services/dcs/dev_config.yaml EKSS_CONFIG_YAML: /workspace/services/ekss/dev_config.yaml FIS_CONFIG_YAML: /workspace/services/fis/dev_config.yaml + FINS_CONFIG_YAML: /workspace/services/fins/dev_config.yaml diff --git a/README.md b/README.md index b6781908..b9a30dad 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ This is a monorepo containing all GHGA file backend microservices. [Download Controller Service](services/dcs/README.md) [Encryption Key Store Service](services/ekss/README.md) +[File Information Service](services/fins/README.md) [File Ingest Service](services/fis/README.md) [Internal File Registry Service](services/ifrs/README.md) [Interrogation Room Service](services/irs/README.md) diff --git a/scripts/update_config_docs.py b/scripts/update_config_docs.py index b0714b35..635400a3 100755 --- a/scripts/update_config_docs.py +++ b/scripts/update_config_docs.py @@ -92,7 +92,8 @@ def get_schema(service: str) -> str: """Returns a JSON schema generated from a Config class.""" config = get_dev_config(service) - return config.schema_json(indent=2) # change eventually to .model_json_schema(...) + # change eventually to .model_json_schema(...) + return config.schema_json(indent=2) def get_example(service: str) -> str: @@ -144,7 +145,8 @@ def check_docs(service: str): if example_expected != example_observed: print_diff(example_expected, example_observed) raise ValidationError( - f"Example config YAML at '{example_config_yaml}' is not up to date." + f"Example config YAML at '{ + example_config_yaml}' is not up to date." ) schema_expected = get_schema(service) diff --git a/services/fins/.readme_generation/description.md b/services/fins/.readme_generation/description.md index 314b7512..2d5e9bd9 100644 --- a/services/fins/.readme_generation/description.md +++ b/services/fins/.readme_generation/description.md @@ -1 +1,2 @@ -The File Information Service exposes metadata about files available TODO. +The File Information Service serves publicly available metadata about files registered with the Internal File Registry service. +Currently this includes the SHA256 checksum of the unencrypted file content and the size of the unencrypted file in bytes. diff --git a/services/fins/README.md b/services/fins/README.md index 5bfcee0b..818e1bfb 100644 --- a/services/fins/README.md +++ b/services/fins/README.md @@ -1,27 +1,25 @@ -# File Ingest Service +# File Information Service -A lightweight service to propagate file upload metadata to the GHGA file backend services +Providing public metadata about files registered with the Internal File Registry ## Description -The File Ingest Service provides an endpoint to populate the Encryption Key Store, -Internal File Registry and Download Controller with output metadata from the S3 upload -script at https://github.com/ghga-de/data-steward-scripts/blob/main/src/s3_upload.py. - +The File Information Service serves publicly available metadata about files registered with the Internal File Registry service. +Currently this includes the SHA256 checksum of the unencrypted file content and the size of the unencrypted file in bytes. ## Installation We recommend using the provided Docker container. -A pre-build version is available at [docker hub](https://hub.docker.com/repository/docker/ghga/file-ingest-service): +A pre-build version is available at [docker hub](https://hub.docker.com/repository/docker/ghga/file-information-service): ```bash -docker pull ghga/file-ingest-service:3.0.0 +docker pull ghga/file-information-service:1.0.0 ``` Or you can build the container yourself from the [`./Dockerfile`](./Dockerfile): ```bash # Execute in the repo's root dir: -docker build -t ghga/file-ingest-service:3.0.0 . +docker build -t ghga/file-information-service:1.0.0 . ``` For production-ready deployment, we recommend using Kubernetes, however, @@ -29,7 +27,7 @@ for simple use cases, you could execute the service using docker on a single server: ```bash # The entrypoint is preconfigured: -docker run -p 8080:8080 ghga/file-ingest-service:3.0.0 --help +docker run -p 8080:8080 ghga/file-information-service:1.0.0 --help ``` If you prefer not to use containers, you may install the service from source: @@ -38,7 +36,7 @@ If you prefer not to use containers, you may install the service from source: pip install . # To run the service: -fis --help +fins --help ``` ## Configuration @@ -46,9 +44,19 @@ fis --help ### Parameters The service requires the following configuration parameters: +- **`files_to_delete_topic`** *(string)*: The name of the topic for events informing about files to be deleted. + + + Examples: + + ```json + "file_deletions" + ``` + + - **`log_level`** *(string)*: The minimum log level to capture. Must be one of: `["CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG", "TRACE"]`. Default: `"INFO"`. -- **`service_name`** *(string)*: Default: `"fis"`. +- **`service_name`** *(string)*: Default: `"fins"`. - **`service_instance_id`** *(string)*: A string that uniquely identifies this instance across all instances of this service. A globally unique Kafka client ID will be created by concatenating the service_name and the service_instance_id. @@ -83,119 +91,82 @@ The service requires the following configuration parameters: - **`log_traceback`** *(boolean)*: Whether to include exception tracebacks in log messages. Default: `true`. -- **`vault_url`** *(string)*: URL of the vault instance to connect to. +- **`kafka_servers`** *(array)*: A list of connection strings to connect to Kafka bootstrap servers. + + - **Items** *(string)* Examples: ```json - "http://127.0.0.1.8200" + [ + "localhost:9092" + ] ``` -- **`vault_role_id`**: Vault role ID to access a specific prefix. Default: `null`. - - - **Any of** - - - *string, format: password* - - - *null* - - - Examples: - - ```json - "example_role" - ``` +- **`kafka_security_protocol`** *(string)*: Protocol used to communicate with brokers. Valid values are: PLAINTEXT, SSL. Must be one of: `["PLAINTEXT", "SSL"]`. Default: `"PLAINTEXT"`. +- **`kafka_ssl_cafile`** *(string)*: Certificate Authority file path containing certificates used to sign broker certificates. If a CA is not specified, the default system CA will be used if found by OpenSSL. Default: `""`. -- **`vault_secret_id`**: Vault secret ID to access a specific prefix. Default: `null`. +- **`kafka_ssl_certfile`** *(string)*: Optional filename of client certificate, as well as any CA certificates needed to establish the certificate's authenticity. Default: `""`. - - **Any of** +- **`kafka_ssl_keyfile`** *(string)*: Optional filename containing the client private key. Default: `""`. - - *string, format: password* +- **`kafka_ssl_password`** *(string, format: password)*: Optional password to be used for the client private key. Default: `""`. - - *null* +- **`generate_correlation_id`** *(boolean)*: A flag, which, if False, will result in an error when inbound requests don't possess a correlation ID. If True, requests without a correlation ID will be assigned a newly generated ID in the correlation ID middleware function. Default: `true`. Examples: ```json - "example_secret" + true ``` -- **`vault_verify`**: SSL certificates (CA bundle) used to verify the identity of the vault, or True to use the default CAs, or False for no verification. Default: `true`. - - - **Any of** - - - *boolean* - - - *string* - - - Examples: - ```json - "/etc/ssl/certs/my_bundle.pem" + false ``` -- **`vault_path`** *(string)*: Path without leading or trailing slashes where secrets should be stored in the vault. - -- **`vault_secrets_mount_point`** *(string)*: Name used to address the secret engine under a custom mount path. Default: `"secret"`. +- **`db_connection_str`** *(string, format: password)*: MongoDB connection string. Might include credentials. For more information see: https://naiveskill.com/mongodb-connection-string/. Examples: ```json - "secret" + "mongodb://localhost:27017" ``` -- **`vault_kube_role`**: Vault role name used for Kubernetes authentication. Default: `null`. - - - **Any of** - - - *string* - - - *null* +- **`db_name`** *(string)*: Name of the database located on the MongoDB server. Examples: ```json - "file-ingest-role" + "my-database" ``` -- **`service_account_token_path`** *(string, format: path)*: Path to service account token used by kube auth adapter. Default: `"/var/run/secrets/kubernetes.io/serviceaccount/token"`. - -- **`private_key`** *(string)*: Base64 encoded private key of the keypair whose public key is used to encrypt the payload. - -- **`source_bucket_id`** *(string)*: ID of the bucket the object(s) corresponding to the upload metadata have been uploaded to. This should currently point to the staging bucket. - -- **`token_hashes`** *(array)*: List of token hashes corresponding to the tokens that can be used to authenticate calls to this service. - - - **Items** *(string)* - -- **`file_upload_validation_success_topic`** *(string)*: The name of the topic use to publish FileUploadValidationSuccess events. +- **`file_registered_event_topic`** *(string)*: The name of the topic for events informing about new registered files for which the metadata should be made available. Examples: ```json - "file_upload_validation_success" + "internal_file_registry" ``` -- **`file_validations_collection`** *(string)*: The name of the collection used to store FileUploadValidationSuccess events. Default: `"file-validations"`. +- **`file_registered_event_type`** *(string)*: The name of the type used for events informing about new registered files for which the metadata should be made available. Examples: ```json - "file-validations" + "file_registered" ``` @@ -291,73 +262,14 @@ The service requires the following configuration parameters: ``` -- **`generate_correlation_id`** *(boolean)*: A flag, which, if False, will result in an error when trying to publish an event without a valid correlation ID set for the context. If True, the a newly correlation ID will be generated and used in the event header. Default: `true`. - - - Examples: - - ```json - true - ``` - - - ```json - false - ``` - - -- **`kafka_servers`** *(array)*: A list of connection strings to connect to Kafka bootstrap servers. - - - **Items** *(string)* - - - Examples: - - ```json - [ - "localhost:9092" - ] - ``` - - -- **`kafka_security_protocol`** *(string)*: Protocol used to communicate with brokers. Valid values are: PLAINTEXT, SSL. Must be one of: `["PLAINTEXT", "SSL"]`. Default: `"PLAINTEXT"`. - -- **`kafka_ssl_cafile`** *(string)*: Certificate Authority file path containing certificates used to sign broker certificates. If a CA is not specified, the default system CA will be used if found by OpenSSL. Default: `""`. - -- **`kafka_ssl_certfile`** *(string)*: Optional filename of client certificate, as well as any CA certificates needed to establish the certificate's authenticity. Default: `""`. - -- **`kafka_ssl_keyfile`** *(string)*: Optional filename containing the client private key. Default: `""`. - -- **`kafka_ssl_password`** *(string, format: password)*: Optional password to be used for the client private key. Default: `""`. - -- **`db_connection_str`** *(string, format: password)*: MongoDB connection string. Might include credentials. For more information see: https://naiveskill.com/mongodb-connection-string/. - - - Examples: - - ```json - "mongodb://localhost:27017" - ``` - - -- **`db_name`** *(string)*: Name of the database located on the MongoDB server. - - - Examples: - - ```json - "my-database" - ``` - - ### Usage: A template YAML for configuring the service can be found at [`./example-config.yaml`](./example-config.yaml). -Please adapt it, rename it to `.fis.yaml`, and place it into one of the following locations: -- in the current working directory were you are execute the service (on unix: `./.fis.yaml`) -- in your home directory (on unix: `~/.fis.yaml`) +Please adapt it, rename it to `.fins.yaml`, and place it into one of the following locations: +- in the current working directory were you are execute the service (on unix: `./.fins.yaml`) +- in your home directory (on unix: `~/.fins.yaml`) The config yaml will be automatically parsed by the service. @@ -366,8 +278,8 @@ The config yaml will be automatically parsed by the service. All parameters mentioned in the [`./example-config.yaml`](./example-config.yaml) could also be set using environment variables or file secrets. -For naming the environment variables, just prefix the parameter name with `fis_`, -e.g. for the `host` set an environment variable named `fis_host` +For naming the environment variables, just prefix the parameter name with `fins_`, +e.g. for the `host` set an environment variable named `fins_host` (you may use both upper or lower cases, however, it is standard to define all env variables in upper cases). diff --git a/services/fins/config_schema.json b/services/fins/config_schema.json index 46b63fa3..3809e62f 100644 --- a/services/fins/config_schema.json +++ b/services/fins/config_schema.json @@ -2,6 +2,14 @@ "additionalProperties": false, "description": "Modifies the orginal Settings class provided by the user", "properties": { + "files_to_delete_topic": { + "description": "The name of the topic for events informing about files to be deleted.", + "examples": [ + "file_deletions" + ], + "title": "Files To Delete Topic", + "type": "string" + }, "log_level": { "default": "INFO", "description": "The minimum log level to capture.", @@ -17,7 +25,7 @@ "type": "string" }, "service_name": { - "default": "fis", + "default": "fins", "title": "Service Name", "type": "string" }, @@ -52,136 +60,97 @@ "title": "Log Traceback", "type": "boolean" }, - "vault_url": { - "description": "URL of the vault instance to connect to", + "kafka_servers": { + "description": "A list of connection strings to connect to Kafka bootstrap servers.", "examples": [ - "http://127.0.0.1.8200" + [ + "localhost:9092" + ] ], - "title": "Vault Url", - "type": "string" + "items": { + "type": "string" + }, + "title": "Kafka Servers", + "type": "array" }, - "vault_role_id": { - "anyOf": [ - { - "format": "password", - "type": "string", - "writeOnly": true - }, - { - "type": "null" - } - ], - "default": null, - "description": "Vault role ID to access a specific prefix", - "examples": [ - "example_role" + "kafka_security_protocol": { + "default": "PLAINTEXT", + "description": "Protocol used to communicate with brokers. Valid values are: PLAINTEXT, SSL.", + "enum": [ + "PLAINTEXT", + "SSL" ], - "title": "Vault Role Id" + "title": "Kafka Security Protocol", + "type": "string" }, - "vault_secret_id": { - "anyOf": [ - { - "format": "password", - "type": "string", - "writeOnly": true - }, - { - "type": "null" - } - ], - "default": null, - "description": "Vault secret ID to access a specific prefix", - "examples": [ - "example_secret" - ], - "title": "Vault Secret Id" + "kafka_ssl_cafile": { + "default": "", + "description": "Certificate Authority file path containing certificates used to sign broker certificates. If a CA is not specified, the default system CA will be used if found by OpenSSL.", + "title": "Kafka Ssl Cafile", + "type": "string" }, - "vault_verify": { - "anyOf": [ - { - "type": "boolean" - }, - { - "type": "string" - } - ], - "default": true, - "description": "SSL certificates (CA bundle) used to verify the identity of the vault, or True to use the default CAs, or False for no verification.", - "examples": [ - "/etc/ssl/certs/my_bundle.pem" - ], - "title": "Vault Verify" + "kafka_ssl_certfile": { + "default": "", + "description": "Optional filename of client certificate, as well as any CA certificates needed to establish the certificate's authenticity.", + "title": "Kafka Ssl Certfile", + "type": "string" }, - "vault_path": { - "description": "Path without leading or trailing slashes where secrets should be stored in the vault.", - "title": "Vault Path", + "kafka_ssl_keyfile": { + "default": "", + "description": "Optional filename containing the client private key.", + "title": "Kafka Ssl Keyfile", "type": "string" }, - "vault_secrets_mount_point": { - "default": "secret", - "description": "Name used to address the secret engine under a custom mount path.", + "kafka_ssl_password": { + "default": "", + "description": "Optional password to be used for the client private key.", + "format": "password", + "title": "Kafka Ssl Password", + "type": "string", + "writeOnly": true + }, + "generate_correlation_id": { + "default": true, + "description": "A flag, which, if False, will result in an error when inbound requests don't possess a correlation ID. If True, requests without a correlation ID will be assigned a newly generated ID in the correlation ID middleware function.", "examples": [ - "secret" + true, + false ], - "title": "Vault Secrets Mount Point", - "type": "string" + "title": "Generate Correlation Id", + "type": "boolean" }, - "vault_kube_role": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Vault role name used for Kubernetes authentication", + "db_connection_str": { + "description": "MongoDB connection string. Might include credentials. For more information see: https://naiveskill.com/mongodb-connection-string/", "examples": [ - "file-ingest-role" + "mongodb://localhost:27017" ], - "title": "Vault Kube Role" - }, - "service_account_token_path": { - "default": "/var/run/secrets/kubernetes.io/serviceaccount/token", - "description": "Path to service account token used by kube auth adapter.", - "format": "path", - "title": "Service Account Token Path", - "type": "string" - }, - "private_key": { - "description": "Base64 encoded private key of the keypair whose public key is used to encrypt the payload.", - "title": "Private Key", - "type": "string" + "format": "password", + "title": "Db Connection Str", + "type": "string", + "writeOnly": true }, - "source_bucket_id": { - "description": "ID of the bucket the object(s) corresponding to the upload metadata have been uploaded to. This should currently point to the staging bucket.", - "title": "Source Bucket Id", + "db_name": { + "description": "Name of the database located on the MongoDB server.", + "examples": [ + "my-database" + ], + "title": "Db Name", "type": "string" }, - "token_hashes": { - "description": "List of token hashes corresponding to the tokens that can be used to authenticate calls to this service.", - "items": { - "type": "string" - }, - "title": "Token Hashes", - "type": "array" - }, - "file_upload_validation_success_topic": { - "description": "The name of the topic use to publish FileUploadValidationSuccess events.", + "file_registered_event_topic": { + "description": "The name of the topic for events informing about new registered files for which the metadata should be made available.", "examples": [ - "file_upload_validation_success" + "internal_file_registry" ], - "title": "File Upload Validation Success Topic", + "title": "File Registered Event Topic", "type": "string" }, - "file_validations_collection": { - "default": "file-validations", - "description": "The name of the collection used to store FileUploadValidationSuccess events.", + "file_registered_event_type": { + "description": "The name of the type used for events informing about new registered files for which the metadata should be made available.", "examples": [ - "file-validations" + "file_registered" ], - "title": "File Validations Collection", + "title": "File Registered Event Type", "type": "string" }, "host": { @@ -306,96 +275,16 @@ [] ], "title": "Cors Allowed Headers" - }, - "generate_correlation_id": { - "default": true, - "description": "A flag, which, if False, will result in an error when trying to publish an event without a valid correlation ID set for the context. If True, the a newly correlation ID will be generated and used in the event header.", - "examples": [ - true, - false - ], - "title": "Generate Correlation Id", - "type": "boolean" - }, - "kafka_servers": { - "description": "A list of connection strings to connect to Kafka bootstrap servers.", - "examples": [ - [ - "localhost:9092" - ] - ], - "items": { - "type": "string" - }, - "title": "Kafka Servers", - "type": "array" - }, - "kafka_security_protocol": { - "default": "PLAINTEXT", - "description": "Protocol used to communicate with brokers. Valid values are: PLAINTEXT, SSL.", - "enum": [ - "PLAINTEXT", - "SSL" - ], - "title": "Kafka Security Protocol", - "type": "string" - }, - "kafka_ssl_cafile": { - "default": "", - "description": "Certificate Authority file path containing certificates used to sign broker certificates. If a CA is not specified, the default system CA will be used if found by OpenSSL.", - "title": "Kafka Ssl Cafile", - "type": "string" - }, - "kafka_ssl_certfile": { - "default": "", - "description": "Optional filename of client certificate, as well as any CA certificates needed to establish the certificate's authenticity.", - "title": "Kafka Ssl Certfile", - "type": "string" - }, - "kafka_ssl_keyfile": { - "default": "", - "description": "Optional filename containing the client private key.", - "title": "Kafka Ssl Keyfile", - "type": "string" - }, - "kafka_ssl_password": { - "default": "", - "description": "Optional password to be used for the client private key.", - "format": "password", - "title": "Kafka Ssl Password", - "type": "string", - "writeOnly": true - }, - "db_connection_str": { - "description": "MongoDB connection string. Might include credentials. For more information see: https://naiveskill.com/mongodb-connection-string/", - "examples": [ - "mongodb://localhost:27017" - ], - "format": "password", - "title": "Db Connection Str", - "type": "string", - "writeOnly": true - }, - "db_name": { - "description": "Name of the database located on the MongoDB server.", - "examples": [ - "my-database" - ], - "title": "Db Name", - "type": "string" } }, "required": [ + "files_to_delete_topic", "service_instance_id", - "vault_url", - "vault_path", - "private_key", - "source_bucket_id", - "token_hashes", - "file_upload_validation_success_topic", "kafka_servers", "db_connection_str", - "db_name" + "db_name", + "file_registered_event_topic", + "file_registered_event_type" ], "title": "ModSettings", "type": "object" diff --git a/services/fins/dev_config.yaml b/services/fins/dev_config.yaml index 6788a7e0..e4c5af07 100644 --- a/services/fins/dev_config.yaml +++ b/services/fins/dev_config.yaml @@ -1,19 +1,11 @@ -service_name: fis -source_bucket_id: staging -private_key: dummy-key -token_hashes: [abcdef, ghijkl] - -file_upload_validation_success_topic: file_upload_validation_success +service_name: fins service_instance_id: '1' kafka_servers: ["kafka:9092"] -vault_url: "http://127.0.0.1:8200" -vault_role_id: dummy-role -vault_secret_id: dummy-secret -vault_verify: True -vault_path: ekss -vault_secrets_mount_point: secret -vault_kube_role: dummy-role db_connection_str: "mongodb://mongodb:27017" db_name: "dev_db" + +files_to_delete_topic: file_deletions +file_registered_event_topic: internal_file_registry +file_registered_event_type: file_registered diff --git a/services/fins/example_config.yaml b/services/fins/example_config.yaml index 94fd1b2f..b6e7dd87 100644 --- a/services/fins/example_config.yaml +++ b/services/fins/example_config.yaml @@ -7,8 +7,9 @@ cors_allowed_origins: null db_connection_str: '**********' db_name: dev_db docs_url: /docs -file_upload_validation_success_topic: file_upload_validation_success -file_validations_collection: file-validations +file_registered_event_topic: internal_file_registry +file_registered_event_type: file_registered +files_to_delete_topic: file_deletions generate_correlation_id: true host: 127.0.0.1 kafka_security_protocol: PLAINTEXT @@ -23,19 +24,6 @@ log_level: INFO log_traceback: true openapi_url: /openapi.json port: 8080 -private_key: dummy-key -service_account_token_path: /var/run/secrets/kubernetes.io/serviceaccount/token service_instance_id: '1' -service_name: fis -source_bucket_id: staging -token_hashes: -- abcdef -- ghijkl -vault_kube_role: dummy-role -vault_path: ekss -vault_role_id: '**********' -vault_secret_id: '**********' -vault_secrets_mount_point: secret -vault_url: http://127.0.0.1:8200 -vault_verify: true +service_name: fins workers: 1 diff --git a/services/fins/openapi.yaml b/services/fins/openapi.yaml index 1f5de499..8494a0ea 100644 --- a/services/fins/openapi.yaml +++ b/services/fins/openapi.yaml @@ -1,17 +1,30 @@ components: schemas: - EncryptedPayload: - description: 'Generic model for an encrypted payload. + FileInformation: + description: 'Basic public information container for files registered with the + Internal File - - Can correspond to current/legacy upload metadata or a file secret.' + Registry service.' properties: - payload: - title: Payload + file_id: + description: Public identifier of the file associated with the given information + title: File Id + type: string + sha256_hash: + description: SHA256 hash of the unencrypted file content encoded as hexadecimal values + as produced by hashlib.hexdigest(). + title: Sha256 Hash type: string + size: + description: Size of the unencrypted file in bytes. + exclusiveMinimum: 0.0 + title: Size + type: integer required: - - payload - title: EncryptedPayload + - file_id + - size + - sha256_hash + title: FileInformation type: object HTTPValidationError: properties: @@ -22,58 +35,36 @@ components: type: array title: HTTPValidationError type: object - UploadMetadata: - description: 'Current model including a secret ID that can be used to retrieve - a stored secret - - in place of the actual secret.' + HttpInformationNotFoundError: + additionalProperties: false properties: - encrypted_md5_checksums: - items: - type: string - title: Encrypted Md5 Checksums - type: array - encrypted_sha256_checksums: - items: - type: string - title: Encrypted Sha256 Checksums - type: array - encrypted_size: - title: Encrypted Size - type: integer - file_id: - title: File Id + data: + $ref: '#/components/schemas/HttpInformationNotFoundErrorData' + description: + description: A human readable message to the client explaining the cause + of the exception. + title: Description type: string - object_id: - title: Object Id + exception_id: + const: informationNotFound + enum: + - informationNotFound + title: Exception Id type: string - part_size: - title: Part Size - type: integer - secret_id: - title: Secret Id - type: string - storage_alias: - title: Storage Alias - type: string - unencrypted_checksum: - title: Unencrypted Checksum + required: + - data + - description + - exception_id + title: HttpInformationNotFoundError + type: object + HttpInformationNotFoundErrorData: + properties: + file_id: + title: File Id type: string - unencrypted_size: - title: Unencrypted Size - type: integer required: - file_id - - object_id - - part_size - - unencrypted_size - - encrypted_size - - unencrypted_checksum - - encrypted_md5_checksums - - encrypted_sha256_checksums - - storage_alias - - secret_id - title: UploadMetadata + title: HttpInformationNotFoundErrorData type: object ValidationError: properties: @@ -96,71 +87,47 @@ components: - type title: ValidationError type: object - securitySchemes: - HTTPBearer: - scheme: bearer - type: http info: - description: A service to ingest s3 file upload metadata produced by thedata-steward-kit - upload command - title: File Ingest Service - version: 3.0.0 + description: Providing public metadata about files registered with the Internal + File Registry + title: File Information Service + version: 1.0.0 openapi: 3.1.0 paths: - /federated/ingest_metadata: - post: - description: Process metadata, file secret id and send success event - operationId: ingestFileUploadMetadata - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/UploadMetadata' + /file_information/{file_id}: + get: + description: Retrieve and serve stored file information. + operationId: getFileInformation + parameters: + - in: path + name: file_id required: true + schema: + title: File Id + type: string responses: - '202': - content: - application/json: - schema: {} - description: Received and decrypted data successfully. - '422': + '200': content: application/json: schema: - $ref: '#/components/schemas/HTTPValidationError' - description: Validation Error - security: - - HTTPBearer: [] - summary: Processes encrypted output data from the S3 upload script and ingests - it into the Encryption Key Store, Internal File Registry and Download Controller. - tags: - - FileIngestService - /federated/ingest_secret: - post: - description: Decrypt payload and deposit file secret in exchange for a secret - id - operationId: ingestSecret - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/EncryptedPayload' - required: true - responses: - '200': + $ref: '#/components/schemas/FileInformation' + description: A configuration or external communication error has occurred + and details should not be communicated to the client + '404': content: application/json: - schema: {} - description: Received and stored secret successfully. + schema: + $ref: '#/components/schemas/HttpInformationNotFoundError' + description: 'Exceptions by ID: + + - informationNotFound: No information registered for the given ID.' '422': content: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' description: Validation Error - security: - - HTTPBearer: [] - summary: Store file encryption/decryption secret and return secret ID. + summary: Return public file information for the given file id, i.e. public accession. tags: - FileIngestService /health: @@ -176,35 +143,5 @@ paths: summary: health tags: - FileIngestService - /legacy/ingest: - post: - deprecated: true - description: Decrypt payload, process metadata, file secret and send success - event - operationId: ingestLegacyFileUploadMetadata - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/EncryptedPayload' - required: true - responses: - '202': - content: - application/json: - schema: {} - description: Received and decrypted data successfully. - '422': - content: - application/json: - schema: - $ref: '#/components/schemas/HTTPValidationError' - description: Validation Error - security: - - HTTPBearer: [] - summary: Processes encrypted output data from the S3 upload script and ingests - it into the Encryption Key Store, Internal File Registry and Download Controller. - tags: - - FileIngestService tags: -- name: FileIngestService +- name: FileInformationService diff --git a/services/fins/pyproject.toml b/services/fins/pyproject.toml index 9a84df47..f836219f 100644 --- a/services/fins/pyproject.toml +++ b/services/fins/pyproject.toml @@ -22,7 +22,7 @@ classifiers = [ ] name = "fins" version = "1.0.0" -description = "File Information Service - TODO" +description = "File Information Service - Providing public metadata about files registered with the Internal File Registry" [project.license] text = "Apache 2.0" diff --git a/services/fins/src/fins/adapters/inbound/fastapi_/configure.py b/services/fins/src/fins/adapters/inbound/fastapi_/configure.py index 9c6c4952..6ed6b08e 100644 --- a/services/fins/src/fins/adapters/inbound/fastapi_/configure.py +++ b/services/fins/src/fins/adapters/inbound/fastapi_/configure.py @@ -28,14 +28,14 @@ config = Config() # type: ignore -def get_openapi_schema(api) -> dict[str, Any]: +def get_openapi_schema(app: FastAPI) -> dict[str, Any]: """Generates a custom openapi schema for the service""" return get_openapi( title="File Information Service", version=__version__, - description="TODO", + description="Providing public metadata about files registered with the Internal File Registry", tags=[{"name": "FileInformationService"}], - routes=api.routes, + routes=app.routes, ) diff --git a/services/fins/src/fins/adapters/inbound/fastapi_/routes.py b/services/fins/src/fins/adapters/inbound/fastapi_/routes.py index d925c351..1d7d605e 100644 --- a/services/fins/src/fins/adapters/inbound/fastapi_/routes.py +++ b/services/fins/src/fins/adapters/inbound/fastapi_/routes.py @@ -19,6 +19,7 @@ from fastapi import APIRouter, Depends, status from fins.adapters.inbound.fastapi_ import dummies, http_exceptions, http_responses +from fins.core import models from fins.ports.inbound.information_service import InformationServicePort router = APIRouter() @@ -29,7 +30,7 @@ "A configuration or external communication error has occurred and details " + "should not be communicated to the client" ), - "model": http_responses.HttpFileInformationResponse, + "model": models.FileInformation, }, "informationNotFound": { "description": (