Allow populating config with env vars (#38)

Allow populating config with env vars (#38)

.github/workflows/release-integrations.yml

@@ -4,6 +4,11 @@ on:
     branches:
       - main
   workflow_dispatch:
+    inputs:
+      LATEST:
+        type: boolean
+        default: true
+        description: "If true, build the Docker image with both 'latest' and version tags. If false, build the Docker image with only the version tag."
 
 jobs:
   release-all:
@@ -41,7 +46,13 @@ jobs:
               echo "Image already exists in $repository: port-ocean-$type:$version"
             else
               echo "Building and pushing new image: port-ocean-$type:$version"
-              docker build -t "ghcr.io/port-labs/port-ocean-$type:$version" -t "ghcr.io/port-labs/port-ocean-$type:latest" "$folder/.."
+              if [[ "${{ inputs.LATEST }}" == "true" ]]; then
+                # If true, build the Docker image with both "latest" and version tags
+                docker build -t "ghcr.io/port-labs/port-ocean-$type:$version" -t "ghcr.io/port-labs/port-ocean-$type:latest" "$folder/.."
+              else
+                  # If false, build the Docker image with only the version tag
+                  docker build -t "ghcr.io/port-labs/port-ocean-$type:$version" "$folder/.."
+              fi
               docker push "ghcr.io/port-labs/port-ocean-$type" --all-tags
             fi
           done

changelog/allow-populating-config-with-env-vars.feature.md

@@ -0,0 +1 @@
+All the settings can nnow be set using environment variables with prefix of OCEAN__{The name of the field} and __ between nested fields

changelog/auto-camelize-aliases.improvment.md

@@ -0,0 +1 @@
+Using pyhumps to automatically camelize the aliases of the settings

changelog/decamelized-config.breaking.md

@@ -0,0 +1 @@
+All integration config is now decamelized and used as snake case inside the integration

changelog/default-broker.feature.md

@@ -0,0 +1 @@
+The broker field in the kafka settings has now the default of port production brokers

changelog/no-resources-crash.bugfix.md

@@ -0,0 +1 @@
+Fixed a crash when there are no resources in the port-app-config

changelog/settings-file-renaming.breaking.md

@@ -0,0 +1 @@
+Renamed port_ocean.config.integration -> port_ocean.config.settings

integrations/gitlab/changelog/update-config-ocean-0-1-2.bump.md

@@ -0,0 +1 @@
+changed the usage of the config according to ocean 0.1.2

integrations/gitlab/config.yaml

@@ -1,24 +1,16 @@
-# This is an example configuration file for the integration service.
-# Please copy this file to conf/config.yaml and edit it to your needs.
-
-# The integrations list to load into the integration port_ocean.
-# You can add as many integrations as you want, and you can use the same type of integration multiple times.
+initializePortResources: true
 port:
   clientId: {{ from env PORT_CLIENT_ID }} # Can be loaded via environment variable: PORT_CLIENT_ID, if both are set, the environment variable will be used.
   clientSecret: {{ from env PORT_CLIENT_SECRET }} # Can be loaded via environment variable: PORT_CLIENT_SECRET, if both are set, the environment variable will be used.
-  baseUrl: http://localhost:3000 # Can be loaded via environment variable: PORT_BASE_URL, if both are set, the environment variable will be used.
-# The trigger channel to use for the integration service.
+# The event listener to use for the integration service.
 eventListener:
-  type: KAFKA
-  brokers: "localhost:9092"
-  kafkaSecurityEnabled: false
+  type: POLLING
 integration:
   # The name of the integration.
-  identifier: "my_integration_2asdfasdfasdfasdfaasdsdfasdfasfd"
+  identifier: "my_gitlab_integration"
   # The type of the integration.
   type: "gitlab"
   # The configuration of the integration.
-  # resyncOnStartup: true
   config:
-    tokenMapping: {{ from env GITLAB_TOKEN_MAPPING }}
-    appHost: https://c0d5-81-199-132-66.ngrok-free.app
+    tokenMapping: {{ from env TOKEN_MAPPING }}
+    appHost: {{ from env APP_HOST }}

integrations/gitlab/gitlab_integration/bootstrap.py

@@ -27,10 +27,10 @@ def setup_listeners(gitlab_service: GitlabService, webhook_id: str | int) -> Non
 
 def setup_application() -> None:
     logic_settings = ocean.integration_config
-    for token, group_mapping in logic_settings["tokenMapping"].items():
-        gitlab_client = Gitlab(logic_settings["gitlabHost"], token)
+    for token, group_mapping in logic_settings["token_mapping"].items():
+        gitlab_client = Gitlab(logic_settings["gitlab_host"], token)
         gitlab_service = GitlabService(
-            gitlab_client, logic_settings["appHost"], group_mapping
+            gitlab_client, logic_settings["app_host"], group_mapping
         )
         webhook_ids = gitlab_service.create_webhooks()
         for webhook_id in webhook_ids:

integrations/gitlab/gitlab_integration/utils.py

@@ -12,12 +12,12 @@ def get_all_services() -> List[GitlabService]:
     all_tokens_services = []
 
     logger.info(
-        f"Creating gitlab clients for {len(logic_settings['tokenMapping'])} tokens"
+        f"Creating gitlab clients for {len(logic_settings['token_mapping'])} tokens"
     )
-    for token, group_mapping in logic_settings["tokenMapping"].items():
-        gitlab_client = Gitlab(logic_settings["gitlabHost"], token)
+    for token, group_mapping in logic_settings["token_mapping"].items():
+        gitlab_client = Gitlab(logic_settings["gitlab_host"], token)
         gitlab_service = GitlabService(
-            gitlab_client, logic_settings["appHost"], group_mapping
+            gitlab_client, logic_settings["app_host"], group_mapping
         )
         all_tokens_services.append(gitlab_service)
 

integrations/gitlab/pyproject.toml

@@ -10,7 +10,7 @@ aiofiles = "^0.6.0"
 python-gitlab = "^3.14.0"
 pathlib = "^1.0.1"
 jsonschema = "^4.17.3"
-port-ocean = {version = "0.1.1", extras = ["cli"]}
+port-ocean = {version = "0.1.2rc3", extras = ["cli"]}
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^7.2"

port_ocean/cli/commands/sail.py

@@ -3,7 +3,7 @@
 import click
 
 from port_ocean.cli.commands.main import cli_start, print_logo, console
-from port_ocean.config.integration import LogLevelType
+from port_ocean.config.settings import LogLevelType
 
 
 @cli_start.command()

port_ocean/config/base.py

@@ -4,7 +4,9 @@
 from typing import Any
 
 import yaml
+from humps import decamelize
 from pydantic import BaseSettings
+from pydantic.env_settings import EnvSettingsSource, InitSettingsSource
 
 PROVIDER_WRAPPER_PATTERN = r"\{\{ from (.*) \}\}"
 PROVIDER_CONFIG_PATTERN = r"^[a-zA-Z0-9]+ .*$"
@@ -47,6 +49,15 @@ def load_from_config_provider(provider_type: str, value: str) -> Any:
         raise ValueError(f"Invalid provider type: {provider_type}")
 
 
+def decamelize_object(obj: Any) -> Any:
+    if isinstance(obj, dict):
+        return {decamelize(k): decamelize_object(v) for k, v in obj.items()}
+    elif isinstance(obj, list):
+        return [decamelize_object(v) for v in obj]
+    else:
+        return obj
+
+
 def load_providers(settings: "BaseOceanSettings", base_path: str) -> dict[str, Any]:
     yaml_content = read_yaml_config_settings_source(settings, base_path)
     matches = re.finditer(PROVIDER_WRAPPER_PATTERN, yaml_content)
@@ -56,18 +67,23 @@ def load_providers(settings: "BaseOceanSettings", base_path: str) -> dict[str, A
         # Replace the provider wrapper with the actual value
         yaml_content = re.sub(re.escape(match.group()), data, yaml_content, count=1)
 
-    return yaml.safe_load(yaml_content)
+    return decamelize_object(yaml.safe_load(yaml_content))
 
 
 class BaseOceanSettings(BaseSettings):
     base_path: str
 
     class Config:
         yaml_file = "./config.yaml"
+        env_prefix = "OCEAN__"
+        env_nested_delimiter = "__"
+        env_file = ".env"
+        env_file_encoding = "utf-8"
 
         @classmethod
-        def customise_sources(cls, init_settings, *_, **__):  # type: ignore
+        def customise_sources(cls, init_settings: InitSettingsSource, env_settings: EnvSettingsSource, *_, **__):  # type: ignore
             return (
+                env_settings,
                 init_settings,
                 lambda s: load_providers(s, init_settings.init_kwargs["base_path"]),
             )

port_ocean/config/dynamic.py

@@ -1,5 +1,6 @@
 from typing import Type, Any, Optional
 
+from humps import decamelize
 from pydantic import BaseModel, AnyUrl, create_model, Extra, parse_obj_as
 
 
@@ -34,7 +35,7 @@ def default_config_factory(configurations: Any) -> Type[BaseModel]:
         default = ... if config.required else None
         if config.default is not None:
             default = parse_obj_as(field_type, config.default)
-        fields[config.name] = (
+        fields[decamelize(config.name)] = (
             field_type,
             default,
         )

port_ocean/config/settings.py

@@ -0,0 +1,42 @@
+from typing import Any, Literal
+
+from pydantic import BaseSettings, BaseModel, Extra, AnyHttpUrl, parse_obj_as
+
+from port_ocean.config.base import BaseOceanSettings
+from port_ocean.core.event_listener import EventListenerSettingsType
+
+LogLevelType = Literal["ERROR", "WARNING", "INFO", "DEBUG", "CRITICAL"]
+
+
+class ApplicationSettings(BaseSettings):
+    log_level: LogLevelType = "DEBUG"
+    port: int = 8000
+
+    class Config:
+        env_prefix = "APPLICATION__"
+        env_file = ".env"
+        env_file_encoding = "utf-8"
+
+        @classmethod
+        def customise_sources(cls, init_settings, env_settings, *_, **__):  # type: ignore
+            return env_settings, init_settings
+
+
+class PortSettings(BaseModel, extra=Extra.allow):
+    client_id: str
+    client_secret: str
+    base_url: AnyHttpUrl = parse_obj_as(AnyHttpUrl, "https://api.getport.io")
+
+
+class IntegrationSettings(BaseModel, extra=Extra.allow):
+    identifier: str
+    type: str
+    config: dict[str, Any]
+
+
+class IntegrationConfiguration(BaseOceanSettings, extra=Extra.allow):
+    port: PortSettings
+    event_listener: EventListenerSettingsType
+    batch_work_size: int = 20
+    initialize_port_resources: bool = False
+    integration: IntegrationSettings

port_ocean/context/ocean.py

@@ -15,7 +15,7 @@
 from port_ocean.exceptions.context import PortOceanContextNotFoundError
 
 if TYPE_CHECKING:
-    from port_ocean.config.integration import IntegrationConfiguration
+    from port_ocean.config.settings import IntegrationConfiguration
     from port_ocean.core.integrations.base import BaseIntegration
     from port_ocean.ocean import Ocean
     from port_ocean.clients.port.client import PortClient

port_ocean/core/event_listener/base.py

@@ -1,7 +1,7 @@
 from abc import abstractmethod
 from typing import TypedDict, Callable, Any, Awaitable
 
-from pydantic import BaseSettings
+from pydantic import BaseModel, Extra
 
 
 class EventListenerEvents(TypedDict):
@@ -20,7 +20,7 @@ async def start(self) -> None:
         pass
 
 
-class EventListenerSettings(BaseSettings):
+class EventListenerSettings(BaseModel, extra=Extra.allow):
     type: str
 
     def to_request(self) -> dict[str, Any]:

port_ocean/core/event_listener/http/event_listener.py

@@ -2,7 +2,7 @@
 
 from fastapi import APIRouter
 from loguru import logger
-from pydantic import AnyHttpUrl, Field
+from pydantic import AnyHttpUrl
 
 from port_ocean.context.ocean import ocean
 from port_ocean.core.event_listener.base import (
@@ -14,7 +14,7 @@
 
 class HttpEventListenerSettings(EventListenerSettings):
     type: Literal["WEBHOOK"]
-    app_host: AnyHttpUrl = Field(alias="appHost")
+    app_host: AnyHttpUrl
 
     def to_request(self) -> dict[str, Any]:
         return {