From b00efd88f55ae3d513afa9346272d9983aaf263c Mon Sep 17 00:00:00 2001 From: Yair Siman Tov <63305203+yairsimantov20@users.noreply.github.com> Date: Thu, 27 Jul 2023 10:04:27 +0300 Subject: [PATCH] Allow populating config with env vars (#38) Allow populating config with env vars (#38) --- .github/workflows/release-integrations.yml | 13 +++++- ...populating-config-with-env-vars.feature.md | 1 + changelog/auto-camelize-aliases.improvment.md | 1 + changelog/decamelized-config.breaking.md | 1 + changelog/default-broker.feature.md | 1 + changelog/no-resources-crash.bugfix.md | 1 + changelog/settings-file-renaming.breaking.md | 1 + .../update-config-ocean-0-1-2.bump.md | 1 + integrations/gitlab/config.yaml | 20 +++------ .../gitlab/gitlab_integration/bootstrap.py | 6 +-- .../gitlab/gitlab_integration/utils.py | 8 ++-- integrations/gitlab/poetry.lock | 20 +++++++-- integrations/gitlab/pyproject.toml | 2 +- poetry.lock | 13 +++++- port_ocean/cli/commands/sail.py | 2 +- port_ocean/config/base.py | 20 ++++++++- port_ocean/config/dynamic.py | 3 +- port_ocean/config/integration.py | 39 ----------------- port_ocean/config/settings.py | 42 +++++++++++++++++++ port_ocean/context/ocean.py | 2 +- port_ocean/core/event_listener/base.py | 4 +- .../event_listener/http/event_listener.py | 4 +- .../event_listener/kafka/event_listener.py | 13 +++--- port_ocean/core/integrations/mixins/sync.py | 2 +- port_ocean/logger_setup.py | 2 +- port_ocean/ocean.py | 2 +- port_ocean/port_defaults.py | 2 +- port_ocean/run.py | 2 +- pyproject.toml | 3 +- 29 files changed, 141 insertions(+), 90 deletions(-) create mode 100644 changelog/allow-populating-config-with-env-vars.feature.md create mode 100644 changelog/auto-camelize-aliases.improvment.md create mode 100644 changelog/decamelized-config.breaking.md create mode 100644 changelog/default-broker.feature.md create mode 100644 changelog/no-resources-crash.bugfix.md create mode 100644 changelog/settings-file-renaming.breaking.md create mode 100644 integrations/gitlab/changelog/update-config-ocean-0-1-2.bump.md delete mode 100644 port_ocean/config/integration.py create mode 100644 port_ocean/config/settings.py diff --git a/.github/workflows/release-integrations.yml b/.github/workflows/release-integrations.yml index 84494ca6f4..629755e998 100644 --- a/.github/workflows/release-integrations.yml +++ b/.github/workflows/release-integrations.yml @@ -4,6 +4,11 @@ on: branches: - main workflow_dispatch: + inputs: + LATEST: + type: boolean + default: true + description: "If true, build the Docker image with both 'latest' and version tags. If false, build the Docker image with only the version tag." jobs: release-all: @@ -41,7 +46,13 @@ jobs: echo "Image already exists in $repository: port-ocean-$type:$version" else echo "Building and pushing new image: port-ocean-$type:$version" - docker build -t "ghcr.io/port-labs/port-ocean-$type:$version" -t "ghcr.io/port-labs/port-ocean-$type:latest" "$folder/.." + if [[ "${{ inputs.LATEST }}" == "true" ]]; then + # If true, build the Docker image with both "latest" and version tags + docker build -t "ghcr.io/port-labs/port-ocean-$type:$version" -t "ghcr.io/port-labs/port-ocean-$type:latest" "$folder/.." + else + # If false, build the Docker image with only the version tag + docker build -t "ghcr.io/port-labs/port-ocean-$type:$version" "$folder/.." + fi docker push "ghcr.io/port-labs/port-ocean-$type" --all-tags fi done diff --git a/changelog/allow-populating-config-with-env-vars.feature.md b/changelog/allow-populating-config-with-env-vars.feature.md new file mode 100644 index 0000000000..49ea687d68 --- /dev/null +++ b/changelog/allow-populating-config-with-env-vars.feature.md @@ -0,0 +1 @@ +All the settings can nnow be set using environment variables with prefix of OCEAN__{The name of the field} and __ between nested fields \ No newline at end of file diff --git a/changelog/auto-camelize-aliases.improvment.md b/changelog/auto-camelize-aliases.improvment.md new file mode 100644 index 0000000000..aab5716d67 --- /dev/null +++ b/changelog/auto-camelize-aliases.improvment.md @@ -0,0 +1 @@ +Using pyhumps to automatically camelize the aliases of the settings \ No newline at end of file diff --git a/changelog/decamelized-config.breaking.md b/changelog/decamelized-config.breaking.md new file mode 100644 index 0000000000..369e8f1c7f --- /dev/null +++ b/changelog/decamelized-config.breaking.md @@ -0,0 +1 @@ +All integration config is now decamelized and used as snake case inside the integration \ No newline at end of file diff --git a/changelog/default-broker.feature.md b/changelog/default-broker.feature.md new file mode 100644 index 0000000000..3c330430f6 --- /dev/null +++ b/changelog/default-broker.feature.md @@ -0,0 +1 @@ +The broker field in the kafka settings has now the default of port production brokers \ No newline at end of file diff --git a/changelog/no-resources-crash.bugfix.md b/changelog/no-resources-crash.bugfix.md new file mode 100644 index 0000000000..a872649ed6 --- /dev/null +++ b/changelog/no-resources-crash.bugfix.md @@ -0,0 +1 @@ +Fixed a crash when there are no resources in the port-app-config \ No newline at end of file diff --git a/changelog/settings-file-renaming.breaking.md b/changelog/settings-file-renaming.breaking.md new file mode 100644 index 0000000000..275a2db31a --- /dev/null +++ b/changelog/settings-file-renaming.breaking.md @@ -0,0 +1 @@ +Renamed port_ocean.config.integration -> port_ocean.config.settings \ No newline at end of file diff --git a/integrations/gitlab/changelog/update-config-ocean-0-1-2.bump.md b/integrations/gitlab/changelog/update-config-ocean-0-1-2.bump.md new file mode 100644 index 0000000000..7d314e4235 --- /dev/null +++ b/integrations/gitlab/changelog/update-config-ocean-0-1-2.bump.md @@ -0,0 +1 @@ +changed the usage of the config according to ocean 0.1.2 \ No newline at end of file diff --git a/integrations/gitlab/config.yaml b/integrations/gitlab/config.yaml index 96084220be..df49457737 100644 --- a/integrations/gitlab/config.yaml +++ b/integrations/gitlab/config.yaml @@ -1,24 +1,16 @@ -# This is an example configuration file for the integration service. -# Please copy this file to conf/config.yaml and edit it to your needs. - -# The integrations list to load into the integration port_ocean. -# You can add as many integrations as you want, and you can use the same type of integration multiple times. +initializePortResources: true port: clientId: {{ from env PORT_CLIENT_ID }} # Can be loaded via environment variable: PORT_CLIENT_ID, if both are set, the environment variable will be used. clientSecret: {{ from env PORT_CLIENT_SECRET }} # Can be loaded via environment variable: PORT_CLIENT_SECRET, if both are set, the environment variable will be used. - baseUrl: http://localhost:3000 # Can be loaded via environment variable: PORT_BASE_URL, if both are set, the environment variable will be used. -# The trigger channel to use for the integration service. +# The event listener to use for the integration service. eventListener: - type: KAFKA - brokers: "localhost:9092" - kafkaSecurityEnabled: false + type: POLLING integration: # The name of the integration. - identifier: "my_integration_2asdfasdfasdfasdfaasdsdfasdfasfd" + identifier: "my_gitlab_integration" # The type of the integration. type: "gitlab" # The configuration of the integration. - # resyncOnStartup: true config: - tokenMapping: {{ from env GITLAB_TOKEN_MAPPING }} - appHost: https://c0d5-81-199-132-66.ngrok-free.app \ No newline at end of file + tokenMapping: {{ from env TOKEN_MAPPING }} + appHost: {{ from env APP_HOST }} \ No newline at end of file diff --git a/integrations/gitlab/gitlab_integration/bootstrap.py b/integrations/gitlab/gitlab_integration/bootstrap.py index 5b30b368d6..dc8813a2bc 100644 --- a/integrations/gitlab/gitlab_integration/bootstrap.py +++ b/integrations/gitlab/gitlab_integration/bootstrap.py @@ -27,10 +27,10 @@ def setup_listeners(gitlab_service: GitlabService, webhook_id: str | int) -> Non def setup_application() -> None: logic_settings = ocean.integration_config - for token, group_mapping in logic_settings["tokenMapping"].items(): - gitlab_client = Gitlab(logic_settings["gitlabHost"], token) + for token, group_mapping in logic_settings["token_mapping"].items(): + gitlab_client = Gitlab(logic_settings["gitlab_host"], token) gitlab_service = GitlabService( - gitlab_client, logic_settings["appHost"], group_mapping + gitlab_client, logic_settings["app_host"], group_mapping ) webhook_ids = gitlab_service.create_webhooks() for webhook_id in webhook_ids: diff --git a/integrations/gitlab/gitlab_integration/utils.py b/integrations/gitlab/gitlab_integration/utils.py index 534452597c..825c81b118 100644 --- a/integrations/gitlab/gitlab_integration/utils.py +++ b/integrations/gitlab/gitlab_integration/utils.py @@ -12,12 +12,12 @@ def get_all_services() -> List[GitlabService]: all_tokens_services = [] logger.info( - f"Creating gitlab clients for {len(logic_settings['tokenMapping'])} tokens" + f"Creating gitlab clients for {len(logic_settings['token_mapping'])} tokens" ) - for token, group_mapping in logic_settings["tokenMapping"].items(): - gitlab_client = Gitlab(logic_settings["gitlabHost"], token) + for token, group_mapping in logic_settings["token_mapping"].items(): + gitlab_client = Gitlab(logic_settings["gitlab_host"], token) gitlab_service = GitlabService( - gitlab_client, logic_settings["appHost"], group_mapping + gitlab_client, logic_settings["app_host"], group_mapping ) all_tokens_services.append(gitlab_service) diff --git a/integrations/gitlab/poetry.lock b/integrations/gitlab/poetry.lock index 3646c5e246..872450506d 100644 --- a/integrations/gitlab/poetry.lock +++ b/integrations/gitlab/poetry.lock @@ -742,13 +742,13 @@ testing = ["pytest", "pytest-benchmark"] [[package]] name = "port-ocean" -version = "0.1.1" +version = "0.1.2rc3" description = "Port Ocean is a CLI tool for managing your Port projects." optional = false python-versions = ">=3.11,<4.0" files = [ - {file = "port_ocean-0.1.1-py3-none-any.whl", hash = "sha256:be798394721fde820b44e01fcb94fcaded376c751d5f0676ac746c9a38b69f19"}, - {file = "port_ocean-0.1.1.tar.gz", hash = "sha256:bf24e7b21f49aa27c7f8bd98905d89297bd545cf704b556140116175ef964e4b"}, + {file = "port_ocean-0.1.2rc3-py3-none-any.whl", hash = "sha256:e82937512697f612cfd1a77e2b51857c432b64d0c7373b81bb5d3e18294e03b6"}, + {file = "port_ocean-0.1.2rc3.tar.gz", hash = "sha256:bf80e878b47ab7f3b6365685b006988172975269a7866ba2424e90e128dfb127"}, ] [package.dependencies] @@ -760,6 +760,7 @@ httpx = ">=0.24.1,<0.25.0" jinja2-time = {version = ">=0.2.0,<0.3.0", optional = true, markers = "extra == \"cli\""} loguru = ">=0.7.0,<0.8.0" pydantic = ">=1.10.8,<2.0.0" +pyhumps = ">=3.8.0,<4.0.0" pyjq = ">=2.6.0,<3.0.0" pyyaml = ">=6.0,<7.0" rich = {version = ">=13.4.1,<14.0.0", optional = true, markers = "extra == \"cli\""} @@ -837,6 +838,17 @@ files = [ [package.extras] plugins = ["importlib-metadata"] +[[package]] +name = "pyhumps" +version = "3.8.0" +description = "🐫 Convert strings (and dictionary keys) between snake case, camel case and pascal case in Python. Inspired by Humps for Node" +optional = false +python-versions = "*" +files = [ + {file = "pyhumps-3.8.0-py3-none-any.whl", hash = "sha256:060e1954d9069f428232a1adda165db0b9d8dfdce1d265d36df7fbff540acfd6"}, + {file = "pyhumps-3.8.0.tar.gz", hash = "sha256:498026258f7ee1a8e447c2e28526c0bea9407f9a59c03260aee4bd6c04d681a3"}, +] + [[package]] name = "pyjq" version = "2.6.0" @@ -1327,4 +1339,4 @@ dev = ["black (>=19.3b0)", "pytest (>=4.6.2)"] [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "3feffa23d4cd37eeeb0ba766a33d31745103e828ebd592e83271209b4de5a3cb" +content-hash = "c9c687a85515a46912ae358df0b731530d431c8326167dda6c4286f1efa4faa1" diff --git a/integrations/gitlab/pyproject.toml b/integrations/gitlab/pyproject.toml index f9be5813d6..16f806020e 100644 --- a/integrations/gitlab/pyproject.toml +++ b/integrations/gitlab/pyproject.toml @@ -10,7 +10,7 @@ aiofiles = "^0.6.0" python-gitlab = "^3.14.0" pathlib = "^1.0.1" jsonschema = "^4.17.3" -port-ocean = {version = "0.1.1", extras = ["cli"]} +port-ocean = {version = "0.1.2rc3", extras = ["cli"]} [tool.poetry.group.dev.dependencies] pytest = "^7.2" diff --git a/poetry.lock b/poetry.lock index 007d248acc..50ea98cf83 100644 --- a/poetry.lock +++ b/poetry.lock @@ -833,6 +833,17 @@ files = [ [package.extras] plugins = ["importlib-metadata"] +[[package]] +name = "pyhumps" +version = "3.8.0" +description = "🐫 Convert strings (and dictionary keys) between snake case, camel case and pascal case in Python. Inspired by Humps for Node" +optional = false +python-versions = "*" +files = [ + {file = "pyhumps-3.8.0-py3-none-any.whl", hash = "sha256:060e1954d9069f428232a1adda165db0b9d8dfdce1d265d36df7fbff540acfd6"}, + {file = "pyhumps-3.8.0.tar.gz", hash = "sha256:498026258f7ee1a8e447c2e28526c0bea9407f9a59c03260aee4bd6c04d681a3"}, +] + [[package]] name = "pyjq" version = "2.6.0" @@ -1302,4 +1313,4 @@ cli = ["click", "cookiecutter", "jinja2-time", "rich"] [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "b1743854ca2b983d02fc56142336acd4c9a2e363336a937ccb9b30cdcbd9e4f3" +content-hash = "99a57ea0b3bc4843006bd5c6b2842f4ecfcb521ce87f46c1f69fd6e72468096d" diff --git a/port_ocean/cli/commands/sail.py b/port_ocean/cli/commands/sail.py index 2635128410..87352edba3 100644 --- a/port_ocean/cli/commands/sail.py +++ b/port_ocean/cli/commands/sail.py @@ -3,7 +3,7 @@ import click from port_ocean.cli.commands.main import cli_start, print_logo, console -from port_ocean.config.integration import LogLevelType +from port_ocean.config.settings import LogLevelType @cli_start.command() diff --git a/port_ocean/config/base.py b/port_ocean/config/base.py index afe1fbc478..0fc7b24ef3 100644 --- a/port_ocean/config/base.py +++ b/port_ocean/config/base.py @@ -4,7 +4,9 @@ from typing import Any import yaml +from humps import decamelize from pydantic import BaseSettings +from pydantic.env_settings import EnvSettingsSource, InitSettingsSource PROVIDER_WRAPPER_PATTERN = r"\{\{ from (.*) \}\}" PROVIDER_CONFIG_PATTERN = r"^[a-zA-Z0-9]+ .*$" @@ -47,6 +49,15 @@ def load_from_config_provider(provider_type: str, value: str) -> Any: raise ValueError(f"Invalid provider type: {provider_type}") +def decamelize_object(obj: Any) -> Any: + if isinstance(obj, dict): + return {decamelize(k): decamelize_object(v) for k, v in obj.items()} + elif isinstance(obj, list): + return [decamelize_object(v) for v in obj] + else: + return obj + + def load_providers(settings: "BaseOceanSettings", base_path: str) -> dict[str, Any]: yaml_content = read_yaml_config_settings_source(settings, base_path) matches = re.finditer(PROVIDER_WRAPPER_PATTERN, yaml_content) @@ -56,7 +67,7 @@ def load_providers(settings: "BaseOceanSettings", base_path: str) -> dict[str, A # Replace the provider wrapper with the actual value yaml_content = re.sub(re.escape(match.group()), data, yaml_content, count=1) - return yaml.safe_load(yaml_content) + return decamelize_object(yaml.safe_load(yaml_content)) class BaseOceanSettings(BaseSettings): @@ -64,10 +75,15 @@ class BaseOceanSettings(BaseSettings): class Config: yaml_file = "./config.yaml" + env_prefix = "OCEAN__" + env_nested_delimiter = "__" + env_file = ".env" + env_file_encoding = "utf-8" @classmethod - def customise_sources(cls, init_settings, *_, **__): # type: ignore + def customise_sources(cls, init_settings: InitSettingsSource, env_settings: EnvSettingsSource, *_, **__): # type: ignore return ( + env_settings, init_settings, lambda s: load_providers(s, init_settings.init_kwargs["base_path"]), ) diff --git a/port_ocean/config/dynamic.py b/port_ocean/config/dynamic.py index db2ab1001a..137da24654 100644 --- a/port_ocean/config/dynamic.py +++ b/port_ocean/config/dynamic.py @@ -1,5 +1,6 @@ from typing import Type, Any, Optional +from humps import decamelize from pydantic import BaseModel, AnyUrl, create_model, Extra, parse_obj_as @@ -34,7 +35,7 @@ def default_config_factory(configurations: Any) -> Type[BaseModel]: default = ... if config.required else None if config.default is not None: default = parse_obj_as(field_type, config.default) - fields[config.name] = ( + fields[decamelize(config.name)] = ( field_type, default, ) diff --git a/port_ocean/config/integration.py b/port_ocean/config/integration.py deleted file mode 100644 index a34f434d6d..0000000000 --- a/port_ocean/config/integration.py +++ /dev/null @@ -1,39 +0,0 @@ -from typing import Any, Literal - -from pydantic import Field, BaseSettings - -from port_ocean.config.base import BaseOceanSettings -from port_ocean.core.event_listener import EventListenerSettingsType - - -class PortSettings(BaseSettings): - client_id: str = Field(alias="clientId") - client_secret: str = Field(alias="clientSecret") - base_url: str = Field(alias="baseUrl", default="https://api.getport.io") - - -class IntegrationSettings(BaseSettings): - identifier: str - type: str - config: dict[str, Any] - - -class IntegrationConfiguration(BaseOceanSettings): - port: PortSettings - event_listener: EventListenerSettingsType = Field(alias="eventListener") - batch_work_size: int = Field(alias="batchWorkSize", default=20) - initialize_port_resources: bool = Field( - alias="initializePortResources", default=False - ) - integration: IntegrationSettings - - -LogLevelType = Literal["ERROR", "WARNING", "INFO", "DEBUG", "CRITICAL"] - - -class ApplicationSettings(BaseSettings): - log_level: LogLevelType = "DEBUG" - port: int = 8000 - - class Config: - env_prefix = "APPLICATION_" diff --git a/port_ocean/config/settings.py b/port_ocean/config/settings.py new file mode 100644 index 0000000000..e3d8137382 --- /dev/null +++ b/port_ocean/config/settings.py @@ -0,0 +1,42 @@ +from typing import Any, Literal + +from pydantic import BaseSettings, BaseModel, Extra, AnyHttpUrl, parse_obj_as + +from port_ocean.config.base import BaseOceanSettings +from port_ocean.core.event_listener import EventListenerSettingsType + +LogLevelType = Literal["ERROR", "WARNING", "INFO", "DEBUG", "CRITICAL"] + + +class ApplicationSettings(BaseSettings): + log_level: LogLevelType = "DEBUG" + port: int = 8000 + + class Config: + env_prefix = "APPLICATION__" + env_file = ".env" + env_file_encoding = "utf-8" + + @classmethod + def customise_sources(cls, init_settings, env_settings, *_, **__): # type: ignore + return env_settings, init_settings + + +class PortSettings(BaseModel, extra=Extra.allow): + client_id: str + client_secret: str + base_url: AnyHttpUrl = parse_obj_as(AnyHttpUrl, "https://api.getport.io") + + +class IntegrationSettings(BaseModel, extra=Extra.allow): + identifier: str + type: str + config: dict[str, Any] + + +class IntegrationConfiguration(BaseOceanSettings, extra=Extra.allow): + port: PortSettings + event_listener: EventListenerSettingsType + batch_work_size: int = 20 + initialize_port_resources: bool = False + integration: IntegrationSettings diff --git a/port_ocean/context/ocean.py b/port_ocean/context/ocean.py index e8e73fc8f1..602aef8101 100644 --- a/port_ocean/context/ocean.py +++ b/port_ocean/context/ocean.py @@ -15,7 +15,7 @@ from port_ocean.exceptions.context import PortOceanContextNotFoundError if TYPE_CHECKING: - from port_ocean.config.integration import IntegrationConfiguration + from port_ocean.config.settings import IntegrationConfiguration from port_ocean.core.integrations.base import BaseIntegration from port_ocean.ocean import Ocean from port_ocean.clients.port.client import PortClient diff --git a/port_ocean/core/event_listener/base.py b/port_ocean/core/event_listener/base.py index c877ffd015..46148cf141 100644 --- a/port_ocean/core/event_listener/base.py +++ b/port_ocean/core/event_listener/base.py @@ -1,7 +1,7 @@ from abc import abstractmethod from typing import TypedDict, Callable, Any, Awaitable -from pydantic import BaseSettings +from pydantic import BaseModel, Extra class EventListenerEvents(TypedDict): @@ -20,7 +20,7 @@ async def start(self) -> None: pass -class EventListenerSettings(BaseSettings): +class EventListenerSettings(BaseModel, extra=Extra.allow): type: str def to_request(self) -> dict[str, Any]: diff --git a/port_ocean/core/event_listener/http/event_listener.py b/port_ocean/core/event_listener/http/event_listener.py index f1d8ddb583..4b04a4835a 100644 --- a/port_ocean/core/event_listener/http/event_listener.py +++ b/port_ocean/core/event_listener/http/event_listener.py @@ -2,7 +2,7 @@ from fastapi import APIRouter from loguru import logger -from pydantic import AnyHttpUrl, Field +from pydantic import AnyHttpUrl from port_ocean.context.ocean import ocean from port_ocean.core.event_listener.base import ( @@ -14,7 +14,7 @@ class HttpEventListenerSettings(EventListenerSettings): type: Literal["WEBHOOK"] - app_host: AnyHttpUrl = Field(alias="appHost") + app_host: AnyHttpUrl def to_request(self) -> dict[str, Any]: return { diff --git a/port_ocean/core/event_listener/kafka/event_listener.py b/port_ocean/core/event_listener/kafka/event_listener.py index 1180428857..f1baa65fbd 100644 --- a/port_ocean/core/event_listener/kafka/event_listener.py +++ b/port_ocean/core/event_listener/kafka/event_listener.py @@ -2,7 +2,6 @@ from typing import Any, Callable, Literal from loguru import logger -from pydantic import Field from port_ocean.consumers.kafka_consumer import KafkaConsumer, KafkaConsumerConfig from port_ocean.context.ocean import ( @@ -19,13 +18,11 @@ class KafkaEventListenerSettings(EventListenerSettings): type: Literal["KAFKA"] - brokers: str = "" - security_protocol: str = Field(alias="securityProtocol", default="SASL_SSL") - authentication_mechanism: str = Field( - alias="authenticationMechanism", default="SCRAM-SHA-512" - ) - kafka_security_enabled: bool = Field(alias="kafkaSecurityEnabled", default=True) - consumer_poll_timeout: int = Field(alias="consumerPollTimeout", default=1) + brokers: str = "b-1-public.publicclusterprod.t9rw6w.c1.kafka.eu-west-1.amazonaws.com:9196,b-2-public.publicclusterprod.t9rw6w.c1.kafka.eu-west-1.amazonaws.com:9196,b-3-public.publicclusterprod.t9rw6w.c1.kafka.eu-west-1.amazonaws.com:9196" + security_protocol: str = "SASL_SSL" + authentication_mechanism: str = "SCRAM-SHA-512" + kafka_security_enabled: bool = True + consumer_poll_timeout: int = 1 class KafkaEventListener(BaseEventListener): diff --git a/port_ocean/core/integrations/mixins/sync.py b/port_ocean/core/integrations/mixins/sync.py index e12fe5ffd0..c5394f6231 100644 --- a/port_ocean/core/integrations/mixins/sync.py +++ b/port_ocean/core/integrations/mixins/sync.py @@ -317,7 +317,7 @@ async def sync_raw_all( resource, user_agent_type, ocean.config.batch_work_size ) ) - flat_created_entities, errors = zip_and_sum(creation_results) + flat_created_entities, errors = zip_and_sum(creation_results) or [[], []] if errors: message = f"Resync failed with {len(errors)}. Skipping delete phase due to incomplete state" diff --git a/port_ocean/logger_setup.py b/port_ocean/logger_setup.py index d1e6cde87b..2e47cf96c4 100644 --- a/port_ocean/logger_setup.py +++ b/port_ocean/logger_setup.py @@ -2,7 +2,7 @@ from loguru import logger -from port_ocean.config.integration import LogLevelType +from port_ocean.config.settings import LogLevelType def setup_logger(level: LogLevelType) -> None: diff --git a/port_ocean/ocean.py b/port_ocean/ocean.py index ecdf8a68ef..20f6c1b622 100644 --- a/port_ocean/ocean.py +++ b/port_ocean/ocean.py @@ -7,7 +7,7 @@ from starlette.types import Scope, Receive, Send from port_ocean.clients.port.client import PortClient -from port_ocean.config.integration import ( +from port_ocean.config.settings import ( IntegrationConfiguration, ) from port_ocean.context.ocean import ( diff --git a/port_ocean/port_defaults.py b/port_ocean/port_defaults.py index 25644a5e9e..f427d1c853 100644 --- a/port_ocean/port_defaults.py +++ b/port_ocean/port_defaults.py @@ -10,7 +10,7 @@ from starlette import status from port_ocean.clients.port.client import PortClient -from port_ocean.config.integration import IntegrationConfiguration +from port_ocean.config.settings import IntegrationConfiguration from port_ocean.context.ocean import ocean from port_ocean.core.handlers.port_app_config.models import PortAppConfig from port_ocean.exceptions.port_defaults import ( diff --git a/port_ocean/run.py b/port_ocean/run.py index 80b0830e48..fd1f43e15b 100644 --- a/port_ocean/run.py +++ b/port_ocean/run.py @@ -7,7 +7,7 @@ import uvicorn from port_ocean.config.dynamic import default_config_factory -from port_ocean.config.integration import LogLevelType, ApplicationSettings +from port_ocean.config.settings import LogLevelType, ApplicationSettings from port_ocean.core.integrations.base import BaseIntegration from port_ocean.logger_setup import setup_logger from port_ocean.ocean import Ocean diff --git a/pyproject.toml b/pyproject.toml index 3079bbb254..396088ea66 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "port-ocean" -version = "0.1.1" +version = "0.1.2" description = "Port Ocean is a CLI tool for managing your Port projects." readme = "README.md" homepage = "https://app.getport.io" @@ -47,6 +47,7 @@ httpx = "^0.24.1" pyjq = "^2.6.0" urllib3 = "^1.26.16" six = "^1.16.0" +pyhumps = "^3.8.0" # CLI click = { version = "^8.1.3", optional = true }