diff --git a/integrations/gitlab/.port/resources/port-app-config.yaml b/integrations/gitlab/.port/resources/port-app-config.yaml index 1f2cb40d03..02d768571c 100644 --- a/integrations/gitlab/.port/resources/port-app-config.yaml +++ b/integrations/gitlab/.port/resources/port-app-config.yaml @@ -31,7 +31,7 @@ resources: link: .web_url relations: microservice: .references.full - - kind: issues + - kind: issue selector: query: 'true' port: diff --git a/integrations/gitlab/.port/spec.yaml b/integrations/gitlab/.port/spec.yaml index 35a179a431..9e1e7442b4 100644 --- a/integrations/gitlab/.port/spec.yaml +++ b/integrations/gitlab/.port/spec.yaml @@ -1,4 +1,4 @@ -version: v0.1.5 +version: v0.1.6 type: gitlab description: Gitlab integration for Port Ocean icon: GitLab diff --git a/integrations/gitlab/changelog/1.improvement.md b/integrations/gitlab/changelog/1.improvement.md new file mode 100644 index 0000000000..09dac9c0c3 --- /dev/null +++ b/integrations/gitlab/changelog/1.improvement.md @@ -0,0 +1 @@ +Upgraded ocean version to use the optimized `on_resync` generator \ No newline at end of file diff --git a/integrations/gitlab/gitlab_integration/ocean.py b/integrations/gitlab/gitlab_integration/ocean.py index 76b47bed11..1cc082af29 100644 --- a/integrations/gitlab/gitlab_integration/ocean.py +++ b/integrations/gitlab/gitlab_integration/ocean.py @@ -1,11 +1,12 @@ -from typing import Any, Dict +from datetime import datetime, timedelta +from typing import Any -from gitlab_integration.bootstrap import event_handler -from gitlab_integration.bootstrap import setup_application -from gitlab_integration.utils import get_all_services, ObjectKind from loguru import logger from starlette.requests import Request +from gitlab_integration.bootstrap import event_handler +from gitlab_integration.bootstrap import setup_application +from gitlab_integration.utils import get_all_services, ObjectKind from port_ocean.context.ocean import ocean from port_ocean.core.ocean_types import RAW_RESULT, ASYNC_GENERATOR_RESYNC_TYPE @@ -13,7 +14,7 @@ @ocean.router.post("/hook/{group_id}") -async def handle_webhook(group_id: str, request: Request) -> Dict[str, Any]: +async def handle_webhook(group_id: str, request: Request) -> dict[str, Any]: event_id = f'{request.headers.get("X-Gitlab-Event")}:{group_id}' body = await request.json() await event_handler.notify(event_id, group_id, body) @@ -42,16 +43,25 @@ async def on_resync(kind: str) -> RAW_RESULT: @ocean.on_resync(ObjectKind.MERGE_REQUEST) async def resync_merge_requests(kind: str) -> RAW_RESULT: - merge_requests = [] + updated_after = datetime.now() - timedelta(days=14) + + result = [] for service in all_tokens_services: for group in service.get_root_groups(): - for merge_request in group.mergerequests.list(all=True): + merge_requests = group.mergerequests.list( + all=True, state="opened" + ) + group.mergerequests.list( + all=True, + state=["closed", "locked", "merged"], + updated_after=updated_after.strftime("%Y-%m-%dT%H:%M:%S.%fZ"), + ) + for merge_request in merge_requests: project_path = merge_request.references.get("full").rstrip( merge_request.references.get("short") ) if service.should_run_for_project(project_path): - merge_requests.append(merge_request.asdict()) - return merge_requests + result.append(merge_request.asdict()) + return result @ocean.on_resync(ObjectKind.ISSUE) @@ -74,18 +84,36 @@ async def resync_jobs(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE: for project in service.get_all_projects(): jobs = project.jobs.list(per_page=100) logger.info(f"Found {len(jobs)} jobs for project {project.id}") - for job in jobs: - yield job.asdict() + yield [job.asdict() for job in jobs] @ocean.on_resync(ObjectKind.PIPELINE) async def resync_pipelines(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE: + from_time = datetime.now() - timedelta(days=14) + created_after = from_time.strftime("%Y-%m-%dT%H:%M:%S.%fZ") + for service in all_tokens_services: for project in service.get_all_projects(): - pipelines = project.pipelines.list(all=True) - logger.info(f"Found {len(pipelines)} pipelines for project {project.id}") - for pipeline in pipelines: - yield { - **pipeline.asdict(), - "__project": project.asdict(), - } + batch_size = 50 + page = 1 + more = True + + while more: + # Process the batch of pipelines here + pipelines = project.pipelines.list( + page=page, per_page=batch_size, created_after=created_after + ) + logger.info( + f"Found {len(pipelines)} pipelines for page number {page} in project {project.id}" + ) + yield [ + { + **pipeline.asdict(), + "__project": project.asdict(), + } + for pipeline in pipelines + ] + + # Fetch the next batch + page += 1 + more = len(pipelines) == batch_size diff --git a/integrations/gitlab/poetry.lock b/integrations/gitlab/poetry.lock index 2555966e5a..a3c91e60c6 100644 --- a/integrations/gitlab/poetry.lock +++ b/integrations/gitlab/poetry.lock @@ -742,13 +742,13 @@ testing = ["pytest", "pytest-benchmark"] [[package]] name = "port-ocean" -version = "0.1.3" +version = "0.2.1" description = "Port Ocean is a CLI tool for managing your Port projects." optional = false python-versions = ">=3.11,<4.0" files = [ - {file = "port_ocean-0.1.3-py3-none-any.whl", hash = "sha256:c4a02ab9b3e867c53b615aaeff02068ac06a3e8b449cffc962a4a38bba694a8f"}, - {file = "port_ocean-0.1.3.tar.gz", hash = "sha256:3ae6bd92a0da31f678b6f6157ce8a9364daebf24a0d7e683adffa338adba7330"}, + {file = "port_ocean-0.2.1-py3-none-any.whl", hash = "sha256:a97ca020eb44c050283f45de2dd3b85891fdc3f4c3536c4cd4bfef4ec4dffeae"}, + {file = "port_ocean-0.2.1.tar.gz", hash = "sha256:6bd532fc6642b366bf88ac429f55c225ffde365ad49e9901b8306869e2e55ff0"}, ] [package.dependencies] @@ -1339,4 +1339,4 @@ dev = ["black (>=19.3b0)", "pytest (>=4.6.2)"] [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "b71d68e42830ee363fbca90ade37c03ed0b6274c5270ad30af3dbc657094c86e" +content-hash = "10663af276bda64364ac2cd764a38ba17c3269f529a1831f2dbaa8739f22bf7b" diff --git a/integrations/gitlab/pyproject.toml b/integrations/gitlab/pyproject.toml index 0383a86b9a..110f0479a0 100644 --- a/integrations/gitlab/pyproject.toml +++ b/integrations/gitlab/pyproject.toml @@ -10,7 +10,7 @@ aiofiles = "^0.6.0" python-gitlab = "^3.14.0" pathlib = "^1.0.1" jsonschema = "^4.17.3" -port-ocean = {version = "0.1.3", extras = ["cli"]} +port-ocean = {version = "0.2.1", extras = ["cli"]} [tool.poetry.group.dev.dependencies] pytest = "^7.2"