Gitlab 0.1.6 (#64)

* upgraded ocean

integrations/gitlab/.port/resources/port-app-config.yaml

@@ -31,7 +31,7 @@ resources:
             link: .web_url
           relations:
             microservice: .references.full
-  - kind: issues
+  - kind: issue
     selector:
       query: 'true'
     port:

integrations/gitlab/.port/spec.yaml

@@ -1,4 +1,4 @@
-version: v0.1.5
+version: v0.1.6
 type: gitlab
 description: Gitlab integration for Port Ocean
 icon: GitLab

integrations/gitlab/changelog/1.improvement.md

@@ -0,0 +1 @@
+Upgraded ocean version to use the optimized `on_resync` generator

integrations/gitlab/gitlab_integration/ocean.py

@@ -1,19 +1,20 @@
-from typing import Any, Dict
+from datetime import datetime, timedelta
+from typing import Any
 
-from gitlab_integration.bootstrap import event_handler
-from gitlab_integration.bootstrap import setup_application
-from gitlab_integration.utils import get_all_services, ObjectKind
 from loguru import logger
 from starlette.requests import Request
 
+from gitlab_integration.bootstrap import event_handler
+from gitlab_integration.bootstrap import setup_application
+from gitlab_integration.utils import get_all_services, ObjectKind
 from port_ocean.context.ocean import ocean
 from port_ocean.core.ocean_types import RAW_RESULT, ASYNC_GENERATOR_RESYNC_TYPE
 
 all_tokens_services = get_all_services()
 
 
 @ocean.router.post("/hook/{group_id}")
-async def handle_webhook(group_id: str, request: Request) -> Dict[str, Any]:
+async def handle_webhook(group_id: str, request: Request) -> dict[str, Any]:
     event_id = f'{request.headers.get("X-Gitlab-Event")}:{group_id}'
     body = await request.json()
     await event_handler.notify(event_id, group_id, body)
@@ -42,16 +43,25 @@ async def on_resync(kind: str) -> RAW_RESULT:
 
 @ocean.on_resync(ObjectKind.MERGE_REQUEST)
 async def resync_merge_requests(kind: str) -> RAW_RESULT:
-    merge_requests = []
+    updated_after = datetime.now() - timedelta(days=14)
+
+    result = []
     for service in all_tokens_services:
         for group in service.get_root_groups():
-            for merge_request in group.mergerequests.list(all=True):
+            merge_requests = group.mergerequests.list(
+                all=True, state="opened"
+            ) + group.mergerequests.list(
+                all=True,
+                state=["closed", "locked", "merged"],
+                updated_after=updated_after.strftime("%Y-%m-%dT%H:%M:%S.%fZ"),
+            )
+            for merge_request in merge_requests:
                 project_path = merge_request.references.get("full").rstrip(
                     merge_request.references.get("short")
                 )
                 if service.should_run_for_project(project_path):
-                    merge_requests.append(merge_request.asdict())
-    return merge_requests
+                    result.append(merge_request.asdict())
+    return result
 
 
 @ocean.on_resync(ObjectKind.ISSUE)
@@ -74,18 +84,36 @@ async def resync_jobs(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE:
         for project in service.get_all_projects():
             jobs = project.jobs.list(per_page=100)
             logger.info(f"Found {len(jobs)} jobs for project {project.id}")
-            for job in jobs:
-                yield job.asdict()
+            yield [job.asdict() for job in jobs]
 
 
 @ocean.on_resync(ObjectKind.PIPELINE)
 async def resync_pipelines(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE:
+    from_time = datetime.now() - timedelta(days=14)
+    created_after = from_time.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+
     for service in all_tokens_services:
         for project in service.get_all_projects():
-            pipelines = project.pipelines.list(all=True)
-            logger.info(f"Found {len(pipelines)} pipelines for project {project.id}")
-            for pipeline in pipelines:
-                yield {
-                    **pipeline.asdict(),
-                    "__project": project.asdict(),
-                }
+            batch_size = 50
+            page = 1
+            more = True
+
+            while more:
+                # Process the batch of pipelines here
+                pipelines = project.pipelines.list(
+                    page=page, per_page=batch_size, created_after=created_after
+                )
+                logger.info(
+                    f"Found {len(pipelines)} pipelines for page number {page} in project {project.id}"
+                )
+                yield [
+                    {
+                        **pipeline.asdict(),
+                        "__project": project.asdict(),
+                    }
+                    for pipeline in pipelines
+                ]
+
+                # Fetch the next batch
+                page += 1
+                more = len(pipelines) == batch_size

integrations/gitlab/pyproject.toml

@@ -10,7 +10,7 @@ aiofiles = "^0.6.0"
 python-gitlab = "^3.14.0"
 pathlib = "^1.0.1"
 jsonschema = "^4.17.3"
-port-ocean = {version = "0.1.3", extras = ["cli"]}
+port-ocean = {version = "0.2.1", extras = ["cli"]}
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^7.2"