diff --git a/integrations/azure-devops/.port/resources/blueprints.json b/integrations/azure-devops/.port/resources/blueprints.json index e9afea2d7c..96c4fe9c26 100644 --- a/integrations/azure-devops/.port/resources/blueprints.json +++ b/integrations/azure-devops/.port/resources/blueprints.json @@ -127,7 +127,11 @@ "type": "string", "icon": "AzureDevops", "description": "The type of work item (e.g., Bug, Task, User Story)", - "enum": ["Issue", "Epic", "Task"], + "enum": [ + "Issue", + "Epic", + "Task" + ], "enumColors": { "Issue": "green", "Epic": "orange", @@ -190,6 +194,66 @@ }, "required": [] }, + "mirrorProperties": { + "board": { + "title": "Board", + "path": "column.board.$title" + } + }, + "calculationProperties": {}, + "aggregationProperties": {}, + "relations": { + "project": { + "title": "Project", + "target": "project", + "required": true, + "many": false + }, + "column": { + "title": "Column", + "description": "The column the entity belongs", + "target": "column", + "required": true, + "many": false + } + } + }, + { + "identifier": "column", + "title": "Column", + "icon": "AzureDevops", + "schema": { + "properties": {}, + "required": [] + }, + "mirrorProperties": {}, + "calculationProperties": {}, + "aggregationProperties": {}, + "relations": { + "board": { + "title": "board", + "target": "board", + "required": true, + "many": false + } + } + }, + { + "identifier": "board", + "title": "Board", + "icon": "AzureDevops", + "schema": { + "properties": { + "link": { + "title": "Link", + "type": "string", + "format": "url", + "icon": "AzureDevops", + "description": "Link to the board in Azure DevOps" + } + }, + "required": [] + }, "mirrorProperties": {}, "calculationProperties": {}, "aggregationProperties": {}, diff --git a/integrations/azure-devops/.port/resources/port-app-config.yaml b/integrations/azure-devops/.port/resources/port-app-config.yaml index 172ce63b5b..8ffc0192df 100644 --- a/integrations/azure-devops/.port/resources/port-app-config.yaml +++ b/integrations/azure-devops/.port/resources/port-app-config.yaml @@ -54,6 +54,19 @@ resources: blueprint: '"service"' properties: workItemLinking: .isEnabled and .isBlocking + - kind: board + selector: + query: 'true' + port: + entity: + mappings: + identifier: .id | gsub(" "; "") + title: .name + blueprint: '"board"' + properties: + link: .url + relations: + project: .__project.id | gsub(" "; "") - kind: work-item selector: query: 'true' @@ -76,3 +89,17 @@ resources: changedDate: .fields."System.ChangedDate" relations: project: .__projectId | gsub(" "; "") + column: >- + .fields."System.WorkItemType"+"-"+.fields."System.BoardColumn"+"-"+.__project.id + | gsub(" "; "") + - kind: column + selector: + query: 'true' + port: + entity: + mappings: + identifier: .__stateType+"-"+.name+"-"+.__board.__project.id | gsub(" "; "") + title: .name + blueprint: '"column"' + relations: + board: .__board.id | gsub(" "; "") diff --git a/integrations/azure-devops/CHANGELOG.md b/integrations/azure-devops/CHANGELOG.md index 1dbda52f6d..e6aa5fcf7b 100644 --- a/integrations/azure-devops/CHANGELOG.md +++ b/integrations/azure-devops/CHANGELOG.md @@ -7,11 +7,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 +## 0.1.74 (2024-10-10) + + +### Improvements + + +- Added support for ingesting boards and columns + + ## 0.1.73 (2024-10-09) ### Improvements + - Bumped ocean version to ^0.12.3 @@ -20,6 +30,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Improvements + - Bumped ocean version to ^0.12.2 @@ -28,6 +39,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Improvements + - Bumped ocean version to ^0.12.1 diff --git a/integrations/azure-devops/azure_devops/client/azure_devops_client.py b/integrations/azure-devops/azure_devops/client/azure_devops_client.py index 31a5b38fa6..b19e98da77 100644 --- a/integrations/azure-devops/azure_devops/client/azure_devops_client.py +++ b/integrations/azure-devops/azure_devops/client/azure_devops_client.py @@ -258,6 +258,52 @@ async def get_repository(self, repository_id: str) -> dict[Any, Any]: repository_data = response.json() return repository_data + async def get_columns(self) -> AsyncGenerator[list[dict[str, Any]], None]: + async for boards in self.get_boards_in_organization(): + for board in boards: + yield [ + { + **column, + "__board": board, + "__stateType": stateType, + "__stateName": stateName, + } + for column in board.get("columns", []) + if column.get("stateMappings") + for stateType, stateName in column.get("stateMappings").items() + ] + + async def _enrich_boards( + self, boards: list[dict[str, Any]], project_id: str + ) -> list[dict[str, Any]]: + for board in boards: + response = await self.send_request( + "GET", + f"{self._organization_base_url}/{project_id}/{API_URL_PREFIX}/work/boards/{board['id']}", + ) + board.update(response.json()) + return boards + + async def _get_boards(self, project_id: str) -> list[dict[str, Any]]: + get_boards_url = ( + f"{self._organization_base_url}/{project_id}/{API_URL_PREFIX}/work/boards" + ) + response = await self.send_request("GET", get_boards_url) + board_data = response.json().get("value", []) + logger.info(f"Found {len(board_data)} boards for project {project_id}") + return await self._enrich_boards(board_data, project_id) + + @cache_iterator_result() + async def get_boards_in_organization( + self, + ) -> AsyncGenerator[list[dict[str, Any]], None]: + async for projects in self.generate_projects(): + yield [ + {**board, "__project": project} + for project in projects + for board in await self._get_boards(project["id"]) + ] + async def generate_subscriptions_webhook_events(self) -> list[WebhookEvent]: headers = {"Content-Type": "application/json"} try: diff --git a/integrations/azure-devops/azure_devops/misc.py b/integrations/azure-devops/azure_devops/misc.py index a873231e70..f527b8d68b 100644 --- a/integrations/azure-devops/azure_devops/misc.py +++ b/integrations/azure-devops/azure_devops/misc.py @@ -19,6 +19,8 @@ class Kind(StrEnum): TEAM = "team" PROJECT = "project" WORK_ITEM = "work-item" + BOARD = "board" + COLUMN = "column" PULL_REQUEST_SEARCH_CRITERIA: list[dict[str, Any]] = [ diff --git a/integrations/azure-devops/examples/example-data.json b/integrations/azure-devops/examples/example-data.json new file mode 100644 index 0000000000..d62451d6fa --- /dev/null +++ b/integrations/azure-devops/examples/example-data.json @@ -0,0 +1,227 @@ +[ + { + "id": "71e60866-7aac-43f2-8aeb-8fed5c66c192", + "name": "To Do", + "itemLimit": 0, + "stateMappings": { + "Epic": "To Do" + }, + "columnType": "incoming", + "__board": { + "id": "15392e3f-8617-4df6-bd27-2871a44eb92b", + "url": "[REDACTED]/6f84ea1f-e939-477f-8992-4145cb37ed35/0a0ed690-62a9-4164-ba02-dc899a29c126/_apis/work/boards/15392e3f-8617-4df6-bd27-2871a44eb92b", + "name": "Epics", + "revision": 0, + "columns": [ + { + "id": "71e60866-7aac-43f2-8aeb-8fed5c66c192", + "name": "To Do", + "itemLimit": 0, + "stateMappings": { + "Epic": "To Do" + }, + "columnType": "incoming" + }, + { + "id": "8c3f0cdc-74be-4a09-abc7-a88d06132d74", + "name": "Doing", + "itemLimit": 5, + "stateMappings": { + "Epic": "Doing" + }, + "isSplit": false, + "description": "", + "columnType": "inProgress" + }, + { + "id": "f71fbad1-3412-4a58-9784-f8404a6e83cb", + "name": "Done", + "itemLimit": 0, + "stateMappings": { + "Epic": "Done" + }, + "columnType": "outgoing" + } + ], + "rows": [ + { + "id": "00000000-0000-0000-0000-000000000000", + "name": null, + "color": null + } + ], + "isValid": true, + "allowedMappings": { + "Incoming": { + "Epic": [ + "To Do" + ] + }, + "InProgress": { + "Epic": [ + "Doing", + "To Do" + ] + }, + "Outgoing": { + "Epic": [ + "Done" + ] + } + }, + "canEdit": true, + "fields": { + "columnField": { + "referenceName": "WEF_C46436FC05DA4646816534AB33DDC48E_Kanban.Column", + "url": "[REDACTED]/_apis/wit/fields/WEF_C46436FC05DA4646816534AB33DDC48E_Kanban.Column" + }, + "rowField": { + "referenceName": "WEF_C46436FC05DA4646816534AB33DDC48E_Kanban.Lane", + "url": "[REDACTED]/_apis/wit/fields/WEF_C46436FC05DA4646816534AB33DDC48E_Kanban.Lane" + }, + "doneField": { + "referenceName": "WEF_C46436FC05DA4646816534AB33DDC48E_Kanban.Column.Done", + "url": "[REDACTED]/_apis/wit/fields/WEF_C46436FC05DA4646816534AB33DDC48E_Kanban.Column.Done" + } + }, + "_links": { + "self": { + "href": "[REDACTED]/6f84ea1f-e939-477f-8992-4145cb37ed35/0a0ed690-62a9-4164-ba02-dc899a29c126/_apis/work/boards/15392e3f-8617-4df6-bd27-2871a44eb92b" + }, + "project": { + "href": "[REDACTED]/_apis/projects/6f84ea1f-e939-477f-8992-4145cb37ed35" + }, + "team": { + "href": "[REDACTED]/_apis/projects/6f84ea1f-e939-477f-8992-4145cb37ed35/teams/0a0ed690-62a9-4164-ba02-dc899a29c126" + }, + "charts": { + "href": "[REDACTED]/6f84ea1f-e939-477f-8992-4145cb37ed35/0a0ed690-62a9-4164-ba02-dc899a29c126/_apis/work/boards/15392e3f-8617-4df6-bd27-2871a44eb92b/charts" + }, + "columns": { + "href": "[REDACTED]/6f84ea1f-e939-477f-8992-4145cb37ed35/0a0ed690-62a9-4164-ba02-dc899a29c126/_apis/work/boards/15392e3f-8617-4df6-bd27-2871a44eb92b/columns" + }, + "rows": { + "href": "[REDACTED]/6f84ea1f-e939-477f-8992-4145cb37ed35/0a0ed690-62a9-4164-ba02-dc899a29c126/_apis/work/boards/15392e3f-8617-4df6-bd27-2871a44eb92b/rows" + } + }, + "__project": { + "id": "6f84ea1f-e939-477f-8992-4145cb37ed35", + "name": "terraform-azure-resource", + "url": "[REDACTED]/_apis/projects/6f84ea1f-e939-477f-8992-4145cb37ed35", + "state": "wellFormed", + "revision": 12, + "visibility": "private", + "lastUpdateTime": "2024-01-08T12:07:57.667Z" + } + }, + "__stateType": "Epic", + "__stateName": "To Do" + }, + { + "id": "11dd908f-62c0-411a-99fa-958fb4651764", + "url": "[REDACTED]/6f84ea1f-e939-477f-8992-4145cb37ed35/0a0ed690-62a9-4164-ba02-dc899a29c126/_apis/work/boards/11dd908f-62c0-411a-99fa-958fb4651764", + "name": "Issues", + "revision": 0, + "columns": [ + { + "id": "7a8153e6-e163-498e-83da-43ef70c8966e", + "name": "To Do", + "itemLimit": 0, + "stateMappings": { + "Issue": "To Do" + }, + "columnType": "incoming" + }, + { + "id": "f142791e-7313-4c05-9e44-d109cd4b8717", + "name": "Doing", + "itemLimit": 5, + "stateMappings": { + "Issue": "Doing" + }, + "isSplit": false, + "description": "", + "columnType": "inProgress" + }, + { + "id": "c6ff6473-1670-4567-a499-cca3d5c45bd1", + "name": "Done", + "itemLimit": 0, + "stateMappings": { + "Issue": "Done" + }, + "columnType": "outgoing" + } + ], + "rows": [ + { + "id": "00000000-0000-0000-0000-000000000000", + "name": null, + "color": null + } + ], + "isValid": true, + "allowedMappings": { + "Incoming": { + "Issue": [ + "To Do" + ] + }, + "InProgress": { + "Issue": [ + "Doing", + "To Do" + ] + }, + "Outgoing": { + "Issue": [ + "Done" + ] + } + }, + "canEdit": true, + "fields": { + "columnField": { + "referenceName": "WEF_D68C9FE95E58442D9298E9272FF0054E_Kanban.Column", + "url": "[REDACTED]/_apis/wit/fields/WEF_D68C9FE95E58442D9298E9272FF0054E_Kanban.Column" + }, + "rowField": { + "referenceName": "WEF_D68C9FE95E58442D9298E9272FF0054E_Kanban.Lane", + "url": "[REDACTED]/_apis/wit/fields/WEF_D68C9FE95E58442D9298E9272FF0054E_Kanban.Lane" + }, + "doneField": { + "referenceName": "WEF_D68C9FE95E58442D9298E9272FF0054E_Kanban.Column.Done", + "url": "[REDACTED]/_apis/wit/fields/WEF_D68C9FE95E58442D9298E9272FF0054E_Kanban.Column.Done" + } + }, + "_links": { + "self": { + "href": "[REDACTED]/6f84ea1f-e939-477f-8992-4145cb37ed35/0a0ed690-62a9-4164-ba02-dc899a29c126/_apis/work/boards/11dd908f-62c0-411a-99fa-958fb4651764" + }, + "project": { + "href": "[REDACTED]/_apis/projects/6f84ea1f-e939-477f-8992-4145cb37ed35" + }, + "team": { + "href": "[REDACTED]/_apis/projects/6f84ea1f-e939-477f-8992-4145cb37ed35/teams/0a0ed690-62a9-4164-ba02-dc899a29c126" + }, + "charts": { + "href": "[REDACTED]/6f84ea1f-e939-477f-8992-4145cb37ed35/0a0ed690-62a9-4164-ba02-dc899a29c126/_apis/work/boards/11dd908f-62c0-411a-99fa-958fb4651764/charts" + }, + "columns": { + "href": "[REDACTED]/6f84ea1f-e939-477f-8992-4145cb37ed35/0a0ed690-62a9-4164-ba02-dc899a29c126/_apis/work/boards/11dd908f-62c0-411a-99fa-958fb4651764/columns" + }, + "rows": { + "href": "[REDACTED]/6f84ea1f-e939-477f-8992-4145cb37ed35/0a0ed690-62a9-4164-ba02-dc899a29c126/_apis/work/boards/11dd908f-62c0-411a-99fa-958fb4651764/rows" + } + }, + "__project": { + "id": "6f84ea1f-e939-477f-8992-4145cb37ed35", + "name": "terraform-azure-resource", + "url": "[REDACTED]/_apis/projects/6f84ea1f-e939-477f-8992-4145cb37ed35", + "state": "wellFormed", + "revision": 12, + "visibility": "private", + "lastUpdateTime": "2024-01-08T12:07:57.667Z" + } + } +] diff --git a/integrations/azure-devops/examples/example-mappings.yaml b/integrations/azure-devops/examples/example-mappings.yaml new file mode 100644 index 0000000000..0294a7210f --- /dev/null +++ b/integrations/azure-devops/examples/example-mappings.yaml @@ -0,0 +1,27 @@ +deleteDependentEntities: true +createMissingRelatedEntities: true +resources: + - kind: board + selector: + query: 'true' + port: + entity: + mappings: + identifier: .id | gsub(" "; "") + title: .name + blueprint: '"board"' + properties: + link: .url + relations: + project: .__project.id | gsub(" "; "") + - kind: column + selector: + query: 'true' + port: + entity: + mappings: + identifier: .__stateType+"-"+.name+"-"+.__board.__project.id | gsub(" "; "") + title: .name + blueprint: '"column"' + relations: + board: .__board.id | gsub(" "; "") diff --git a/integrations/azure-devops/examples/example-output.json b/integrations/azure-devops/examples/example-output.json new file mode 100644 index 0000000000..cc93b28fa9 --- /dev/null +++ b/integrations/azure-devops/examples/example-output.json @@ -0,0 +1,21 @@ +[ + { + "identifier": "Epic-ToDo-6f84ea1f-e939-477f-8992-4145cb37ed35", + "title": "To Do", + "blueprint": "column", + "relations": { + "board": "15392e3f-8617-4df6-bd27-2871a44eb92b" + } + }, + { + "identifier": "11dd908f-62c0-411a-99fa-958fb4651764", + "title": "Issues", + "blueprint": "board", + "properties": { + "link": "[REDACTED]/6f84ea1f-e939-477f-8992-4145cb37ed35/0a0ed690-62a9-4164-ba02-dc899a29c126/_apis/work/boards/11dd908f-62c0-411a-99fa-958fb4651764" + }, + "relations": { + "project": "6f84ea1f-e939-477f-8992-4145cb37ed35" + } + } +] diff --git a/integrations/azure-devops/main.py b/integrations/azure-devops/main.py index 89ca64a858..f01bf0d5c3 100644 --- a/integrations/azure-devops/main.py +++ b/integrations/azure-devops/main.py @@ -111,6 +111,22 @@ async def resync_workitems(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE: yield work_items +@ocean.on_resync(Kind.COLUMN) +async def resync_columns(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE: + azure_devops_client = AzureDevopsClient.create_from_ocean_config() + async for columns in azure_devops_client.get_columns(): + logger.info(f"Resyncing {len(columns)} columns") + yield columns + + +@ocean.on_resync(Kind.BOARD) +async def resync_boards(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE: + azure_devops_client = AzureDevopsClient.create_from_ocean_config() + async for boards in azure_devops_client.get_boards_in_organization(): + logger.info(f"Resyncing {len(boards)} boards") + yield boards + + @ocean.router.post("/webhook") async def webhook(request: Request) -> dict[str, Any]: body = await request.json() diff --git a/integrations/azure-devops/pyproject.toml b/integrations/azure-devops/pyproject.toml index 05064499cc..4a53b0c1b8 100644 --- a/integrations/azure-devops/pyproject.toml +++ b/integrations/azure-devops/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "azure-devops" -version = "0.1.73" +version = "0.1.74" description = "An Azure Devops Ocean integration" authors = ["Matan Geva "] diff --git a/integrations/azure-devops/tests/azure_devops/__init__.py b/integrations/azure-devops/tests/azure_devops/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/integrations/azure-devops/tests/azure_devops/client/__init__.py b/integrations/azure-devops/tests/azure_devops/client/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/integrations/azure-devops/tests/azure_devops/client/test_azure_devops_client.py b/integrations/azure-devops/tests/azure_devops/client/test_azure_devops_client.py new file mode 100644 index 0000000000..beb747ca2e --- /dev/null +++ b/integrations/azure-devops/tests/azure_devops/client/test_azure_devops_client.py @@ -0,0 +1,676 @@ +import pytest +from unittest.mock import patch, MagicMock +from typing import Any, AsyncGenerator, Dict, Generator, List, Optional +from port_ocean.exceptions.context import PortOceanContextAlreadyInitializedError +from azure_devops.client.azure_devops_client import AzureDevopsClient +from port_ocean.context.ocean import initialize_port_ocean_context +from port_ocean.context.event import EventContext, event_context +from httpx import Response +from azure_devops.webhooks.webhook_event import WebhookEvent + +MOCK_ORG_URL = "https://your_organization_url.com" +MOCK_PERSONAL_ACCESS_TOKEN = "personal_access_token" +MOCK_PROJECT_ID = "12345" +MOCK_PROJECT_NAME = "My Project" +EXPECTED_PROJECT = {"name": MOCK_PROJECT_NAME, "id": MOCK_PROJECT_ID} + +MOCK_BOARD_ID = "board1" +MOCK_BOARD_NAME = "Board One" +EXPECTED_BOARDS = [ + { + "id": MOCK_BOARD_ID, + "name": MOCK_BOARD_NAME, + "columns": [ + {"name": "To Do", "stateMappings": {"Bug": "New"}}, + {"name": "Doing", "stateMappings": {"Bug": "Active"}}, + ], + } +] + +EXPECTED_COLUMNS = [ + { + "name": "To Do", + "stateMappings": {"Bug": "New"}, + "__board": EXPECTED_BOARDS[0], + "__stateType": "Bug", + "__stateName": "New", + }, + { + "name": "Doing", + "stateMappings": {"Bug": "Active"}, + "__board": EXPECTED_BOARDS[0], + "__stateType": "Bug", + "__stateName": "Active", + }, +] + +EXPECTED_BOARDS_IN_ORG = [ + {**board, "__project": {"id": "proj1", "name": "Project One"}} + for board in [ + {"id": "board1", "name": "Board One"}, + {"id": "board2", "name": "Board Two"}, + ] +] + +EXPECTED_PROJECTS = [ + {"id": "proj1", "name": "Project One"}, + {"id": "proj2", "name": "Project Two"}, +] + +EXPECTED_TEAMS = [ + {"id": "team1", "name": "Team One"}, + {"id": "team2", "name": "Team Two"}, +] + +EXPECTED_MEMBERS = [ + {"id": "member1", "name": "Member One", "__teamId": "team1"}, + {"id": "member2", "name": "Member Two", "__teamId": "team1"}, +] + +EXPECTED_REPOSITORIES = [ + {"id": "repo1", "name": "Repo One", "isDisabled": False}, + {"id": "repo2", "name": "Repo Two", "isDisabled": False}, +] + +EXPECTED_PULL_REQUESTS = [ + { + "pullRequestId": "pr1", + "title": "Pull Request One", + "repository": {"id": "repo1"}, + }, + { + "pullRequestId": "pr2", + "title": "Pull Request Two", + "repository": {"id": "repo1"}, + }, +] + +EXPECTED_PIPELINES = [ + {"id": "pipeline1", "name": "Pipeline One", "__projectId": "proj1"}, + {"id": "pipeline2", "name": "Pipeline Two", "__projectId": "proj1"}, +] + +EXPECTED_POLICIES = [ + {"id": "policy1", "name": "Policy One", "__repository": {"id": "repo1"}}, + {"id": "policy2", "name": "Policy Two", "__repository": {"id": "repo1"}}, +] + +EXPECTED_WORK_ITEMS = [ + { + "id": 1, + "fields": {}, + "__projectId": "proj1", + "__project": {"id": "proj1", "name": "Project One"}, + }, + { + "id": 2, + "fields": {}, + "__projectId": "proj1", + "__project": {"id": "proj1", "name": "Project One"}, + }, + { + "id": 3, + "fields": {}, + "__projectId": "proj1", + "__project": {"id": "proj1", "name": "Project One"}, + }, +] + +EXPECTED_PULL_REQUEST = {"id": "pr123", "title": "My Pull Request"} +EXPECTED_REPOSITORY = {"id": "repo123", "name": "My Repository"} + +EXPECTED_WEBHOOK_EVENTS = [ + { + "id": "sub1", + "publisherId": "tfs", + "eventType": "workitem.created", + "consumerId": "webHooks", + "consumerActionId": "httpRequest", + "consumerInputs": None, + "publisherInputs": None, + "status": None, + }, + { + "id": "sub2", + "publisherId": "tfs", + "eventType": "git.push", + "consumerId": "webHooks", + "consumerActionId": "httpRequest", + "consumerInputs": None, + "publisherInputs": None, + "status": None, + }, +] + +EXPECTED_SUBSCRIPTION_CREATION_RESPONSE = { + "id": "subscription123", + "eventType": "git.push", +} + +MOCK_FILE_CONTENT = b"file content" +MOCK_FILE_PATH = "/path/to/file.txt" +MOCK_REPOSITORY_ID = "repo123" +MOCK_BRANCH_NAME = "main" +MOCK_COMMIT_ID = "abc123" + + +async def async_generator(items: List[Any]) -> AsyncGenerator[Any, None]: + for item in items: + yield item + + +@pytest.fixture(autouse=True) +def mock_ocean_context() -> None: + try: + mock_ocean_app = MagicMock() + mock_ocean_app.config.integration.config = { + "organization_url": MOCK_ORG_URL, + "personal_access_token": MOCK_PERSONAL_ACCESS_TOKEN, + } + mock_ocean_app.integration_router = MagicMock() + mock_ocean_app.port_client = MagicMock() + initialize_port_ocean_context(mock_ocean_app) + except PortOceanContextAlreadyInitializedError: + pass + + +@pytest.fixture +def mock_event_context() -> Generator[MagicMock, None, None]: + mock_event = MagicMock(spec=EventContext) + mock_event.event_type = "test_event" + mock_event.trigger_type = "manual" + mock_event.attributes = {} + mock_event._deadline = 999999999.0 + mock_event._aborted = False + + with patch("port_ocean.context.event.event", mock_event): + yield mock_event + + +@pytest.fixture +def mock_azure_client() -> AzureDevopsClient: + return AzureDevopsClient(MOCK_ORG_URL, MOCK_PERSONAL_ACCESS_TOKEN) + + +@pytest.mark.asyncio +async def test_get_single_project() -> None: + client = AzureDevopsClient(MOCK_ORG_URL, MOCK_PERSONAL_ACCESS_TOKEN) + + # MOCK + with patch.object(client, "send_request") as mock_send_request: + mock_send_request.return_value = Response( + status_code=200, json=EXPECTED_PROJECT + ) + + # ACT + project_id = MOCK_PROJECT_ID + project = await client.get_single_project(project_id) + + # ASSERT + assert project == EXPECTED_PROJECT + mock_send_request.assert_called_once_with( + "GET", + f"{MOCK_ORG_URL}/_apis/projects/{project_id}", + ) + + +@pytest.mark.asyncio +async def test_generate_projects(mock_event_context: MagicMock) -> None: + client = AzureDevopsClient(MOCK_ORG_URL, MOCK_PERSONAL_ACCESS_TOKEN) + + # MOCK + async def mock_get_paginated_by_top_and_continuation_token( + *args: Any, **kwargs: Any + ) -> AsyncGenerator[List[Dict[str, Any]], None]: + yield [EXPECTED_PROJECTS[0]] + yield [EXPECTED_PROJECTS[1]] + + with patch.object( + client, + "_get_paginated_by_top_and_continuation_token", + side_effect=mock_get_paginated_by_top_and_continuation_token, + ): + async with event_context("test_event"): + # ACT + projects: List[Dict[str, Any]] = [] + async for project_batch in client.generate_projects(): + projects.extend(project_batch) + + # ASSERT + assert projects == EXPECTED_PROJECTS + + +@pytest.mark.asyncio +async def test_generate_teams(mock_event_context: MagicMock) -> None: + client = AzureDevopsClient(MOCK_ORG_URL, MOCK_PERSONAL_ACCESS_TOKEN) + + # MOCK + async def mock_get_paginated_by_top_and_skip( + *args: Any, **kwargs: Any + ) -> AsyncGenerator[List[Dict[str, Any]], None]: + yield [EXPECTED_TEAMS[0]] + yield [EXPECTED_TEAMS[1]] + + with patch.object( + client, + "_get_paginated_by_top_and_skip", + side_effect=mock_get_paginated_by_top_and_skip, + ): + async with event_context("test_event"): + # ACT + teams: List[Dict[str, Any]] = [] + async for team_batch in client.generate_teams(): + teams.extend(team_batch) + + # ASSERT + assert teams == EXPECTED_TEAMS + + +@pytest.mark.asyncio +async def test_generate_members() -> None: + client = AzureDevopsClient(MOCK_ORG_URL, MOCK_PERSONAL_ACCESS_TOKEN) + + # MOCK + async def mock_generate_teams() -> AsyncGenerator[List[Dict[str, Any]], None]: + yield [{"id": "team1", "name": "Team One", "projectId": "proj1"}] + + async def mock_get_paginated_by_top_and_skip( + url: str, **kwargs: Any + ) -> AsyncGenerator[List[Dict[str, Any]], None]: + if "members" in url: + yield [ + {"id": "member1", "name": "Member One"}, + {"id": "member2", "name": "Member Two"}, + ] + else: + yield [] + + with patch.object(client, "generate_teams", side_effect=mock_generate_teams): + with patch.object( + client, + "_get_paginated_by_top_and_skip", + side_effect=mock_get_paginated_by_top_and_skip, + ): + # ACT + members: List[Dict[str, Any]] = [] + async for member_batch in client.generate_members(): + for member in member_batch: + member["__teamId"] = "team1" + members.extend(member_batch) + + # ASSERT + assert members == EXPECTED_MEMBERS + + +@pytest.mark.asyncio +async def test_generate_repositories(mock_event_context: MagicMock) -> None: + client = AzureDevopsClient(MOCK_ORG_URL, MOCK_PERSONAL_ACCESS_TOKEN) + + # MOCK + async def mock_generate_projects() -> AsyncGenerator[List[Dict[str, Any]], None]: + yield [{"id": "proj1", "name": "Project One"}] + + with patch.object(client, "generate_projects", side_effect=mock_generate_projects): + with patch.object(client, "send_request") as mock_send_request: + mock_send_request.return_value = Response( + status_code=200, + json={"value": EXPECTED_REPOSITORIES}, + ) + + async with event_context("test_event"): + # ACT + repositories: List[Dict[str, Any]] = [] + async for repo_batch in client.generate_repositories( + include_disabled_repositories=False + ): + repositories.extend(repo_batch) + + # ASSERT + assert repositories == EXPECTED_REPOSITORIES + + +@pytest.mark.asyncio +async def test_generate_pull_requests(mock_event_context: MagicMock) -> None: + client = AzureDevopsClient(MOCK_ORG_URL, MOCK_PERSONAL_ACCESS_TOKEN) + + # MOCK + async def mock_generate_repositories( + *args: Any, **kwargs: Any + ) -> AsyncGenerator[List[Dict[str, Any]], None]: + yield [ + { + "id": "repo1", + "name": "Repository One", + "project": {"id": "proj1", "name": "Project One"}, + } + ] + + async def mock_get_paginated_by_top_and_skip( + url: str, additional_params: Optional[Dict[str, Any]] = None, **kwargs: Any + ) -> AsyncGenerator[List[Dict[str, Any]], None]: + if "pullrequests" in url: + yield EXPECTED_PULL_REQUESTS + else: + yield [] + + async with event_context("test_event"): + + with patch.object( + client, "generate_repositories", side_effect=mock_generate_repositories + ): + with patch.object( + client, + "_get_paginated_by_top_and_skip", + side_effect=mock_get_paginated_by_top_and_skip, + ): + # ACT + pull_requests: List[Dict[str, Any]] = [] + async for pr_batch in client.generate_pull_requests(): + pull_requests.extend(pr_batch) + + # ASSERT + assert pull_requests == EXPECTED_PULL_REQUESTS + + +@pytest.mark.asyncio +async def test_generate_pipelines() -> None: + client = AzureDevopsClient(MOCK_ORG_URL, MOCK_PERSONAL_ACCESS_TOKEN) + + # MOCK + async def mock_generate_projects() -> AsyncGenerator[List[Dict[str, Any]], None]: + yield [{"id": "proj1", "name": "Project One"}] + + async def mock_get_paginated_by_top_and_continuation_token( + url: str, **kwargs: Any + ) -> AsyncGenerator[List[Dict[str, Any]], None]: + yield [ + {"id": "pipeline1", "name": "Pipeline One"}, + {"id": "pipeline2", "name": "Pipeline Two"}, + ] + + with patch.object(client, "generate_projects", side_effect=mock_generate_projects): + with patch.object( + client, + "_get_paginated_by_top_and_continuation_token", + side_effect=mock_get_paginated_by_top_and_continuation_token, + ): + # ACT + pipelines: List[Dict[str, Any]] = [] + async for pipeline_batch in client.generate_pipelines(): + for pipeline in pipeline_batch: + pipeline["__projectId"] = "proj1" + pipelines.extend(pipeline_batch) + + # ASSERT + assert pipelines == EXPECTED_PIPELINES + + +@pytest.mark.asyncio +async def test_generate_repository_policies() -> None: + client = AzureDevopsClient(MOCK_ORG_URL, MOCK_PERSONAL_ACCESS_TOKEN) + + # MOCK + async def mock_generate_repositories( + *args: Any, **kwargs: Any + ) -> AsyncGenerator[List[Dict[str, Any]], None]: + yield [ + { + "id": "repo1", + "name": "Repository One", + "project": {"id": "proj1", "name": "Project One"}, + "defaultBranch": "refs/heads/main", + } + ] + + with patch.object( + client, "generate_repositories", side_effect=mock_generate_repositories + ): + with patch.object(client, "send_request") as mock_send_request: + mock_send_request.return_value = Response( + status_code=200, + json={ + "value": [ + {"id": "policy1", "name": "Policy One"}, + {"id": "policy2", "name": "Policy Two"}, + ] + }, + ) + + # ACT + policies: List[Dict[str, Any]] = [] + async for policy_batch in client.generate_repository_policies(): + for policy in policy_batch: + policy["__repository"] = {"id": "repo1"} + policies.extend(policy_batch) + + # ASSERT + assert policies == EXPECTED_POLICIES + + +@pytest.mark.asyncio +async def test_get_pull_request() -> None: + client = AzureDevopsClient(MOCK_ORG_URL, MOCK_PERSONAL_ACCESS_TOKEN) + + # MOCK + with patch.object(client, "send_request") as mock_send_request: + mock_send_request.return_value = Response( + status_code=200, json=EXPECTED_PULL_REQUEST + ) + + # ACT + pull_request_id = "pr123" + pull_request = await client.get_pull_request(pull_request_id) + + # ASSERT + assert pull_request == EXPECTED_PULL_REQUEST + mock_send_request.assert_called_once_with( + "GET", + f"{MOCK_ORG_URL}/_apis/git/pullrequests/{pull_request_id}", + ) + + +@pytest.mark.asyncio +async def test_get_repository() -> None: + client = AzureDevopsClient(MOCK_ORG_URL, MOCK_PERSONAL_ACCESS_TOKEN) + + # MOCK + with patch.object(client, "send_request") as mock_send_request: + mock_send_request.return_value = Response( + status_code=200, json=EXPECTED_REPOSITORY + ) + + # ACT + repository_id = "repo123" + repository = await client.get_repository(repository_id) + + # ASSERT + assert repository == EXPECTED_REPOSITORY + mock_send_request.assert_called_once_with( + "GET", + f"{MOCK_ORG_URL}/_apis/git/repositories/{repository_id}", + ) + + +@pytest.mark.asyncio +async def test_get_columns() -> None: + client = AzureDevopsClient(MOCK_ORG_URL, MOCK_PERSONAL_ACCESS_TOKEN) + + # MOCK + async def mock_get_boards_in_organization() -> ( + AsyncGenerator[List[Dict[str, Any]], None] + ): + yield EXPECTED_BOARDS + + with patch.object( + client, + "get_boards_in_organization", + side_effect=mock_get_boards_in_organization, + ): + # ACT + columns: List[Dict[str, Any]] = [] + async for column_batch in client.get_columns(): + columns.extend(column_batch) + + # ASSERT + assert columns == EXPECTED_COLUMNS + + +@pytest.mark.asyncio +async def test_get_boards_in_organization(mock_event_context: MagicMock) -> None: + client = AzureDevopsClient(MOCK_ORG_URL, MOCK_PERSONAL_ACCESS_TOKEN) + + # MOCK + async def mock_generate_projects() -> AsyncGenerator[List[Dict[str, Any]], None]: + yield [{"id": "proj1", "name": "Project One"}] + + async def mock_get_boards(project_id: str) -> List[Dict[str, Any]]: + return [ + {"id": "board1", "name": "Board One"}, + {"id": "board2", "name": "Board Two"}, + ] + + async with event_context("test_event"): + with patch.object( + client, "generate_projects", side_effect=mock_generate_projects + ): + with patch.object(client, "_get_boards", side_effect=mock_get_boards): + # ACT + boards: List[Dict[str, Any]] = [] + async for board_batch in client.get_boards_in_organization(): + boards.extend(board_batch) + + # ASSERT + assert boards == EXPECTED_BOARDS_IN_ORG + + +@pytest.mark.asyncio +async def test_generate_subscriptions_webhook_events() -> None: + client = AzureDevopsClient(MOCK_ORG_URL, MOCK_PERSONAL_ACCESS_TOKEN) + + # MOCK + with patch.object(client, "send_request") as mock_send_request: + mock_send_request.return_value = Response( + status_code=200, + json={"value": EXPECTED_WEBHOOK_EVENTS}, + ) + + # ACT + events = await client.generate_subscriptions_webhook_events() + + # ASSERT + assert [event.dict() for event in events] == EXPECTED_WEBHOOK_EVENTS + + +@pytest.mark.asyncio +async def test_create_subscription() -> None: + client = AzureDevopsClient(MOCK_ORG_URL, MOCK_PERSONAL_ACCESS_TOKEN) + webhook_event = WebhookEvent( + id=None, + eventType="git.push", + publisherId="tfs", + consumerId="webHooks", + consumerActionId="httpRequest", + status="enabled", + consumerInputs={"url": "https://example.com/webhook"}, + ) + + # MOCK + with patch.object(client, "send_request") as mock_send_request: + mock_send_request.return_value = Response( + status_code=200, json=EXPECTED_SUBSCRIPTION_CREATION_RESPONSE + ) + + # ACT + await client.create_subscription(webhook_event) + + # ASSERT + mock_send_request.assert_called_once_with( + "POST", + f"{MOCK_ORG_URL}/_apis/hooks/subscriptions", + params={"api-version": "7.1-preview.1"}, + headers={"Content-Type": "application/json"}, + data=webhook_event.json(), + ) + + +@pytest.mark.asyncio +async def test_delete_subscription() -> None: + client = AzureDevopsClient(MOCK_ORG_URL, MOCK_PERSONAL_ACCESS_TOKEN) + webhook_event = WebhookEvent( + id="subscription123", + publisherId="tfs", + eventType="git.push", + consumerId="webHooks", + consumerActionId="httpRequest", + status="enabled", + consumerInputs={"url": "https://example.com/webhook"}, + ) + + # MOCK + with patch.object(client, "send_request") as mock_send_request: + mock_send_request.return_value = Response(status_code=204) + + # ACT + await client.delete_subscription(webhook_event) + + # ASSERT + mock_send_request.assert_called_once_with( + "DELETE", + f"{MOCK_ORG_URL}/_apis/hooks/subscriptions/{webhook_event.id}", + headers={"Content-Type": "application/json"}, + params={"api-version": "7.1-preview.1"}, + ) + + +@pytest.mark.asyncio +async def test_get_file_by_branch() -> None: + client = AzureDevopsClient(MOCK_ORG_URL, MOCK_PERSONAL_ACCESS_TOKEN) + + # MOCK + with patch.object(client, "send_request") as mock_send_request: + mock_response = Response(status_code=200, content=MOCK_FILE_CONTENT) + mock_send_request.return_value = mock_response + + # ACT + file_content = await client.get_file_by_branch( + MOCK_FILE_PATH, MOCK_REPOSITORY_ID, MOCK_BRANCH_NAME + ) + + # ASSERT + assert file_content == MOCK_FILE_CONTENT + mock_send_request.assert_called_once_with( + method="GET", + url=f"{MOCK_ORG_URL}/_apis/git/repositories/{MOCK_REPOSITORY_ID}/items", + params={ + "versionType": "Branch", + "version": MOCK_BRANCH_NAME, + "path": MOCK_FILE_PATH, + }, + ) + + +@pytest.mark.asyncio +async def test_get_file_by_commit() -> None: + client = AzureDevopsClient(MOCK_ORG_URL, MOCK_PERSONAL_ACCESS_TOKEN) + + # MOCK + with patch.object(client, "send_request") as mock_send_request: + mock_response = Response(status_code=200, content=MOCK_FILE_CONTENT) + mock_send_request.return_value = mock_response + + # ACT + file_content = await client.get_file_by_commit( + MOCK_FILE_PATH, MOCK_REPOSITORY_ID, MOCK_COMMIT_ID + ) + + # ASSERT + assert file_content == MOCK_FILE_CONTENT + mock_send_request.assert_called_once_with( + method="GET", + url=f"{MOCK_ORG_URL}/_apis/git/repositories/{MOCK_REPOSITORY_ID}/items", + params={ + "versionType": "Commit", + "version": MOCK_COMMIT_ID, + "path": MOCK_FILE_PATH, + }, + ) diff --git a/integrations/azure-devops/tests/conftest.py b/integrations/azure-devops/tests/conftest.py new file mode 100644 index 0000000000..c0daae0da6 --- /dev/null +++ b/integrations/azure-devops/tests/conftest.py @@ -0,0 +1,5 @@ +# ruff: noqa +from port_ocean.tests.helpers.fixtures import ( + get_mocked_ocean_app, + get_mock_ocean_resource_configs, +)