Merge pull request #7 from christopherfrige/feat/managing-new-manwhas…

…-to-scrape

Feat: improve manwha chapters management + global exception handlers

.docker/postgres/initdb.sql

@@ -23,6 +23,8 @@ CREATE TABLE IF NOT EXISTS chapter.chapter (
     manwha_id INT NOT NULL,
     chapter_number FLOAT NOT NULL,
     pages INT NOT NULL,
+    origin_url TEXT,
+    downloaded BOOLEAN NOT NULL DEFAULT false,
     created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
     updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
     FOREIGN KEY (manwha_id)
@@ -69,6 +71,7 @@ CREATE TABLE IF NOT EXISTS scraper.manwha (
     reader_id INT NOT NULL,
     manwha_id INT,
     url TEXT NOT NULL,
+    chapter_start INT NOT NULL DEFAULT 0,
     created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
     updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
     FOREIGN KEY (reader_id)

backend/src/domain/entities/chapter.py

@@ -1,4 +1,4 @@
-from sqlalchemy import Column, Integer, ForeignKey, TIMESTAMP, Float
+from sqlalchemy import Boolean, Column, Integer, ForeignKey, TIMESTAMP, Float, Text
 from src.domain.entities import Base
 from sqlalchemy.sql.functions import now
 
@@ -14,5 +14,7 @@ class Chapter(Base, ChapterSchema):
     manwha_id = Column(Integer, ForeignKey("manwha.manwha.id"), nullable=False)
     chapter_number = Column(Float, nullable=False)
     pages = Column(Integer, nullable=False)
+    origin_url = Column(Text, nullable=True)
+    downloaded = Column(Boolean, nullable=False, default=False)
     created_at = Column(TIMESTAMP, nullable=False, default=now())
     updated_at = Column(TIMESTAMP, default=now())

backend/src/domain/entities/scraper.py

@@ -33,5 +33,6 @@ class ScraperManwha(Base, ScraperSchema):
         nullable=True,
     )
     url = Column(Text, nullable=False)
+    chapter_start = Column(Integer, nullable=False, default=0)
     created_at = Column(TIMESTAMP, nullable=False, default=now())
     updated_at = Column(TIMESTAMP, default=now())

backend/src/domain/exceptions/__init__.py

@@ -0,0 +1,3 @@
+class DefaultException(Exception):
+    def __init__(self, message):
+        self.message = message

backend/src/domain/exceptions/client.py

@@ -0,0 +1,9 @@
+from src.domain.exceptions import DefaultException
+
+
+class BadRequestException(DefaultException):
+    ...
+
+
+class ConflictException(DefaultException):
+    ...

backend/src/domain/repository/__init__.py

@@ -18,7 +18,7 @@ def get(self, field: str, value: str | int | float | bool) -> list:
     def get_all(self) -> list:
         return self.session.query(self.model).all()
 
-    def add(self, obj: object) -> None:
+    def add(self, obj: object) -> int | None:
         self.session.add(obj)
         self.session.flush()
         if hasattr(obj, "id"):

backend/src/domain/repository/scraper.py

@@ -1,8 +1,13 @@
 from src.domain.repository import BaseRepository
-from src.domain.entities.scraper import ScraperManwha
+from src.domain.entities.scraper import ScraperManwha, Reader
 from sqlalchemy.orm import Session
 
 
 class ScraperManwhaRepository(BaseRepository):
     def __init__(self, session: Session) -> None:
         super().__init__(session, ScraperManwha)
+
+
+class ReaderRepository(BaseRepository):
+    def __init__(self, session: Session) -> None:
+        super().__init__(session, Reader)

backend/src/domain/schemas/manwha.py

@@ -41,3 +41,8 @@ class GetManwhaResponse(BaseModel):
 class SearchManwhasResponse(BaseModel):
     records: list[ManwhaSchema]
     pagination: Pagination
+
+
+class CreateManwhaToScrapeRequest(BaseModel):
+    reader_id: int
+    url: str

backend/src/domain/schemas/scraper.py

@@ -0,0 +1,13 @@
+from pydantic import BaseModel
+
+from src.domain.schemas import Pagination
+
+
+class ReaderData(BaseModel):
+    id: int
+    name: str
+
+
+class GetReadersResponse(BaseModel):
+    records: list[ReaderData]
+    pagination: Pagination

backend/src/domain/use_cases/chapter/check_new_chapters.py

@@ -1,3 +1,4 @@
+from src.domain.repository.scraper import ScraperManwhaRepository
 from src.domain.entities.chapter import Chapter
 from sqlalchemy.orm import Session
 from sqlalchemy import select
@@ -6,19 +7,30 @@
 class CheckNewChaptersUseCase:
     def __init__(self, session: Session):
         self.session = session
+        self.scraper_manwha_repository = ScraperManwhaRepository(session)
 
     def execute(self, manwha_id: int, chapters_incoming: list):
         chapters_registered = self._get_chapters_registered(manwha_id)
 
+        manwha = self.scraper_manwha_repository.get("manwha_id", manwha_id)
+
+        chapter_start = 0
+        if manwha:
+            chapter_start = manwha[0].chapter_start
+
         chapters_difference = len(chapters_incoming) - len(chapters_registered)
         if chapters_difference == 0:
             return []
 
         new_chapters = []
         for chapter in chapters_incoming:
+            if chapter["number"] < chapter_start:
+                continue
             if chapter["number"] not in chapters_registered:
                 new_chapters.append(chapter)
 
+        new_chapters.sort(key=self._chapter_sort_criteria)
+
         return new_chapters
 
     def _get_chapters_registered(self, manwha_id: int):
@@ -28,3 +40,6 @@ def _get_chapters_registered(self, manwha_id: int):
             .order_by(Chapter.chapter_number.desc())
         )
         return self.session.execute(query).scalars().all()
+
+    def _chapter_sort_criteria(self, chapter):
+        return chapter["number"]

backend/src/domain/use_cases/chapter/get_chapter_pages.py

@@ -20,7 +20,7 @@ def execute(self, chapter_id: int) -> GetChapterPagesResponse:
                     Chapter.pages.label("chapter_pages"),
                 )
                 .join(Chapter, Chapter.manwha_id == Manwha.id)
-                .filter(Chapter.id == chapter_id)
+                .filter(Chapter.id == chapter_id, Chapter.downloaded)
             ).first()
 
             if not result:

backend/src/domain/use_cases/chapter/manage_chapters.py

@@ -13,9 +13,15 @@ def __init__(self, session: Session, storage: S3Service):
         self.storage = storage
         self.chapter_repository = ChapterRepository(session)
 
-    def execute(self, manwha_id: int, chapter_number: float, pages: int):
+    def execute(self, manwha_id: int, chapter_number: float, pages: int, origin_url: str):
         chapter_id = self.chapter_repository.add(
-            Chapter(manwha_id=manwha_id, chapter_number=chapter_number, pages=pages)
+            Chapter(
+                manwha_id=manwha_id,
+                chapter_number=chapter_number,
+                pages=pages,
+                downloaded=True,
+                origin_url=origin_url,
+            )
         )
         self._upload_chapter_images(manwha_id, chapter_id)
 

backend/src/domain/use_cases/manwha/create_manwha_to_scrape.py

@@ -0,0 +1,28 @@
+from src.infrastructure.persistence.unit_of_work import UnitOfWork
+from src.domain.schemas.manwha import CreateManwhaToScrapeRequest
+from src.domain.repository.scraper import ReaderRepository, ScraperManwhaRepository
+from src.domain.entities.scraper import ScraperManwha
+from src.domain.exceptions.client import BadRequestException, ConflictException
+
+
+class CreateManwhaToScrapeUseCase:
+    def __init__(self, db: UnitOfWork) -> None:
+        with db.get_session() as session:
+            self.session = session
+            self.reader_repository = ReaderRepository(session)
+            self.scraper_manwha_repository = ScraperManwhaRepository(session)
+
+    def execute(self, payload: CreateManwhaToScrapeRequest):
+        reader = self.reader_repository.get("id", payload.reader_id)
+        if not reader:
+            raise BadRequestException("The provided reader_id was not found in database")
+
+        manwha = self.scraper_manwha_repository.get("url", payload.url)
+        if manwha:
+            raise ConflictException("A manwha is already registered with the provided url")
+
+        self.scraper_manwha_repository.add(
+            ScraperManwha(reader_id=payload.reader_id, url=payload.url)
+        )
+
+        self.session.commit()

backend/src/domain/use_cases/pagination/prepare_pagination.py

@@ -8,11 +8,14 @@ class PreparePaginationUseCase:
     @staticmethod
     def execute(
         endpoint: str,
-        query: RowReturningQuery,
+        records: list | RowReturningQuery,
         current_page: int,
         per_page: int,
     ) -> Pagination:
-        entries_quantity = query.count()
+        if type(records) is list:
+            entries_quantity = len(records)
+        else:
+            entries_quantity = records.count()
 
         first_page = 1
         last_page = ceil(entries_quantity / per_page)

backend/src/domain/use_cases/scraper/base_scraper.py

@@ -76,7 +76,7 @@ def execute(self):
                             chapter_pages = self.scrape_manwha_chapter_images(chapter["url"])
 
                             self.manage_chapters.execute(
-                                manwha_id, chapter["number"], chapter_pages
+                                manwha_id, chapter["number"], chapter_pages, chapter["url"]
                             )
                             self.session.commit()
                         except TimeoutException:

backend/src/domain/use_cases/scraper/get_readers.py

@@ -0,0 +1,22 @@
+from src.domain.schemas.scraper import GetReadersResponse, ReaderData
+from src.domain.repository.scraper import ReaderRepository
+from src.domain.use_cases.pagination.prepare_pagination import PreparePaginationUseCase
+from src.infrastructure.persistence.unit_of_work import UnitOfWork
+
+
+class GetReadersUseCase:
+    def __init__(self, db: UnitOfWork):
+        with db.get_session() as session:
+            self.session = session
+            self.reader_repository = ReaderRepository(session)
+            self.prepare_pagination = PreparePaginationUseCase().execute
+
+    def execute(self, page, per_page) -> GetReadersResponse:
+        readers = self.reader_repository.get_all()
+
+        records = [ReaderData(id=reader.id, name=reader.name) for reader in readers]
+
+        return GetReadersResponse(
+            records=records,
+            pagination=self.prepare_pagination("/v1/readers", readers, page, per_page),
+        )

backend/src/presentation/api/exception_handlers.py

@@ -0,0 +1,27 @@
+from fastapi import Request
+from fastapi.responses import JSONResponse
+
+from src.domain.exceptions.client import BadRequestException, ConflictException
+from src.infrastructure.log import logger
+
+
+async def bad_request_exception_handler(request: Request, exc: BadRequestException):
+    return JSONResponse(
+        status_code=400,
+        content={"status": 400, "message": exc.message},
+    )
+
+
+async def conflict_exception_handler(request: Request, exc: ConflictException):
+    return JSONResponse(
+        status_code=409,
+        content={"status": 409, "message": exc.message},
+    )
+
+
+async def general_exception_handler(request: Request, exc: Exception):
+    logger.error(f"{exc.__class__.__name__}: {exc}")
+    return JSONResponse(
+        status_code=500,
+        content={"status": 500, "message": "Internal Server Error"},
+    )

backend/src/presentation/api/main.py

@@ -2,9 +2,16 @@
 from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
 
+from src.presentation.api.exception_handlers import (
+    bad_request_exception_handler,
+    conflict_exception_handler,
+    general_exception_handler,
+)
+from src.domain.exceptions.client import BadRequestException, ConflictException
 from src.presentation.api.v1.routers.manwhas import router as v1_manwhas
 from src.presentation.api.v1.routers.chapters import router as v1_chapters
 from src.presentation.api.v1.routers.scrapers import router as v1_scrapers
+from src.presentation.api.v1.routers.readers import router as v1_readers
 from src.infrastructure.config import SETTINGS
 
 app = FastAPI()
@@ -20,6 +27,13 @@
 app.include_router(v1_manwhas)
 app.include_router(v1_chapters)
 app.include_router(v1_scrapers)
+app.include_router(v1_readers)
+
+
+app.add_exception_handler(BadRequestException, bad_request_exception_handler)
+app.add_exception_handler(ConflictException, conflict_exception_handler)
+app.add_exception_handler(Exception, general_exception_handler)
+
 
 if __name__ == "__main__":
     uvicorn.run(

backend/src/presentation/api/v1/routers/manwhas.py

@@ -1,18 +1,20 @@
-from fastapi import APIRouter, Depends
+from fastapi import APIRouter, Depends, Response
 
 from src.infrastructure.persistence.unit_of_work import UnitOfWork
 from src.domain.use_cases.manwha.get_manwhas import GetManwhasUseCase
 from src.domain.use_cases.manwha.get_manwha import GetManwhaUseCase
+from src.domain.use_cases.manwha.create_manwha_to_scrape import CreateManwhaToScrapeUseCase
 from src.domain.schemas.manwha import (
     GetManwhasResponse,
     GetManwhaResponse,
+    CreateManwhaToScrapeRequest,
 )
 
 router = APIRouter(prefix="/api/v1/manwhas", tags=["v1"])
 
 
 @router.get("/", response_model=GetManwhasResponse, status_code=200)
-def get_manwhas(
+async def get_manwhas(
     search: str = "",
     page: int = 1,
     per_page: int = 20,
@@ -22,5 +24,11 @@ def get_manwhas(
 
 
 @router.get("/{manwha_id}", response_model=GetManwhaResponse, status_code=200)
-def get_manwha(manwha_id: int, db=Depends(UnitOfWork)):
+async def get_manwha(manwha_id: int, db=Depends(UnitOfWork)):
     return GetManwhaUseCase().execute(db, manwha_id)
+
+
+@router.post("/", status_code=201)
+async def create_manwha_to_scrape(payload: CreateManwhaToScrapeRequest, db=Depends(UnitOfWork)):
+    CreateManwhaToScrapeUseCase(db).execute(payload)
+    return Response(status_code=201)

backend/src/presentation/api/v1/routers/readers.py

@@ -0,0 +1,12 @@
+from fastapi import APIRouter, Depends
+
+from src.domain.schemas.scraper import GetReadersResponse
+from src.domain.use_cases.scraper.get_readers import GetReadersUseCase
+from src.infrastructure.persistence.unit_of_work import UnitOfWork
+
+router = APIRouter(prefix="/api/v1/readers", tags=["v1"])
+
+
+@router.get("/", response_model=GetReadersResponse, status_code=200)
+def get_readers(page: int = 1, per_page: int = 20, db=Depends(UnitOfWork)):
+    return GetReadersUseCase(db).execute(page, per_page)