diff --git a/.dockerignore b/.dockerignore index 100a9f7..b484586 100644 --- a/.dockerignore +++ b/.dockerignore @@ -14,3 +14,7 @@ frontend/vite.config.d.ts frontend/vite.config.js __pycache__ *.pyc + +# The dedicated model-api image copies the checked-in runtime artifacts. +!runtime_resources +!runtime_resources/** diff --git a/.env.example b/.env.example index ce8c964..777a38f 100644 --- a/.env.example +++ b/.env.example @@ -9,6 +9,7 @@ HEYBLOG_BACKEND_BASE_URL=http://127.0.0.1:8000 HEYBLOG_CRAWLER_BASE_URL=http://127.0.0.1:8010 HEYBLOG_SEARCH_BASE_URL=http://127.0.0.1:8020 HEYBLOG_PERSISTENCE_BASE_URL=http://127.0.0.1:8030 +HEYBLOG_MODEL_API_BASE_URL=http://127.0.0.1:8040 # Docker-only internal service URLs HEYBLOG_DOCKER_BACKEND_BASE_URL=http://backend:8000 diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..1a2b988 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "HeyBlog_Model_API"] + path = HeyBlog_Model_API + url = git@github.com:heyblog/HeyBlog_Model_API.git diff --git a/HeyBlog_Model_API b/HeyBlog_Model_API new file mode 160000 index 0000000..1939da2 --- /dev/null +++ b/HeyBlog_Model_API @@ -0,0 +1 @@ +Subproject commit 1939da2257f6c3ccae1be3dd739ec422bc2f44af diff --git a/about-avatar-desktop.png b/about-avatar-desktop.png deleted file mode 100644 index de1bb7e..0000000 Binary files a/about-avatar-desktop.png and /dev/null differ diff --git a/backend/main.py b/backend/main.py index f1d5ca8..37b773c 100644 --- a/backend/main.py +++ b/backend/main.py @@ -475,6 +475,13 @@ def optional_user(request: Request) -> dict[str, Any] | None: except httpx.HTTPStatusError as exc: _raise_upstream_http_error(exc, default="auth_required", detail_override="auth_required") + def optional_public_user(request: Request) -> dict[str, Any] | None: + """Treat an invalid optional session as anonymous for public routes.""" + try: + return optional_user(request) + except HTTPException: + return None + def require_user(request: Request) -> dict[str, Any]: user = optional_user(request) if user is None: @@ -566,7 +573,7 @@ def lookup_blog_candidates(url: str) -> dict[str, Any]: @app.post("/api/recommendations/random-blog-batches") def post_random_recommendation_batch( payload: CreateRandomRecommendationBatchRequest, - user: dict[str, Any] | None = Depends(optional_user), + user: dict[str, Any] | None = Depends(optional_public_user), ) -> dict[str, Any]: return _call_upstream_with_http_error_translation( lambda: get_state().persistence.create_random_recommendation_batch( diff --git a/crawler/crawling/decisions/chain.py b/crawler/crawling/decisions/chain.py index a740e84..975166b 100644 --- a/crawler/crawling/decisions/chain.py +++ b/crawler/crawling/decisions/chain.py @@ -84,6 +84,7 @@ def _build_blocked_tld_filter(settings: Settings) -> BaseUrlFilter: def _build_model_consensus_filter(settings: Settings) -> BaseUrlFilter: return ModelConsensusFilter( model_root=settings.decision_model_root, + model_api_base_url=settings.model_api_base_url, strategy=settings.decision_model_consensus_strategy, consensus_threshold=settings.decision_model_consensus_threshold, ) diff --git a/crawler/crawling/decisions/consensus.py b/crawler/crawling/decisions/consensus.py index 02c9989..a1b205d 100644 --- a/crawler/crawling/decisions/consensus.py +++ b/crawler/crawling/decisions/consensus.py @@ -12,6 +12,8 @@ import sys from typing import Any +import httpx + from crawler.crawling.decisions.base import DECIDER_ROLE_SUCCESS from crawler.crawling.decisions.base import FilterDecision from crawler.crawling.decisions.base import StaticStatusUrlFilter @@ -20,6 +22,7 @@ from crawler.domain.decision_outcome import DecisionOutcome from shared.observability import get_logger from shared.observability import log_event +from shared.http_clients.model_api import ModelApiClient DEFAULT_MODEL_THRESHOLD = 0.5 DEFAULT_MODEL_WEIGHT = 1.0 @@ -211,6 +214,7 @@ class ModelConsensusFilter(StaticStatusUrlFilter): """ model_root: Path + model_api_base_url: str | None = None strategy: str = "weighted_average" consensus_threshold: float = DEFAULT_MODEL_THRESHOLD kind: str = field(init=False, default="model_consensus") @@ -218,6 +222,7 @@ class ModelConsensusFilter(StaticStatusUrlFilter): filter_reason: str = field(init=False, default="model_consensus_all_non_blog") decider_role: str = field(init=False, default=DECIDER_ROLE_SUCCESS) loaded_models: tuple[LoadedConsensusModel, ...] | None = field(default=None, init=False, repr=False) + model_api: ModelApiClient | None = field(default=None, init=False, repr=False) def __post_init__(self) -> None: """Normalize and validate the configured model-consensus strategy.""" @@ -226,6 +231,8 @@ def __post_init__(self) -> None: raise ValueError(f"unknown_model_consensus_strategy:{self.strategy}") self.strategy = normalized_strategy self.consensus_threshold = float(self.consensus_threshold) + if self.model_api_base_url: + self.model_api = ModelApiClient(self.model_api_base_url) def _ensure_models_loaded(self) -> tuple[LoadedConsensusModel, ...]: """Load and cache the latest available trainer models on demand. @@ -348,6 +355,27 @@ def _should_reject(self, probabilities: list[tuple[float, LoadedConsensusModel]] def apply(self, candidate: UrlCandidateContext) -> FilterDecision: """Keep or reject a URL using the configured model consensus strategy.""" + if self.model_api is not None: + try: + prediction = self.model_api.classify_url( + candidate.normalized_url, + title=candidate.link_text or candidate.context_text, + ) + if prediction.get("label") == "blog": + return self.confirm(accepted_by="model") + return self.reject() + except (httpx.HTTPError, ValueError, KeyError, TypeError) as exc: + log_event( + LOGGER, + event="model.api.classify_failed", + message="model API unavailable; candidate left undecided", + level=logging.WARNING, + stage="model_consensus", + url=candidate.normalized_url, + error_type=type(exc).__name__, + error_message=str(exc), + ) + return self.accept() models = self._ensure_models_loaded() if not models: return self.accept() diff --git a/crawler/crawling/discovery.py b/crawler/crawling/discovery.py index f9195cd..b0ddb01 100644 --- a/crawler/crawling/discovery.py +++ b/crawler/crawling/discovery.py @@ -28,6 +28,7 @@ "朋友", "友人", "friend", + "左邻右舍", ) NEGATIVE_KEYWORDS = ( "about", diff --git a/doc/api-docs.md b/doc/api-docs.md index 73f7876..89fe077 100644 --- a/doc/api-docs.md +++ b/doc/api-docs.md @@ -20,6 +20,7 @@ - [backend/main.py](../backend/main.py) - [crawler/main.py](../crawler/main.py) +- [HeyBlog_Model_API/app.py](../HeyBlog_Model_API/app.py) - [search/main.py](../search/main.py) - [persistence_api/main.py](../persistence_api/main.py) - [frontend/src/lib/api.ts](../frontend/src/lib/api.ts) @@ -38,6 +39,7 @@ - `frontend`：public discovery surface + protected admin surface - `backend`：统一对外 API 聚合层 - `crawler`：爬虫执行与运行时控制 +- `model-api`：供 crawler 调用的 URL 二分类内部服务 - `search`：搜索索引与查询 - `persistence-api`：持久化读写接口 - `persistence-db`：PostgreSQL 数据库 @@ -57,6 +59,7 @@ - `crawler`: `8010` - `search`: `8020` - `persistence-api`: `8030` +- `model-api`: `8040` - `persistence-db`: `5432` ## 2. API 分层总览 @@ -548,6 +551,7 @@ Admin API 同样由 `backend` 暴露，但统一位于 `/api/admin/*` 下，并 - 未登录也可调用；`visitor_id` 与 `session_id` 由前端本地生成，用于匿名统计。 - 登录后可带 `Authorization: Bearer `；backend 会把用户 ID 转发给 persistence 以便后续用户维度分析。 +- Authorization 是可选的；缺失、过期或无效 token 都按匿名用户处理，不会阻止随机博客加载。 行为说明： diff --git a/doc/config-reference.md b/doc/config-reference.md index ba0ea1d..6443585 100644 --- a/doc/config-reference.md +++ b/doc/config-reference.md @@ -59,6 +59,7 @@ Docker Compose 也会从仓库根目录的 `.env` 读取变量。 | `HEYBLOG_CRAWLER_BASE_URL` | `http://127.0.0.1:8010` | `backend` | `backend` 调用 `crawler` 的内部地址 | | `HEYBLOG_SEARCH_BASE_URL` | `http://127.0.0.1:8020` | `backend` | `backend` 调用 `search` 的内部地址 | | `HEYBLOG_PERSISTENCE_BASE_URL` | `http://127.0.0.1:8030` | `backend`、`crawler`、`search` | 三个服务访问持久化边界的内部地址 | +| `HEYBLOG_MODEL_API_BASE_URL` | `http://127.0.0.1:8040` | `crawler` | HeyBlog_Model_API URL 二分类服务地址 | | `HEYBLOG_USER_AGENT` | `HeyBlogBot/0.1 (+https://example.invalid/heyblog)` | `crawler` | 抓取请求使用的 User-Agent | | `HEYBLOG_REQUEST_TIMEOUT_SECONDS` | `10.0` | `backend`、`crawler`、`search` | 内部 HTTP client 默认超时 | | `HEYBLOG_MAX_NODES_PER_RUN` | `10` | `crawler` | 单次 crawl 默认节点上限 | diff --git a/doc/service-architecture.md b/doc/service-architecture.md index 1f3ad44..51dfa63 100644 --- a/doc/service-architecture.md +++ b/doc/service-architecture.md @@ -22,6 +22,7 @@ - [backend/main.py](../backend/main.py) - [crawler/runtime/service.py](../crawler/runtime/service.py) - [crawler/crawling/pipeline.py](../crawler/crawling/pipeline.py) +- [shared/http_clients/model_api.py](../shared/http_clients/model_api.py) - [persistence_api/graph_service.py](../persistence_api/graph_service.py) ## 1. 总体拓扑 @@ -38,6 +39,7 @@ -> SQLite 或 PostgreSQL crawler -> persistence-api +crawler -> model-api search -> persistence-api ``` @@ -59,6 +61,7 @@ search -> persistence-api | `backend` | `search` | HTTP client | 搜索查询、重建索引 | | `backend` | `persistence-api` | HTTP client | 读取 blogs、edges、stats、graph、snapshot | | `crawler` | `persistence-api` | HTTP client | 领取任务、写 blog、写 edge、导出图 | +| `crawler` | `model-api` | HTTP client | 对候选 URL 执行 blog / non-blog 二分类 | | `search` | `persistence-api` | HTTP client | 拉取搜索快照 | | `persistence-api` | SQLite / PostgreSQL | Repository | 持久化事实数据与聚合读模型 | @@ -83,13 +86,19 @@ search -> persistence-api 它不直接 import crawler 业务逻辑，也不直接操作数据库。 -### 3.3 crawler / search -> persistence-api +### 3.3 crawler -> model-api + +[shared/http_clients/model_api.py](../shared/http_clients/model_api.py) 调用 +`HeyBlog_Model_API` 的 `POST /v1/classify`，模型和 embedding 资源由独立 +`model-api` 容器加载；crawler 不再在默认路径中直接加载模型文件。 + +### 3.4 crawler / search -> persistence-api `crawler` 和 `search` 都不直接访问数据库，它们都通过 `PersistenceHttpClient` 调用 `persistence-api`。这让 SQLite / PostgreSQL 差异被集中收口在 [persistence_api/repository.py](../persistence_api/repository.py)。 -### 3.4 persistence-api -> 存储后端 +### 3.5 persistence-api -> 存储后端 `persistence-api` 内部根据 [shared/config.py](../shared/config.py) 选择后端： @@ -131,6 +140,7 @@ search -> persistence-api -> crawler -> persistence-api /internal/queue/next -> crawler 抓取首页并发现友链页 -> crawler 抽取候选链接并过滤 + -> crawler -> model-api /v1/classify -> crawler -> persistence-api /internal/blogs/upsert -> crawler -> persistence-api /internal/edges -> crawler -> persistence-api /internal/blogs/{id}/result diff --git a/doc/services-overview.md b/doc/services-overview.md index 53a9b4b..f3b2c72 100644 --- a/doc/services-overview.md +++ b/doc/services-overview.md @@ -29,9 +29,10 @@ | --- | --- | --- | --- | --- | | `frontend` | `3000` | [frontend/server.py](../frontend/server.py) | 浏览器 | `backend` | | `backend` | `8000` | [backend/main.py](../backend/main.py) | 前端与外部调用方 | `crawler`、`search`、`persistence-api` | -| `crawler` | `8010` | [crawler/main.py](../crawler/main.py) | `backend` | `persistence-api` | +| `crawler` | `8010` | [crawler/main.py](../crawler/main.py) | `backend` | `persistence-api`、`model-api` | | `search` | `8020` | [search/main.py](../search/main.py) | `backend` | `persistence-api` | | `persistence-api` | `8030` | [persistence_api/main.py](../persistence_api/main.py) | `backend`、`crawler`、`search` | SQLite 或 PostgreSQL | +| `model-api` | `8040` | [HeyBlog_Model_API/app.py](../HeyBlog_Model_API/app.py) | `crawler` | URL 二分类模型、embedding 模型 | | `persistence-db` | `5432` | [docker-compose.yml](../docker-compose.yml) 中的 Postgres 服务 | `persistence-api` | 本地卷 `volumes/postgres` | ## 2. frontend @@ -86,6 +87,7 @@ - 从 `seed.csv` 导入种子 - 抓取首页并发现友链页候选 - 从候选页抽取友链链接并过滤 +- 通过 `model-api` 对候选 URL 执行 blog / non-blog 二分类 - 将 blog / edge / log 写回 `persistence-api` - 导出 `nodes.csv`、`edges.csv`、`graph.json` - 提供同步执行与运行时控制两类能力 @@ -101,6 +103,7 @@ - 抽取： [crawler/crawling/extraction.py](../crawler/crawling/extraction.py) - 过滤： [crawler/filters.py](../crawler/filters.py) - 归一化： [crawler/crawling/normalization.py](../crawler/crawling/normalization.py) +- 模型服务 client： [shared/http_clients/model_api.py](../shared/http_clients/model_api.py) ### 4.3 什么时候改这里 diff --git a/docker-compose.yml b/docker-compose.yml index 85d9a1d..51defb6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,27 @@ services: + model-api: + build: + context: . + dockerfile: docker/model-api.Dockerfile + ports: + - "${HEYBLOG_MODEL_API_PORT:-8040}:${HEYBLOG_MODEL_API_PORT:-8040}" + environment: + HEYBLOG_MODEL_PATH: /app/runtime_resources/binary_classification_model/model.joblib + HEYBLOG_EMBEDDING_MODEL_PATH: /app/runtime_resources/embedding_model + HEYBLOG_DATABASE_PATH: /data/model-api.sqlite3 + HEYBLOG_DEVICE: ${HEYBLOG_MODEL_DEVICE:-cpu} + volumes: + - ./volumes/model-api:/data + - ./runtime_resources:/app/runtime_resources:ro + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8040/health')"] + interval: 15s + timeout: 5s + retries: 5 + start_period: 30s + networks: + - heyblog_net + frontend: build: . ports: @@ -72,6 +95,7 @@ services: - "${HEYBLOG_CRAWLER_PORT:-8010}:${HEYBLOG_CRAWLER_PORT:-8010}" environment: HEYBLOG_PERSISTENCE_BASE_URL: ${HEYBLOG_DOCKER_PERSISTENCE_BASE_URL:-http://persistence-api:8030} + HEYBLOG_MODEL_API_BASE_URL: ${HEYBLOG_DOCKER_MODEL_API_BASE_URL:-http://model-api:8040} HEYBLOG_SEED_PATH: ${HEYBLOG_DOCKER_SEED_PATH:-/data/seeds/seed.csv} HEYBLOG_EXPORT_DIR: ${HEYBLOG_DOCKER_EXPORT_DIR:-/data/exports} HEYBLOG_REQUEST_TIMEOUT_SECONDS: ${HEYBLOG_REQUEST_TIMEOUT_SECONDS:-10.0} @@ -107,6 +131,8 @@ services: depends_on: persistence-api: condition: service_healthy + model-api: + condition: service_healthy volumes: - ./volumes/exports:/data/exports - ./volumes/logs:/data/logs diff --git a/docker/model-api.Dockerfile b/docker/model-api.Dockerfile new file mode 100644 index 0000000..2848bdd --- /dev/null +++ b/docker/model-api.Dockerfile @@ -0,0 +1,16 @@ +FROM python:3.11-slim + +WORKDIR /app +ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 + +COPY HeyBlog_Model_API /app/model_api +COPY runtime_resources /app/runtime_resources + +RUN pip install --no-cache-dir \ + "/app/model_api/external/HeyBlog_Model[trainer]" \ + "/app/model_api/external/HeyBlog_Model_Agent" \ + "/app/model_api" + +WORKDIR /app/model_api +EXPOSE 8040 +CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8040"] diff --git a/shared/config.py b/shared/config.py index 6d20b08..bb4ab85 100644 --- a/shared/config.py +++ b/shared/config.py @@ -27,6 +27,7 @@ DEFAULT_DECISION_MODEL_ROOT = PROJECT_ROOT / "runtime_resources" / "models" / "url_decision" / "current" DEFAULT_FILTER_CHAIN_CONFIG_PATH = PROJECT_ROOT / "runtime_resources" / "filter_chain.toml" DEFAULT_PERSISTENCE_BASE_URL = "http://127.0.0.1:8030" +DEFAULT_MODEL_API_BASE_URL = "http://127.0.0.1:8040" DEFAULT_CRAWLER_BASE_URL = "http://127.0.0.1:8010" DEFAULT_SEARCH_BASE_URL = "http://127.0.0.1:8020" DEFAULT_BACKEND_BASE_URL = "http://127.0.0.1:8000" @@ -95,6 +96,7 @@ class Settings: export_dir: Path db_dsn: str | None = None persistence_base_url: str = "http://127.0.0.1:8030" + model_api_base_url: str = DEFAULT_MODEL_API_BASE_URL crawler_base_url: str = "http://127.0.0.1:8010" search_base_url: str = "http://127.0.0.1:8020" backend_base_url: str = "http://127.0.0.1:8000" @@ -159,6 +161,7 @@ def from_env(cls) -> "Settings": seed_path=seed_path, export_dir=export_dir, persistence_base_url=os.getenv("HEYBLOG_PERSISTENCE_BASE_URL", DEFAULT_PERSISTENCE_BASE_URL), + model_api_base_url=os.getenv("HEYBLOG_MODEL_API_BASE_URL", DEFAULT_MODEL_API_BASE_URL).rstrip("/"), crawler_base_url=os.getenv("HEYBLOG_CRAWLER_BASE_URL", DEFAULT_CRAWLER_BASE_URL), search_base_url=os.getenv("HEYBLOG_SEARCH_BASE_URL", DEFAULT_SEARCH_BASE_URL), backend_base_url=os.getenv("HEYBLOG_BACKEND_BASE_URL", DEFAULT_BACKEND_BASE_URL), diff --git a/shared/http_clients/model_api.py b/shared/http_clients/model_api.py new file mode 100644 index 0000000..3dfc5ce --- /dev/null +++ b/shared/http_clients/model_api.py @@ -0,0 +1,41 @@ +"""HTTP client for the HeyBlog model classification service.""" + +from __future__ import annotations + +from typing import Any + +import httpx + +from shared.http_clients.context import context_header_kwargs + + +class ModelApiClient: + """Classify one public URL through HeyBlog_Model_API.""" + + def __init__(self, base_url: str, *, timeout_seconds: float = 30.0) -> None: + self.client = httpx.Client(base_url=base_url.rstrip("/"), timeout=timeout_seconds) + + def classify_url(self, url: str, *, title: str = "") -> dict[str, Any]: + """Return the model API prediction for one URL. + + Args: + url: Absolute public URL to classify. + title: Optional anchor/context title forwarded to feature extraction. + + Returns: + The single prediction object returned by the model API. + """ + response = self.client.post( + "/v1/classify", + json={"url": {"url": url, "title": title}, "fetch_connections": False}, + **context_header_kwargs(), + ) + response.raise_for_status() + payload = response.json() + predictions = payload.get("predictions") + if not isinstance(predictions, list) or len(predictions) != 1: + raise ValueError("model_api_invalid_prediction_response") + prediction = predictions[0] + if not isinstance(prediction, dict): + raise ValueError("model_api_invalid_prediction") + return prediction diff --git a/tests/test_crawler_model_consensus.py b/tests/test_crawler_model_consensus.py index c49e3f5..268675d 100644 --- a/tests/test_crawler_model_consensus.py +++ b/tests/test_crawler_model_consensus.py @@ -6,6 +6,8 @@ from pathlib import Path from crawler.crawling.decisions.consensus import ModelConsensusDecider +from crawler.crawling.decisions.consensus import ModelConsensusFilter +from crawler.crawling.decisions.base import UrlCandidateContext from crawler.crawling.pipeline import CrawlPipeline from persistence_api.repository import Repository from shared.config import Settings @@ -236,6 +238,27 @@ def test_model_consensus_skips_cleanly_when_no_models_exist(tmp_path: Path) -> N assert decision.reasons == ("model_consensus_skipped_no_models",) +def test_model_api_consensus_uses_single_prediction(monkeypatch, tmp_path: Path) -> None: + """The configured runtime path delegates URL classification to Model API.""" + monkeypatch.setattr( + "crawler.crawling.decisions.consensus.ModelApiClient.classify_url", + lambda self, url, title="": {"url": url, "label": "blog", "probability": 0.99}, + ) + decision = ModelConsensusFilter( + model_root=tmp_path / "unused", + model_api_base_url="http://model-api:8040", + ).apply( + UrlCandidateContext( + source_blog_id=1, + source_domain="source.example.com", + normalized_url="https://friend.example.com/", + link_text="My blog", + ) + ) + assert decision.confirmed is True + assert decision.accepted_by == "model" + + def test_pipeline_appends_model_consensus_step_when_enabled(tmp_path: Path) -> None: """Pipeline should wire the consensus step after deterministic hard rules.""" settings = Settings( diff --git a/tests/test_service_split.py b/tests/test_service_split.py index 86fae93..7d8f64f 100644 --- a/tests/test_service_split.py +++ b/tests/test_service_split.py @@ -1169,6 +1169,34 @@ def get_admin_hourly_stats(self, *, limit: int = 24) -> dict[str, object]: assert persistence.event_payload["entrance_url"] == "http://localhost/random" +def test_random_recommendation_ignores_invalid_optional_session() -> None: + """Public random recommendations should fall back to anonymous access.""" + + class PersistenceStub: + def get_user_by_session_token(self, *, token: str) -> dict[str, object] | None: + response = httpx.Response(401, request=httpx.Request("GET", "http://persistence")) + raise httpx.HTTPStatusError("expired", request=response.request, response=response) + + def create_random_recommendation_batch(self, **kwargs: object) -> dict[str, object]: + assert kwargs["user_id"] is None + return {"request_uuid": "request-1", "items": []} + + app = create_backend_app( + BackendState( + persistence=PersistenceStub(), + crawler=StubCrawler(), + search=StubSearch(), + ) + ) + response = TestClient(app).post( + "/api/recommendations/random-blog-batches", + headers={"authorization": "Bearer expired-token"}, + json={"count": 9, "visitor_id": "visitor-1", "session_id": "session-1"}, + ) + + assert response.status_code == 200 + + def test_settings_can_enable_postgres_runtime(tmp_path: Path, monkeypatch) -> None: """Environment loading should allow the split runtime to point at Postgres.""" monkeypatch.setenv("HEYBLOG_DB_DSN", "postgresql://heyblog:heyblog@persistence-db:5432/heyblog")