ligeaaa · ligeaaa · Aug 4, 2026 · Aug 4, 2026
diff --git a/.dockerignore b/.dockerignore
@@ -14,3 +14,7 @@ frontend/vite.config.d.ts
 frontend/vite.config.js
 __pycache__
 *.pyc
+
+# The dedicated model-api image copies the checked-in runtime artifacts.
+!runtime_resources
+!runtime_resources/**
diff --git a/.env.example b/.env.example
@@ -9,6 +9,7 @@ HEYBLOG_BACKEND_BASE_URL=http://127.0.0.1:8000
 HEYBLOG_CRAWLER_BASE_URL=http://127.0.0.1:8010
 HEYBLOG_SEARCH_BASE_URL=http://127.0.0.1:8020
 HEYBLOG_PERSISTENCE_BASE_URL=http://127.0.0.1:8030
+HEYBLOG_MODEL_API_BASE_URL=http://127.0.0.1:8040
 
 # Docker-only internal service URLs
 HEYBLOG_DOCKER_BACKEND_BASE_URL=http://backend:8000

diff --git a/.gitmodules b/.gitmodules
@@ -0,0 +1,3 @@
+[submodule "HeyBlog_Model_API"]
+	path = HeyBlog_Model_API
+	url = git@github.com:heyblog/HeyBlog_Model_API.git
diff --git a/HeyBlog_Model_API b/HeyBlog_Model_API
diff --git a/about-avatar-desktop.png b/about-avatar-desktop.png
diff --git a/backend/main.py b/backend/main.py
@@ -475,6 +475,13 @@ def optional_user(request: Request) -> dict[str, Any] | None:
         except httpx.HTTPStatusError as exc:
             _raise_upstream_http_error(exc, default="auth_required", detail_override="auth_required")
 
+    def optional_public_user(request: Request) -> dict[str, Any] | None:
+        """Treat an invalid optional session as anonymous for public routes."""
+        try:
+            return optional_user(request)
+        except HTTPException:
+            return None
+
     def require_user(request: Request) -> dict[str, Any]:
         user = optional_user(request)
         if user is None:
@@ -566,7 +573,7 @@ def lookup_blog_candidates(url: str) -> dict[str, Any]:
     @app.post("/api/recommendations/random-blog-batches")
     def post_random_recommendation_batch(
         payload: CreateRandomRecommendationBatchRequest,
-        user: dict[str, Any] | None = Depends(optional_user),
+        user: dict[str, Any] | None = Depends(optional_public_user),
     ) -> dict[str, Any]:
         return _call_upstream_with_http_error_translation(
             lambda: get_state().persistence.create_random_recommendation_batch(

diff --git a/crawler/crawling/decisions/chain.py b/crawler/crawling/decisions/chain.py
@@ -84,6 +84,7 @@ def _build_blocked_tld_filter(settings: Settings) -> BaseUrlFilter:
 def _build_model_consensus_filter(settings: Settings) -> BaseUrlFilter:
     return ModelConsensusFilter(
         model_root=settings.decision_model_root,
+        model_api_base_url=settings.model_api_base_url,
         strategy=settings.decision_model_consensus_strategy,
         consensus_threshold=settings.decision_model_consensus_threshold,
     )

diff --git a/crawler/crawling/decisions/consensus.py b/crawler/crawling/decisions/consensus.py
@@ -12,6 +12,8 @@
 import sys
 from typing import Any
 
+import httpx
+
 from crawler.crawling.decisions.base import DECIDER_ROLE_SUCCESS
 from crawler.crawling.decisions.base import FilterDecision
 from crawler.crawling.decisions.base import StaticStatusUrlFilter
@@ -20,6 +22,7 @@
 from crawler.domain.decision_outcome import DecisionOutcome
 from shared.observability import get_logger
 from shared.observability import log_event
+from shared.http_clients.model_api import ModelApiClient
 
 DEFAULT_MODEL_THRESHOLD = 0.5
 DEFAULT_MODEL_WEIGHT = 1.0
@@ -211,13 +214,15 @@ class ModelConsensusFilter(StaticStatusUrlFilter):
     """
 
     model_root: Path
+    model_api_base_url: str | None = None
     strategy: str = "weighted_average"
     consensus_threshold: float = DEFAULT_MODEL_THRESHOLD
     kind: str = field(init=False, default="model_consensus")
     filter_kind: str = field(init=False, default="model")
     filter_reason: str = field(init=False, default="model_consensus_all_non_blog")
     decider_role: str = field(init=False, default=DECIDER_ROLE_SUCCESS)
     loaded_models: tuple[LoadedConsensusModel, ...] | None = field(default=None, init=False, repr=False)
+    model_api: ModelApiClient | None = field(default=None, init=False, repr=False)
 
     def __post_init__(self) -> None:
         """Normalize and validate the configured model-consensus strategy."""
@@ -226,6 +231,8 @@ def __post_init__(self) -> None:
             raise ValueError(f"unknown_model_consensus_strategy:{self.strategy}")
         self.strategy = normalized_strategy
         self.consensus_threshold = float(self.consensus_threshold)
+        if self.model_api_base_url:
+            self.model_api = ModelApiClient(self.model_api_base_url)
 
     def _ensure_models_loaded(self) -> tuple[LoadedConsensusModel, ...]:
         """Load and cache the latest available trainer models on demand.
@@ -348,6 +355,27 @@ def _should_reject(self, probabilities: list[tuple[float, LoadedConsensusModel]]
 
     def apply(self, candidate: UrlCandidateContext) -> FilterDecision:
         """Keep or reject a URL using the configured model consensus strategy."""
+        if self.model_api is not None:
+            try:
+                prediction = self.model_api.classify_url(
+                    candidate.normalized_url,
+                    title=candidate.link_text or candidate.context_text,
+                )
+                if prediction.get("label") == "blog":
+                    return self.confirm(accepted_by="model")
+                return self.reject()
+            except (httpx.HTTPError, ValueError, KeyError, TypeError) as exc:
+                log_event(
+                    LOGGER,
+                    event="model.api.classify_failed",
+                    message="model API unavailable; candidate left undecided",
+                    level=logging.WARNING,
+                    stage="model_consensus",
+                    url=candidate.normalized_url,
+                    error_type=type(exc).__name__,
+                    error_message=str(exc),
+                )
+                return self.accept()
         models = self._ensure_models_loaded()
         if not models:
             return self.accept()

diff --git a/crawler/crawling/discovery.py b/crawler/crawling/discovery.py
@@ -28,6 +28,7 @@
     "朋友",
     "友人",
     "friend",
+    "左邻右舍",
 )
 NEGATIVE_KEYWORDS = (
     "about",

diff --git a/doc/api-docs.md b/doc/api-docs.md
@@ -20,6 +20,7 @@
 
 - [backend/main.py](../backend/main.py)
 - [crawler/main.py](../crawler/main.py)
+- [HeyBlog_Model_API/app.py](../HeyBlog_Model_API/app.py)
 - [search/main.py](../search/main.py)
 - [persistence_api/main.py](../persistence_api/main.py)
 - [frontend/src/lib/api.ts](../frontend/src/lib/api.ts)
@@ -38,6 +39,7 @@
 - `frontend`：public discovery surface + protected admin surface
 - `backend`：统一对外 API 聚合层
 - `crawler`：爬虫执行与运行时控制
+- `model-api`：供 crawler 调用的 URL 二分类内部服务
 - `search`：搜索索引与查询
 - `persistence-api`：持久化读写接口
 - `persistence-db`：PostgreSQL 数据库
@@ -57,6 +59,7 @@
 - `crawler`: `8010`
 - `search`: `8020`
 - `persistence-api`: `8030`
+- `model-api`: `8040`
 - `persistence-db`: `5432`
 
 ## 2. API 分层总览
@@ -548,6 +551,7 @@ Admin API 同样由 `backend` 暴露，但统一位于 `/api/admin/*` 下，并
 
 - 未登录也可调用；`visitor_id` 与 `session_id` 由前端本地生成，用于匿名统计。
 - 登录后可带 `Authorization: Bearer <session-token>`；backend 会把用户 ID 转发给 persistence 以便后续用户维度分析。
+- Authorization 是可选的；缺失、过期或无效 token 都按匿名用户处理，不会阻止随机博客加载。
 
 行为说明：
 

diff --git a/doc/config-reference.md b/doc/config-reference.md
@@ -59,6 +59,7 @@ Docker Compose 也会从仓库根目录的 `.env` 读取变量。
 | `HEYBLOG_CRAWLER_BASE_URL` | `http://127.0.0.1:8010` | `backend` | `backend` 调用 `crawler` 的内部地址 |
 | `HEYBLOG_SEARCH_BASE_URL` | `http://127.0.0.1:8020` | `backend` | `backend` 调用 `search` 的内部地址 |
 | `HEYBLOG_PERSISTENCE_BASE_URL` | `http://127.0.0.1:8030` | `backend`、`crawler`、`search` | 三个服务访问持久化边界的内部地址 |
+| `HEYBLOG_MODEL_API_BASE_URL` | `http://127.0.0.1:8040` | `crawler` | HeyBlog_Model_API URL 二分类服务地址 |
 | `HEYBLOG_USER_AGENT` | `HeyBlogBot/0.1 (+https://example.invalid/heyblog)` | `crawler` | 抓取请求使用的 User-Agent |
 | `HEYBLOG_REQUEST_TIMEOUT_SECONDS` | `10.0` | `backend`、`crawler`、`search` | 内部 HTTP client 默认超时 |
 | `HEYBLOG_MAX_NODES_PER_RUN` | `10` | `crawler` | 单次 crawl 默认节点上限 |

diff --git a/doc/service-architecture.md b/doc/service-architecture.md
@@ -22,6 +22,7 @@
 - [backend/main.py](../backend/main.py)
 - [crawler/runtime/service.py](../crawler/runtime/service.py)
 - [crawler/crawling/pipeline.py](../crawler/crawling/pipeline.py)
+- [shared/http_clients/model_api.py](../shared/http_clients/model_api.py)
 - [persistence_api/graph_service.py](../persistence_api/graph_service.py)
 
 ## 1. 总体拓扑
@@ -38,6 +39,7 @@
            -> SQLite 或 PostgreSQL
 
 crawler -> persistence-api
+crawler -> model-api
 search  -> persistence-api
 ```
 
@@ -59,6 +61,7 @@ search  -> persistence-api
 | `backend` | `search` | HTTP client | 搜索查询、重建索引 |
 | `backend` | `persistence-api` | HTTP client | 读取 blogs、edges、stats、graph、snapshot |
 | `crawler` | `persistence-api` | HTTP client | 领取任务、写 blog、写 edge、导出图 |
+| `crawler` | `model-api` | HTTP client | 对候选 URL 执行 blog / non-blog 二分类 |
 | `search` | `persistence-api` | HTTP client | 拉取搜索快照 |
 | `persistence-api` | SQLite / PostgreSQL | Repository | 持久化事实数据与聚合读模型 |
 
@@ -83,13 +86,19 @@ search  -> persistence-api
 
 它不直接 import crawler 业务逻辑，也不直接操作数据库。
 
-### 3.3 crawler / search -> persistence-api
+### 3.3 crawler -> model-api
+
+[shared/http_clients/model_api.py](../shared/http_clients/model_api.py) 调用
+`HeyBlog_Model_API` 的 `POST /v1/classify`，模型和 embedding 资源由独立
+`model-api` 容器加载；crawler 不再在默认路径中直接加载模型文件。
+
+### 3.4 crawler / search -> persistence-api
 
 `crawler` 和 `search` 都不直接访问数据库，它们都通过 `PersistenceHttpClient`
 调用 `persistence-api`。这让 SQLite / PostgreSQL 差异被集中收口在
 [persistence_api/repository.py](../persistence_api/repository.py)。
 
-### 3.4 persistence-api -> 存储后端
+### 3.5 persistence-api -> 存储后端
 
 `persistence-api` 内部根据 [shared/config.py](../shared/config.py) 选择后端：
 
@@ -131,6 +140,7 @@ search  -> persistence-api
   -> crawler -> persistence-api /internal/queue/next
   -> crawler 抓取首页并发现友链页
   -> crawler 抽取候选链接并过滤
+  -> crawler -> model-api /v1/classify
   -> crawler -> persistence-api /internal/blogs/upsert
   -> crawler -> persistence-api /internal/edges
   -> crawler -> persistence-api /internal/blogs/{id}/result

diff --git a/doc/services-overview.md b/doc/services-overview.md
@@ -29,9 +29,10 @@
 | --- | --- | --- | --- | --- |
 | `frontend` | `3000` | [frontend/server.py](../frontend/server.py) | 浏览器 | `backend` |
 | `backend` | `8000` | [backend/main.py](../backend/main.py) | 前端与外部调用方 | `crawler`、`search`、`persistence-api` |
-| `crawler` | `8010` | [crawler/main.py](../crawler/main.py) | `backend` | `persistence-api` |
+| `crawler` | `8010` | [crawler/main.py](../crawler/main.py) | `backend` | `persistence-api`、`model-api` |
 | `search` | `8020` | [search/main.py](../search/main.py) | `backend` | `persistence-api` |
 | `persistence-api` | `8030` | [persistence_api/main.py](../persistence_api/main.py) | `backend`、`crawler`、`search` | SQLite 或 PostgreSQL |
+| `model-api` | `8040` | [HeyBlog_Model_API/app.py](../HeyBlog_Model_API/app.py) | `crawler` | URL 二分类模型、embedding 模型 |
 | `persistence-db` | `5432` | [docker-compose.yml](../docker-compose.yml) 中的 Postgres 服务 | `persistence-api` | 本地卷 `volumes/postgres` |
 
 ## 2. frontend
@@ -86,6 +87,7 @@
 - 从 `seed.csv` 导入种子
 - 抓取首页并发现友链页候选
 - 从候选页抽取友链链接并过滤
+- 通过 `model-api` 对候选 URL 执行 blog / non-blog 二分类
 - 将 blog / edge / log 写回 `persistence-api`
 - 导出 `nodes.csv`、`edges.csv`、`graph.json`
 - 提供同步执行与运行时控制两类能力
@@ -101,6 +103,7 @@
 - 抽取： [crawler/crawling/extraction.py](../crawler/crawling/extraction.py)
 - 过滤： [crawler/filters.py](../crawler/filters.py)
 - 归一化： [crawler/crawling/normalization.py](../crawler/crawling/normalization.py)
+- 模型服务 client： [shared/http_clients/model_api.py](../shared/http_clients/model_api.py)
 
 ### 4.3 什么时候改这里
 

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -1,4 +1,27 @@
 services:
+  model-api:
+    build:
+      context: .
+      dockerfile: docker/model-api.Dockerfile
+    ports:
+      - "${HEYBLOG_MODEL_API_PORT:-8040}:${HEYBLOG_MODEL_API_PORT:-8040}"
+    environment:
+      HEYBLOG_MODEL_PATH: /app/runtime_resources/binary_classification_model/model.joblib
+      HEYBLOG_EMBEDDING_MODEL_PATH: /app/runtime_resources/embedding_model
+      HEYBLOG_DATABASE_PATH: /data/model-api.sqlite3
+      HEYBLOG_DEVICE: ${HEYBLOG_MODEL_DEVICE:-cpu}
+    volumes:
+      - ./volumes/model-api:/data
+      - ./runtime_resources:/app/runtime_resources:ro
+    healthcheck:
+      test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8040/health')"]
+      interval: 15s
+      timeout: 5s
+      retries: 5
+      start_period: 30s
+    networks:
+      - heyblog_net
+
   frontend:
     build: .
     ports:
@@ -72,6 +95,7 @@ services:
       - "${HEYBLOG_CRAWLER_PORT:-8010}:${HEYBLOG_CRAWLER_PORT:-8010}"
     environment:
       HEYBLOG_PERSISTENCE_BASE_URL: ${HEYBLOG_DOCKER_PERSISTENCE_BASE_URL:-http://persistence-api:8030}
+      HEYBLOG_MODEL_API_BASE_URL: ${HEYBLOG_DOCKER_MODEL_API_BASE_URL:-http://model-api:8040}
       HEYBLOG_SEED_PATH: ${HEYBLOG_DOCKER_SEED_PATH:-/data/seeds/seed.csv}
       HEYBLOG_EXPORT_DIR: ${HEYBLOG_DOCKER_EXPORT_DIR:-/data/exports}
       HEYBLOG_REQUEST_TIMEOUT_SECONDS: ${HEYBLOG_REQUEST_TIMEOUT_SECONDS:-10.0}
@@ -107,6 +131,8 @@ services:
     depends_on:
       persistence-api:
         condition: service_healthy
+      model-api:
+        condition: service_healthy
     volumes:
       - ./volumes/exports:/data/exports
       - ./volumes/logs:/data/logs

diff --git a/docker/model-api.Dockerfile b/docker/model-api.Dockerfile
@@ -0,0 +1,16 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1
+
+COPY HeyBlog_Model_API /app/model_api
+COPY runtime_resources /app/runtime_resources
+
+RUN pip install --no-cache-dir \
+    "/app/model_api/external/HeyBlog_Model[trainer]" \
+    "/app/model_api/external/HeyBlog_Model_Agent" \
+    "/app/model_api"
+
+WORKDIR /app/model_api
+EXPOSE 8040
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8040"]
diff --git a/shared/config.py b/shared/config.py
@@ -27,6 +27,7 @@
 DEFAULT_DECISION_MODEL_ROOT = PROJECT_ROOT / "runtime_resources" / "models" / "url_decision" / "current"
 DEFAULT_FILTER_CHAIN_CONFIG_PATH = PROJECT_ROOT / "runtime_resources" / "filter_chain.toml"
 DEFAULT_PERSISTENCE_BASE_URL = "http://127.0.0.1:8030"
+DEFAULT_MODEL_API_BASE_URL = "http://127.0.0.1:8040"
 DEFAULT_CRAWLER_BASE_URL = "http://127.0.0.1:8010"
 DEFAULT_SEARCH_BASE_URL = "http://127.0.0.1:8020"
 DEFAULT_BACKEND_BASE_URL = "http://127.0.0.1:8000"
@@ -95,6 +96,7 @@ class Settings:
     export_dir: Path
     db_dsn: str | None = None
     persistence_base_url: str = "http://127.0.0.1:8030"
+    model_api_base_url: str = DEFAULT_MODEL_API_BASE_URL
     crawler_base_url: str = "http://127.0.0.1:8010"
     search_base_url: str = "http://127.0.0.1:8020"
     backend_base_url: str = "http://127.0.0.1:8000"
@@ -159,6 +161,7 @@ def from_env(cls) -> "Settings":
             seed_path=seed_path,
             export_dir=export_dir,
             persistence_base_url=os.getenv("HEYBLOG_PERSISTENCE_BASE_URL", DEFAULT_PERSISTENCE_BASE_URL),
+            model_api_base_url=os.getenv("HEYBLOG_MODEL_API_BASE_URL", DEFAULT_MODEL_API_BASE_URL).rstrip("/"),
             crawler_base_url=os.getenv("HEYBLOG_CRAWLER_BASE_URL", DEFAULT_CRAWLER_BASE_URL),
             search_base_url=os.getenv("HEYBLOG_SEARCH_BASE_URL", DEFAULT_SEARCH_BASE_URL),
             backend_base_url=os.getenv("HEYBLOG_BACKEND_BASE_URL", DEFAULT_BACKEND_BASE_URL),

diff --git a/shared/http_clients/model_api.py b/shared/http_clients/model_api.py
@@ -0,0 +1,41 @@
+"""HTTP client for the HeyBlog model classification service."""
+
+from __future__ import annotations
+
+from typing import Any
+
+import httpx
+
+from shared.http_clients.context import context_header_kwargs
+
+
+class ModelApiClient:
+    """Classify one public URL through HeyBlog_Model_API."""
+
+    def __init__(self, base_url: str, *, timeout_seconds: float = 30.0) -> None:
+        self.client = httpx.Client(base_url=base_url.rstrip("/"), timeout=timeout_seconds)
+
+    def classify_url(self, url: str, *, title: str = "") -> dict[str, Any]:
+        """Return the model API prediction for one URL.
+
+        Args:
+            url: Absolute public URL to classify.
+            title: Optional anchor/context title forwarded to feature extraction.
+
+        Returns:
+            The single prediction object returned by the model API.
+        """
+        response = self.client.post(
+            "/v1/classify",
+            json={"url": {"url": url, "title": title}, "fetch_connections": False},
+            **context_header_kwargs(),
+        )
+        response.raise_for_status()
+        payload = response.json()
+        predictions = payload.get("predictions")
+        if not isinstance(predictions, list) or len(predictions) != 1:
+            raise ValueError("model_api_invalid_prediction_response")
+        prediction = predictions[0]
+        if not isinstance(prediction, dict):
+            raise ValueError("model_api_invalid_prediction")
+        return prediction