From 209a6d4f457976f81c9e8f825d8824909101073d Mon Sep 17 00:00:00 2001 From: kanywst Date: Mon, 31 Aug 2026 00:25:57 +0900 Subject: [PATCH 1/2] feat(ci): cache-bust image refs when only an asset changes Re-rendering a diagram in place reached nobody. devto-cli skips an article whose markdown is byte-identical to what is live, and dev.to's Bunny CDN serves every image `immutable` for a year, so even a push that did go out left readers on the old PNG. scripts/bump_asset_versions.py rewrites `?v=` on every reference to a changed asset: the markdown now differs (devto-cli pushes) and the URL now differs (the CDN refetches). publish.yml also triggers on articles/assets/** and folds the affected articles into the same dev push batch, so an asset-only commit republishes its articles on its own. Ownership is resolved by reference, not by directory name. 40 asset directories have no same-named article (assets/spire/ backs spiffe-spire-deep-dive.md, assets/IssueHub/ backs issuehub.md), so every top-level article is scanned for a reference to each changed asset. Markdown images, , and the absolute cover_image URL are all covered; fenced blocks and inline code are not, and cover_image is only matched inside frontmatter so a YAML sample in the body cannot be corrupted. The token is a content hash rather than a counter, which makes reruns idempotent and self-healing: a run that bumps but then fails before the commit recomputes the same value next time. This replaces the manual `?v=2` bump the docs used to prescribe. --- .github/workflows/publish.yml | 23 +++- Makefile | 9 +- README.md | 3 + scripts/bump_asset_versions.py | 210 +++++++++++++++++++++++++++++++++ 4 files changed, 243 insertions(+), 2 deletions(-) create mode 100644 scripts/bump_asset_versions.py diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index f506735..994c0d1 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -6,6 +6,9 @@ on: - main paths: - 'articles/**/*.md' + # A re-rendered diagram changes no markdown, so without this the push + # never reaches the workflow. See the cache-busting step below. + - 'articles/assets/**' workflow_dispatch: inputs: scope: @@ -84,6 +87,22 @@ jobs: base="$(git hash-object -t tree /dev/null)" fi git diff -z --name-only --diff-filter=d "$base" "$AFTER" -- ':(glob)articles/*.md' > "$list" + + # An asset-only change leaves the owning article's markdown byte + # identical, so devto-cli would skip it and Bunny CDN would keep + # serving the old image for a year. Rewriting ?v= on + # every reference makes both notice. The script prints the articles + # it touched; fold them into the batch. + assets="${RUNNER_TEMP}/assets.nul" + git diff -z --name-only --diff-filter=d "$base" "$AFTER" -- ':(glob)articles/assets/**' > "$assets" + if [ -s "$assets" ]; then + mapfile -d '' -t changed_assets < "$assets" + python scripts/bump_asset_versions.py "${changed_assets[@]}" > "${RUNNER_TEMP}/bumped.txt" + # sort -z keeps the union NUL-delimited and drops the articles that + # are already in the batch from their own markdown edit. + { cat "$list"; tr '\n' '\0' < "${RUNNER_TEMP}/bumped.txt"; } | sort -z -u > "${list}.union" + mv "${list}.union" "$list" + fi fi files=() @@ -141,13 +160,15 @@ jobs: echo "Pushing ${#files[@]} article(s)." dev push "${args[@]}" - - name: Commit and push metadata written back by devto-cli + - name: Commit and push frontmatter and asset-version changes if: steps.collect.outputs.count != '0' run: | set -euo pipefail git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" + # Two sources of change here: the id/date devto-cli writes back, and + # the ?v= tokens bump_asset_versions.py rewrote during collection. git add -- ':(glob)articles/*.md' if git diff --cached --quiet; then echo "No frontmatter changes to commit." diff --git a/Makefile b/Makefile index 8b822d4..dbf3437 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ CHANGED := $(shell git diff --name-only --diff-filter=d $(BASE)...HEAD -- 'artic git diff --name-only --diff-filter=d -- 'articles/*.md' 2>/dev/null; \ git ls-files --others --exclude-standard -- 'articles/*.md' 2>/dev/null) -.PHONY: help setup check validate validate-changed lint lint-changed links links-external index index-check diagrams schedule-dry clean +.PHONY: help setup check validate validate-changed lint lint-changed links links-external index index-check diagrams bump-assets schedule-dry clean help: ## Show this help @grep -hE '^[a-z-]+:.*?## ' $(MAKEFILE_LIST) | awk 'BEGIN{FS=":.*?## "}{printf " \033[36m%-16s\033[0m %s\n", $$1, $$2}' @@ -60,6 +60,13 @@ diagrams: ## Re-render every D2 source to PNG done; \ echo "[+] Rendered $$count diagram(s)" +bump-assets: ## Cache-bust image refs for assets changed vs $(BASE) (CI does this too) + @assets="$$(git diff --name-only --diff-filter=d $(BASE)...HEAD -- 'articles/assets/' 2>/dev/null; \ + git diff --name-only --diff-filter=d -- 'articles/assets/' 2>/dev/null; \ + git ls-files --others --exclude-standard -- 'articles/assets/' 2>/dev/null)"; \ + if [ -z "$$assets" ]; then echo "[-] No changed assets."; \ + else echo "$$assets" | sort -u | xargs $(PYTHON) scripts/bump_asset_versions.py; fi + schedule-dry: ## Show which articles the scheduler would publish (writes nothing) $(PYTHON) scripts/publish_scheduler.py --dry-run diff --git a/README.md b/README.md index 0038b3f..f46e505 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,8 @@ The slug becomes part of the dev.to URL (dev.to appends a random suffix on first Images and hands-on resources go under `articles/assets//`. The publish step runs `dev push -r ${{ github.repository }}`, which rewrites relative asset paths to `raw.githubusercontent.com` URLs before sending to dev.to. Cover images at the canonical size (1000x420) can be generated with `scripts/gen_cover_image.py`. +Re-rendering an image in place used to be invisible to readers: `devto-cli` skips an article whose markdown has not changed, and dev.to's CDN caches every image URL `immutable` for a year. `scripts/bump_asset_versions.py` closes both gaps by rewriting `?v=` on every reference to a changed asset, and the publish workflow runs it automatically. + @@ -97,6 +99,7 @@ make lint-changed validate-changed # exactly what CI gates on make links-external # slow: reports third-party link rot make index # regenerate INDEX.md and README stats make diagrams # re-render every D2 source to PNG +make bump-assets # cache-bust image refs whose asset changed make schedule-dry # what the scheduler would publish, writes nothing ``` diff --git a/scripts/bump_asset_versions.py b/scripts/bump_asset_versions.py new file mode 100644 index 0000000..3e74d5b --- /dev/null +++ b/scripts/bump_asset_versions.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python3 +"""Cache-bust image references whose backing asset file changed. + +Two problems stack up when a diagram gets re-rendered in place: + +1. dev.to's Bunny CDN serves every image with `cache-control: public, + max-age=31536000, immutable`. Overwriting the PNG at the same URL never + reaches a reader whose browser already has it. +2. devto-cli only re-pushes an article whose markdown differs from what is + live. Re-rendering a diagram does not touch the markdown at all, so the + push is a no-op even if the CDN would have refetched. + +Rewriting `?v=` on every reference to the changed asset fixes both: the +URL is new (so the CDN misses) and the markdown changed (so devto-cli pushes). +The token is a content hash of the asset, so reruns are idempotent and the +query changes exactly when the image does. + +Ownership is resolved by *reference*, not by directory name: 40 of this repo's +asset directories do not have a same-named article, and some images are shared. +Every top-level article is scanned for a reference to each changed asset. + +Usage: + python scripts/bump_asset_versions.py articles/assets/foo/diagrams/01-bar.png + python scripts/bump_asset_versions.py --dry-run articles/assets/foo/cover.png + +Prints the article paths it rewrote to stdout, one per line, for the publish +workflow to fold into its push batch. An asset nobody references is a warning +on stderr, never an error. +""" + +from __future__ import annotations + +import argparse +import glob +import hashlib +import os +import re +import sys + +ARTICLES_DIR = "articles" +REPO = "0-draft/dev.to" +TOKEN_LENGTH = 8 + +# Deliberately the same patterns validate_articles.py uses, so the two scripts +# agree on what counts as an image reference. Each match ends right after the +# URL, which keeps the span arithmetic in _rewrite trivial and leaves any +# trailing title text (`![](url "title")`) untouched. +MD_IMAGE_RE = re.compile(r"!\[[^\]]*\]\(\s*(?P[^)\s]+)") +HTML_IMAGE_RE = re.compile(r"]+src=[\"'](?P[^\"']+)[\"']") +COVER_IMAGE_RE = re.compile(r"^cover_image:\s*[\"']?(?P[^\"'\s]+)", re.MULTILINE) + +FRONTMATTER_RE = re.compile(r"^---\n.*?\n---\n?", re.DOTALL) +FENCE_OPEN_RE = re.compile(r"(```+|~~~+)") +FENCE_CLOSE_RE = re.compile(r"(```+|~~~+)\s*$") +INLINE_CODE_RE = re.compile(r"`[^`\n]*`") +# https://raw.githubusercontent.com///refs/heads/main/articles/... +RAW_URL_RE = re.compile(r"/(?:refs/heads/)?[^/]+/(articles/.+)$") + + +def content_token(path: str) -> str: + """Short content hash of the asset, stable across runs and machines.""" + with open(path, "rb") as handle: + return hashlib.sha256(handle.read()).hexdigest()[:TOKEN_LENGTH] + + +def code_mask(text: str) -> bytearray: + """Mark every character that sits inside a fenced block or inline code span. + + Mirrors validate_articles.py's strip_code line-by-line fence tracking rather + than a `.*?` regex over the whole document, so `~~~` fences and four-backtick + fences are handled the same way in both scripts. + """ + mask = bytearray(len(text)) + position = 0 + fence = None + for line in text.splitlines(keepends=True): + stripped = line.lstrip() + if fence is None: + opening = FENCE_OPEN_RE.match(stripped) + if opening: + fence = opening.group(1)[0] + mask[position : position + len(line)] = b"\x01" * len(line) + else: + for match in INLINE_CODE_RE.finditer(line): + start, end = position + match.start(), position + match.end() + mask[start:end] = b"\x01" * (end - start) + else: + mask[position : position + len(line)] = b"\x01" * len(line) + if stripped[:1] == fence and FENCE_CLOSE_RE.match(stripped): + fence = None + position += len(line) + return mask + + +def resolve(url: str, article: str) -> str | None: + """Map an image reference to a repo-relative path, or None if it is not ours.""" + clean = url.split("?", 1)[0] + if clean.startswith("data:") or clean.startswith("#"): + return None + if clean.startswith("http"): + # Articles legitimately link to raw files in other repos; only this + # repo's own assets can be cache-busted from here. + if REPO not in clean: + return None + match = RAW_URL_RE.search(clean) + return os.path.normpath(match.group(1)) if match else None + return os.path.normpath(os.path.join(os.path.dirname(article), clean)) + + +def _bump(url: str, token: str) -> str: + # Everything after '?' is dropped. No image reference in this repo carries a + # query other than the version token this script owns. + return f"{url.split('?', 1)[0]}?v={token}" + + +def _rewrite(text: str, mask: bytearray, patterns, article: str, tokens, hits) -> str: + """Rewrite every reference to a changed asset in one pass. + + Collect first, splice second. Running `pattern.sub` per pattern would let + the first pattern's insertions shift `text` out from under `mask`, which is + indexed against the original: the second pattern then reads the wrong mask + byte and, once the text has grown past the mask, raises IndexError. + """ + edits = [] + for pattern in patterns: + for match in pattern.finditer(text): + if mask[match.start()]: + continue + url = match.group("url") + target = resolve(url, article) + token = tokens.get(target) + if token is None: + continue + hits.add(target) + start, end = match.span("url") + edits.append((start, end, _bump(url, token))) + + if not edits: + return text + + pieces = [] + cursor = 0 + for start, end, replacement in sorted(edits): + # The two body patterns cannot claim the same URL span, but an overlap + # would duplicate text, so drop anything that starts behind the cursor. + if start < cursor: + continue + pieces.append(text[cursor:start]) + pieces.append(replacement) + cursor = end + pieces.append(text[cursor:]) + return "".join(pieces) + + +def rewrite_article(article: str, tokens, hits) -> str | None: + """Return the article's new text, or None when nothing referenced a changed asset.""" + with open(article, encoding="utf-8") as handle: + text = handle.read() + + # cover_image only ever lives in frontmatter. Restricting its substitution to + # that span stops a fenced YAML sample in the body (an article documenting + # this very pipeline, say) from being rewritten as if it were the real key. + match = FRONTMATTER_RE.match(text) + split = match.end() if match else 0 + head, body = text[:split], text[split:] + + new_head = _rewrite(head, bytearray(len(head)), [COVER_IMAGE_RE], article, tokens, hits) + new_body = _rewrite(body, code_mask(body), [MD_IMAGE_RE, HTML_IMAGE_RE], article, tokens, hits) + + updated = new_head + new_body + return updated if updated != text else None + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("asset_paths", nargs="+", help="changed asset paths, repo-relative") + parser.add_argument("--dry-run", action="store_true", help="report what would change, write nothing") + args = parser.parse_args() + + tokens = {} + for raw in args.asset_paths: + path = os.path.normpath(raw) + if not os.path.isfile(path): + print(f"warning: {path} is not a file; skipping", file=sys.stderr) + continue + tokens[path] = content_token(path) + if not tokens: + return 0 + + hits = set() + touched = [] + for article in sorted(glob.glob(os.path.join(ARTICLES_DIR, "*.md"))): + updated = rewrite_article(article, tokens, hits) + if updated is None: + continue + if not args.dry_run: + with open(article, "w", encoding="utf-8") as handle: + handle.write(updated) + touched.append(article) + + for path in sorted(set(tokens) - hits): + print(f"warning: no article references {path}; it will not be republished", file=sys.stderr) + + for article in touched: + print(article) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 9f094bfc73717fa3b7bfb34d12311f04efb9a824 Mon Sep 17 00:00:00 2001 From: kanywst Date: Mon, 31 Aug 2026 00:35:43 +0900 Subject: [PATCH 2/2] docs(ci): note the indented-code-block blind spot in code_mask --- scripts/bump_asset_versions.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/bump_asset_versions.py b/scripts/bump_asset_versions.py index 3e74d5b..3598cdc 100644 --- a/scripts/bump_asset_versions.py +++ b/scripts/bump_asset_versions.py @@ -69,6 +69,12 @@ def code_mask(text: str) -> bytearray: Mirrors validate_articles.py's strip_code line-by-line fence tracking rather than a `.*?` regex over the whole document, so `~~~` fences and four-backtick fences are handled the same way in both scripts. + + Shares strip_code's one blind spot: a four-space indented code block is not + recognised. That is deliberate rather than merely unimplemented, because an + indented line is far more often a list-item continuation than a code block, + and the two image refs in this corpus that look indented are both list + continuations that do want bumping. Fence your code and this cannot bite. """ mask = bytearray(len(text)) position = 0