diff --git a/.github/workflows/feishu-pages.yml b/.github/workflows/feishu-pages.yml index 987bc51..c46b59d 100644 --- a/.github/workflows/feishu-pages.yml +++ b/.github/workflows/feishu-pages.yml @@ -19,13 +19,11 @@ jobs: FEISHU_APP_SECRET: ${{ secrets.FEISHU_APP_SECRET }} FEISHU_SPACE_ID: '7273324757679325186' OUTPUT_DIR: ./dist - ASSET_BASE_URL: 'https://longbridgeapp.github.io/feishu-pages/assets' uses: longbridgeapp/feishu-pages@main - name: Build Pages run: | mkdir -p example-website/public/assets/ - cp -R dist/docs/* example-website/ - cp -R dist/assets/* example-website/public/assets/ + cp -R dist/docs example-website/ cp dist/docs.json example-website/ yarn yarn workspace example-website build diff --git a/.gitignore b/.gitignore index 76cb3cb..e4b933c 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,5 @@ dist/ out/ node_modules/ .DS_Store -**/*/.vitepress/cache \ No newline at end of file +**/*/.vitepress/cache +.cache/ \ No newline at end of file diff --git a/README.md b/README.md index c91ed9d..2dd5507 100644 --- a/README.md +++ b/README.md @@ -31,14 +31,13 @@ yarn add feishu-pages ## Configuration -| Name | Description | Required | Default | -| ------------------- | ----------------------------------------------------------------------- | -------- | --------- | -| `FEISHU_APP_ID` | 飞书应用 ID | YES | | -| `FEISHU_APP_SECRET` | 飞书应用 Secret | YES | | -| `FEISHU_SPACE_ID` | 飞书知识库 ID | YES | | -| `ASSET_BASE_URL` | 资源文件（图片、附件）的 Base URL，通过这个配置配置 img src 的 URL 前缀 | NO | `/assets` | -| `OUTPUT_DIR` | 输出目录 | NO | `./dist` | -| `ROOT_NODE_TOKEN` | 根节点，导出节点以下（不含此节点）的所有内容。 | NO | | +| Name | Description | Required | Default | +| ------------------- | ---------------------------------------------- | -------- | -------- | +| `FEISHU_APP_ID` | 飞书应用 ID | YES | | +| `FEISHU_APP_SECRET` | 飞书应用 Secret | YES | | +| `FEISHU_SPACE_ID` | 飞书知识库 ID | YES | | +| `OUTPUT_DIR` | 输出目录 | NO | `./dist` | +| `ROOT_NODE_TOKEN` | 根节点，导出节点以下（不含此节点）的所有内容。 | NO | | ## Usage diff --git a/example-website/.vitepress/config.mts b/example-website/.vitepress/config.mts index 7f82426..fabfae3 100644 --- a/example-website/.vitepress/config.mts +++ b/example-website/.vitepress/config.mts @@ -1,6 +1,6 @@ import { DefaultTheme, defineConfig } from 'vitepress'; -const docs = require('../docs.json'); +import docs from '../docs.json'; /** * Convert feishu-pages's docs.json into VitePress's sidebar config @@ -12,7 +12,7 @@ const convertDocsToSidebars = (docs: any) => { for (const doc of docs) { let sidebar: DefaultTheme.SidebarItem = { text: doc.title, - link: doc.slug, + link: 'docs/' + doc.slug, }; if (doc.children.length > 0) { sidebar.items = convertDocsToSidebars(doc.children); @@ -23,17 +23,35 @@ const convertDocsToSidebars = (docs: any) => { return sidebars; }; +const docsSidebar = convertDocsToSidebars(docs); + // https://vitepress.dev/reference/site-config export default defineConfig({ - title: 'Feishu Pages Example', + title: 'Feishu Pages', base: '/feishu-pages/', ignoreDeadLinks: true, cleanUrls: true, + srcExclude: ['SUMMARY.md'], themeConfig: { // https://vitepress.dev/reference/default-theme-config - nav: [{ text: 'Home', link: '/' }], + nav: [ + { text: 'Home', link: '/' }, + { + text: 'Releases', + link: 'https://github.com/longbridgeapp/feishu-pages/releases', + }, + { + text: 'GitHub', + link: 'https://github.com/longbridgeapp/feishu-pages', + }, + ], - sidebar: convertDocsToSidebars(docs), + sidebar: [ + { + text: 'Guides', + items: docsSidebar, + }, + ], socialLinks: [ { icon: 'github', link: 'https://github.com/longbridgeapp/feishu-pages' }, diff --git a/example-website/index.md b/example-website/index.md index d520228..bf996f2 100644 --- a/example-website/index.md +++ b/example-website/index.md @@ -1,10 +1,9 @@ --- -title: Home -navbar: true +title: Feishu Pages --- -# Feishu Pages - Example +# Feishu Pages -Welcome to Feishu Pages Example Site. +> Generate Feishu Wiki into a Markdown for work with Static Page Generators. -This site is generated from Feishu Wiki. +本网站左侧的文档中列表，均来自飞书知识库，通过 [Feishu Pages](https://github.com/longbridgeapp/feishu-pages) 导出为 Markdown 文件，并通过 [VitePress](https://vitepress.dev) + GitHub Pages 生成的静态网站。 diff --git a/feishu-pages/.env.default b/feishu-pages/.env.default index 72a6812..699adaf 100644 --- a/feishu-pages/.env.default +++ b/feishu-pages/.env.default @@ -2,6 +2,5 @@ FEISHU_APP_ID= FEISHU_APP_SECRET= FEISHU_SPACE_ID= FEISHU_LOG_LEVEL=1 -ASSET_BASE_URL=/assets OUTPUT_DIR=./dist ROOT_NODE_TOKEN= \ No newline at end of file diff --git a/feishu-pages/package.json b/feishu-pages/package.json index 347c890..6f91a3e 100644 --- a/feishu-pages/package.json +++ b/feishu-pages/package.json @@ -24,7 +24,8 @@ "@types/node": "^20.5.7", "axios": "^1.5.0", "dotenv": "^16.3.1", - "feishu-docx": "^0.1.2", + "feishu-docx": "^0.2.0", + "mime-types": "^2.1.35", "typescript": "^5.2.2" }, "devDependencies": { @@ -32,4 +33,4 @@ "jest": "^29.6.4", "ts-jest": "^29.1.1" } -} \ No newline at end of file +} diff --git a/feishu-pages/src/doc.ts b/feishu-pages/src/doc.ts index bcba817..627f7c0 100644 --- a/feishu-pages/src/doc.ts +++ b/feishu-pages/src/doc.ts @@ -28,11 +28,11 @@ export const fetchDocBody = async (document_id: string) => { const render = new MarkdownRenderer(doc as any); const content = render.parse(); - const imageTokens = render.imageTokens; + const fileTokens = render.fileTokens; return { content, - imageTokens, + fileTokens, }; }; diff --git a/feishu-pages/src/feishu.ts b/feishu-pages/src/feishu.ts index 04c5bab..5d43c8e 100644 --- a/feishu-pages/src/feishu.ts +++ b/feishu-pages/src/feishu.ts @@ -3,9 +3,19 @@ import { Client } from '@larksuiteoapi/node-sdk'; import axios from 'axios'; import 'dotenv/config'; import fs from 'fs'; +import mime from 'mime-types'; import path from 'path'; import { humanizeFileSize } from './utils'; +export const OUTPUT_DIR: string = path.resolve( + process.env.OUTPUT_DIR || './dist' +); +export const DOCS_DIR: string = path.join(OUTPUT_DIR, 'docs'); +export const ROOT_NODE_TOKEN: string = process.env.ROOT_NODE_TOKEN || ''; +export const CACHE_DIR = path.resolve( + process.env.CACHE_DIR || path.join(OUTPUT_DIR, '.cache') +); + const feishuConfig = { endpoint: 'https://open.feishu.cn', /** @@ -193,50 +203,59 @@ export const feishuFetch = async (method, path, payload): Promise => { * @param localPath * @returns */ -export const feishuDownload = async ( - fileToken: string, - urlPrefix: string, - localPath: string -) => { - const dir = path.dirname(localPath); - fs.mkdirSync(dir, { recursive: true }); - - // trim urlPrefix last / - urlPrefix = urlPrefix.replace(/\/$/, ''); - const result = urlPrefix + '/' + fileToken; - - if (fs.existsSync(localPath)) { - console.info(' -> Skip exist:', fileToken); - return result; +export const feishuDownload = async (fileToken: string, localPath: string) => { + const cacheFilePath = path.join(CACHE_DIR, fileToken); + const cacheFileMetaPath = path.join(CACHE_DIR, `${fileToken}.headers.json`); + fs.mkdirSync(CACHE_DIR, { recursive: true }); + + let res: any = {}; + if (fs.existsSync(cacheFilePath) && fs.existsSync(cacheFileMetaPath)) { + res.data = fs.readFileSync(cacheFilePath); + res.headers = JSON.parse(fs.readFileSync(cacheFileMetaPath, 'utf-8')); + console.info(' -> Cache hit:', fileToken); + } else { + console.info('Download file', fileToken, '...'); + const res: any = await axios + .get( + `${feishuConfig.endpoint}/open-apis/drive/v1/medias/${fileToken}/download`, + { + responseType: 'arraybuffer', + headers: { + Authorization: `Bearer ${feishuConfig.tenantAccessToken}`, + 'User-Agent': 'feishu-pages', + }, + } + ) + .then((res) => { + // Write cache info + fs.writeFileSync(cacheFilePath, res.data); + fs.writeFileSync(cacheFileMetaPath, JSON.stringify(res.headers)); + return res; + }) + .catch((err) => { + const { message } = err; + console.error(' -> Failed to download image:', fileToken, message); + }); } - console.info('Download file', fileToken, '...'); - const res: any = await axios - .get( - `${feishuConfig.endpoint}/open-apis/drive/v1/medias/${fileToken}/download`, - { - responseType: 'arraybuffer', - headers: { - Authorization: `Bearer ${feishuConfig.tenantAccessToken}`, - 'User-Agent': 'feishu-pages', - }, - } - ) - .catch((err) => { - const { message } = err; - console.error(' -> Failed to download image:', fileToken, message); - }); - if (res.data) { + let extension = mime.extension(res.headers['content-type']); console.info( ' =>', res.headers['content-type'], humanizeFileSize(res.data.length) ); - fs.writeFileSync(localPath, res.data); + + if (extension) { + localPath = localPath + '.' + extension; + } + const dir = path.dirname(localPath); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(cacheFilePath, res.data); + fs.copyFileSync(cacheFilePath, localPath); } - return result; + return localPath; }; /** diff --git a/feishu-pages/src/index.ts b/feishu-pages/src/index.ts index 3e48ac6..9eab8b2 100644 --- a/feishu-pages/src/index.ts +++ b/feishu-pages/src/index.ts @@ -1,25 +1,25 @@ #!/usr/bin/env node +import { FileToken } from 'feishu-docx'; import fs from 'fs'; import path from 'path'; import { fetchDocBody, generateFileMeta } from './doc'; -import { feishuConfig, feishuDownload, fetchTenantAccessToken } from './feishu'; +import { + DOCS_DIR, + OUTPUT_DIR, + ROOT_NODE_TOKEN, + feishuConfig, + feishuDownload, + fetchTenantAccessToken, +} from './feishu'; import { FileDoc, generateSummary, prepareDocSlugs } from './summary'; import { humanizeFileSize } from './utils'; import { fetchAllDocs } from './wiki'; -const OUTPUT_DIR: string = path.resolve(process.env.OUTPUT_DIR || './dist'); -const ASSET_BASE_URL: string = process.env.ASSET_BASE_URL || '/assets'; - -const ASSET_DIR: string = path.join(OUTPUT_DIR, 'assets'); -const DOCS_DIR: string = path.join(OUTPUT_DIR, 'docs'); -const ROOT_NODE_TOKEN: string = process.env.ROOT_NODE_TOKEN || ''; - // App entry (async () => { await fetchTenantAccessToken(); console.info('OUTPUT_DIR:', OUTPUT_DIR); - console.info('ASSET_BASE_URL:', ASSET_BASE_URL); console.info('FEISHU_APP_ID:', feishuConfig.appId); console.info('FEISHU_SPACE_ID:', feishuConfig.spaceId); console.info('ROOT_NODE_TOKEN:', ROOT_NODE_TOKEN); @@ -60,8 +60,9 @@ const fetchDocAndWriteFile = async (outputDir: string, docs: FileDoc[]) => { let out = ''; out += meta + '\n\n'; - let { content, imageTokens } = await fetchDocBody(doc.obj_token); - content = await downloadFiles(content, imageTokens); + let { content, fileTokens } = await fetchDocBody(doc.obj_token); + + content = await downloadFiles(content, fileTokens, folder); out += content; @@ -77,16 +78,20 @@ const fetchDocAndWriteFile = async (outputDir: string, docs: FileDoc[]) => { } }; -const downloadFiles = async (content: string, imageTokens: string[]) => { - for (const imageToken of imageTokens) { - const imagePath = await feishuDownload( - imageToken, - ASSET_BASE_URL, - path.join(ASSET_DIR, imageToken) +const downloadFiles = async ( + content: string, + fileTokens: Record, + docFolder: string +) => { + for (const fileToken in fileTokens) { + const filePath = await feishuDownload( + fileToken, + path.join(path.join(docFolder, 'assets'), fileToken) ); + const extension = path.extname(filePath); - const re = new RegExp(`${imageToken}`, 'gm'); - content = content.replace(re, imagePath); + const re = new RegExp(`${fileToken}`, 'gm'); + content = content.replace(re, './assets/' + fileToken + extension); } return content; diff --git a/package.json b/package.json index 15bfa7b..64394b4 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,8 @@ "scripts": { "build": "yarn workspace feishu-docx build", "test": "yarn workspace feishu-docx test && yarn workspace feishu-pages test", - "dev": "yarn workspace feishu-pages dev" + "dev": "yarn workspace feishu-pages dev", + "clean": "rm -Rf dist/" }, "devDependencies": { "autocorrect-node": "^2.8.4" diff --git a/yarn.lock b/yarn.lock index 9d65885..7de3a78 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1632,6 +1632,17 @@ fb-watchman@^2.0.0: dependencies: bser "2.1.1" +feishu-docx@^0.1.3: + version "0.1.4" + resolved "https://registry.yarnpkg.com/feishu-docx/-/feishu-docx-0.1.4.tgz#53e506ec7d8acc8641522b55c9e8625d2fff0648" + integrity sha512-yzp/Lim9Qgg0WbaEbTTaKuYxNyPTPVCwe5KruybMM31psQbqqkl4jwVTpUYrkkDQHFTHMWHtk4hzfuAw9zEjsw== + dependencies: + "@types/node" "^20.5.7" + dotenv "^16.3.1" + feishu-docx "^0.1.3" + jsdom "^22.1.0" + typescript "^5.2.2" + fill-range@^7.0.1: version "7.0.1" resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-7.0.1.tgz#1919a6a7c75fe38b2c7c77e5198535da9acdda40" @@ -2440,7 +2451,7 @@ mime-db@1.52.0: resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.52.0.tgz#bbabcdc02859f4987301c856e3387ce5ec43bf70" integrity sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg== -mime-types@^2.1.12: +mime-types@^2.1.12, mime-types@^2.1.35: version "2.1.35" resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.35.tgz#381a871b62a734450660ae3deee44813f70d959a" integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==