Improve assets download, store into save folder as doc file and add f…

…ile extension.
longbridgeapp · Sep 1, 2023 · fce7dc0 · fce7dc0
1 parent 901b3ba
commit fce7dc0
Show file tree

Hide file tree

Showing 12 changed files with 133 additions and 82 deletions.
diff --git a/.github/workflows/feishu-pages.yml b/.github/workflows/feishu-pages.yml
@@ -19,13 +19,11 @@ jobs:
           FEISHU_APP_SECRET: ${{ secrets.FEISHU_APP_SECRET }}
           FEISHU_SPACE_ID: '7273324757679325186'
           OUTPUT_DIR: ./dist
-          ASSET_BASE_URL: 'https://longbridgeapp.github.io/feishu-pages/assets'
         uses: longbridgeapp/feishu-pages@main
       - name: Build Pages
         run: |
           mkdir -p example-website/public/assets/
-          cp -R dist/docs/* example-website/
-          cp -R dist/assets/* example-website/public/assets/
+          cp -R dist/docs example-website/
           cp dist/docs.json example-website/
           yarn
           yarn workspace example-website build

diff --git a/.gitignore b/.gitignore
@@ -3,4 +3,5 @@ dist/
 out/
 node_modules/
 .DS_Store
-**/*/.vitepress/cache
+**/*/.vitepress/cache
+.cache/
diff --git a/README.md b/README.md
@@ -31,14 +31,13 @@ yarn add feishu-pages
 
 ## Configuration
 
-| Name                | Description                                                             | Required | Default   |
-| ------------------- | ----------------------------------------------------------------------- | -------- | --------- |
-| `FEISHU_APP_ID`     | 飞书应用 ID                                                             | YES      |           |
-| `FEISHU_APP_SECRET` | 飞书应用 Secret                                                         | YES      |           |
-| `FEISHU_SPACE_ID`   | 飞书知识库 ID                                                           | YES      |           |
-| `ASSET_BASE_URL`    | 资源文件（图片、附件）的 Base URL，通过这个配置配置 img src 的 URL 前缀 | NO       | `/assets` |
-| `OUTPUT_DIR`        | 输出目录                                                                | NO       | `./dist`  |
-| `ROOT_NODE_TOKEN`   | 根节点，导出节点以下（不含此节点）的所有内容。                          | NO       |           |
+| Name                | Description                                    | Required | Default  |
+| ------------------- | ---------------------------------------------- | -------- | -------- |
+| `FEISHU_APP_ID`     | 飞书应用 ID                                    | YES      |          |
+| `FEISHU_APP_SECRET` | 飞书应用 Secret                                | YES      |          |
+| `FEISHU_SPACE_ID`   | 飞书知识库 ID                                  | YES      |          |
+| `OUTPUT_DIR`        | 输出目录                                       | NO       | `./dist` |
+| `ROOT_NODE_TOKEN`   | 根节点，导出节点以下（不含此节点）的所有内容。 | NO       |          |
 
 ## Usage
 

diff --git a/example-website/.vitepress/config.mts b/example-website/.vitepress/config.mts
@@ -1,6 +1,6 @@
 import { DefaultTheme, defineConfig } from 'vitepress';
 
-const docs = require('../docs.json');
+import docs from '../docs.json';
 
 /**
  * Convert feishu-pages's docs.json into VitePress's sidebar config
@@ -12,7 +12,7 @@ const convertDocsToSidebars = (docs: any) => {
   for (const doc of docs) {
     let sidebar: DefaultTheme.SidebarItem = {
       text: doc.title,
-      link: doc.slug,
+      link: 'docs/' + doc.slug,
     };
     if (doc.children.length > 0) {
       sidebar.items = convertDocsToSidebars(doc.children);
@@ -23,17 +23,35 @@ const convertDocsToSidebars = (docs: any) => {
   return sidebars;
 };
 
+const docsSidebar = convertDocsToSidebars(docs);
+
 // https://vitepress.dev/reference/site-config
 export default defineConfig({
-  title: 'Feishu Pages Example',
+  title: 'Feishu Pages',
   base: '/feishu-pages/',
   ignoreDeadLinks: true,
   cleanUrls: true,
+  srcExclude: ['SUMMARY.md'],
   themeConfig: {
     // https://vitepress.dev/reference/default-theme-config
-    nav: [{ text: 'Home', link: '/' }],
+    nav: [
+      { text: 'Home', link: '/' },
+      {
+        text: 'Releases',
+        link: 'https://github.com/longbridgeapp/feishu-pages/releases',
+      },
+      {
+        text: 'GitHub',
+        link: 'https://github.com/longbridgeapp/feishu-pages',
+      },
+    ],
 
-    sidebar: convertDocsToSidebars(docs),
+    sidebar: [
+      {
+        text: 'Guides',
+        items: docsSidebar,
+      },
+    ],
 
     socialLinks: [
       { icon: 'github', link: 'https://github.com/longbridgeapp/feishu-pages' },

diff --git a/example-website/index.md b/example-website/index.md
@@ -1,10 +1,9 @@
 ---
-title: Home
-navbar: true
+title: Feishu Pages
 ---
 
-# Feishu Pages - Example
+# Feishu Pages
 
-Welcome to Feishu Pages Example Site.
+> Generate Feishu Wiki into a Markdown for work with Static Page Generators.
 
-This site is generated from Feishu Wiki.
+本网站左侧的文档中列表，均来自飞书知识库，通过 [Feishu Pages](https://github.com/longbridgeapp/feishu-pages) 导出为 Markdown 文件，并通过 [VitePress](https://vitepress.dev) + GitHub Pages 生成的静态网站。
diff --git a/feishu-pages/.env.default b/feishu-pages/.env.default
@@ -2,6 +2,5 @@ FEISHU_APP_ID=
 FEISHU_APP_SECRET=
 FEISHU_SPACE_ID=
 FEISHU_LOG_LEVEL=1
-ASSET_BASE_URL=/assets
 OUTPUT_DIR=./dist
 ROOT_NODE_TOKEN=
diff --git a/feishu-pages/package.json b/feishu-pages/package.json
@@ -24,12 +24,13 @@
     "@types/node": "^20.5.7",
     "axios": "^1.5.0",
     "dotenv": "^16.3.1",
-    "feishu-docx": "^0.1.2",
+    "feishu-docx": "^0.2.0",
+    "mime-types": "^2.1.35",
     "typescript": "^5.2.2"
   },
   "devDependencies": {
     "@jest/globals": "^29.6.4",
     "jest": "^29.6.4",
     "ts-jest": "^29.1.1"
   }
-}
+}
diff --git a/feishu-pages/src/doc.ts b/feishu-pages/src/doc.ts
@@ -28,11 +28,11 @@ export const fetchDocBody = async (document_id: string) => {
 
   const render = new MarkdownRenderer(doc as any);
   const content = render.parse();
-  const imageTokens = render.imageTokens;
+  const fileTokens = render.fileTokens;
 
   return {
     content,
-    imageTokens,
+    fileTokens,
   };
 };
 

diff --git a/feishu-pages/src/feishu.ts b/feishu-pages/src/feishu.ts
@@ -3,9 +3,19 @@ import { Client } from '@larksuiteoapi/node-sdk';
 import axios from 'axios';
 import 'dotenv/config';
 import fs from 'fs';
+import mime from 'mime-types';
 import path from 'path';
 import { humanizeFileSize } from './utils';
 
+export const OUTPUT_DIR: string = path.resolve(
+  process.env.OUTPUT_DIR || './dist'
+);
+export const DOCS_DIR: string = path.join(OUTPUT_DIR, 'docs');
+export const ROOT_NODE_TOKEN: string = process.env.ROOT_NODE_TOKEN || '';
+export const CACHE_DIR = path.resolve(
+  process.env.CACHE_DIR || path.join(OUTPUT_DIR, '.cache')
+);
+
 const feishuConfig = {
   endpoint: 'https://open.feishu.cn',
   /**
@@ -193,50 +203,59 @@ export const feishuFetch = async (method, path, payload): Promise<any> => {
  * @param localPath
  * @returns
  */
-export const feishuDownload = async (
-  fileToken: string,
-  urlPrefix: string,
-  localPath: string
-) => {
-  const dir = path.dirname(localPath);
-  fs.mkdirSync(dir, { recursive: true });
-
-  // trim urlPrefix last /
-  urlPrefix = urlPrefix.replace(/\/$/, '');
-  const result = urlPrefix + '/' + fileToken;
-
-  if (fs.existsSync(localPath)) {
-    console.info(' -> Skip exist:', fileToken);
-    return result;
+export const feishuDownload = async (fileToken: string, localPath: string) => {
+  const cacheFilePath = path.join(CACHE_DIR, fileToken);
+  const cacheFileMetaPath = path.join(CACHE_DIR, `${fileToken}.headers.json`);
+  fs.mkdirSync(CACHE_DIR, { recursive: true });
+
+  let res: any = {};
+  if (fs.existsSync(cacheFilePath) && fs.existsSync(cacheFileMetaPath)) {
+    res.data = fs.readFileSync(cacheFilePath);
+    res.headers = JSON.parse(fs.readFileSync(cacheFileMetaPath, 'utf-8'));
+    console.info(' -> Cache hit:', fileToken);
+  } else {
+    console.info('Download file', fileToken, '...');
+    const res: any = await axios
+      .get(
+        `${feishuConfig.endpoint}/open-apis/drive/v1/medias/${fileToken}/download`,
+        {
+          responseType: 'arraybuffer',
+          headers: {
+            Authorization: `Bearer ${feishuConfig.tenantAccessToken}`,
+            'User-Agent': 'feishu-pages',
+          },
+        }
+      )
+      .then((res) => {
+        // Write cache info
+        fs.writeFileSync(cacheFilePath, res.data);
+        fs.writeFileSync(cacheFileMetaPath, JSON.stringify(res.headers));
+        return res;
+      })
+      .catch((err) => {
+        const { message } = err;
+        console.error(' -> Failed to download image:', fileToken, message);
+      });
   }
 
-  console.info('Download file', fileToken, '...');
-  const res: any = await axios
-    .get(
-      `${feishuConfig.endpoint}/open-apis/drive/v1/medias/${fileToken}/download`,
-      {
-        responseType: 'arraybuffer',
-        headers: {
-          Authorization: `Bearer ${feishuConfig.tenantAccessToken}`,
-          'User-Agent': 'feishu-pages',
-        },
-      }
-    )
-    .catch((err) => {
-      const { message } = err;
-      console.error(' -> Failed to download image:', fileToken, message);
-    });
-
   if (res.data) {
+    let extension = mime.extension(res.headers['content-type']);
     console.info(
       ' =>',
       res.headers['content-type'],
       humanizeFileSize(res.data.length)
     );
-    fs.writeFileSync(localPath, res.data);
+
+    if (extension) {
+      localPath = localPath + '.' + extension;
+    }
+    const dir = path.dirname(localPath);
+    fs.mkdirSync(dir, { recursive: true });
+    fs.writeFileSync(cacheFilePath, res.data);
+    fs.copyFileSync(cacheFilePath, localPath);
   }
 
-  return result;
+  return localPath;
 };
 
 /**

diff --git a/feishu-pages/src/index.ts b/feishu-pages/src/index.ts
@@ -1,25 +1,25 @@
 #!/usr/bin/env node
+import { FileToken } from 'feishu-docx';
 import fs from 'fs';
 import path from 'path';
 import { fetchDocBody, generateFileMeta } from './doc';
-import { feishuConfig, feishuDownload, fetchTenantAccessToken } from './feishu';
+import {
+  DOCS_DIR,
+  OUTPUT_DIR,
+  ROOT_NODE_TOKEN,
+  feishuConfig,
+  feishuDownload,
+  fetchTenantAccessToken,
+} from './feishu';
 import { FileDoc, generateSummary, prepareDocSlugs } from './summary';
 import { humanizeFileSize } from './utils';
 import { fetchAllDocs } from './wiki';
 
-const OUTPUT_DIR: string = path.resolve(process.env.OUTPUT_DIR || './dist');
-const ASSET_BASE_URL: string = process.env.ASSET_BASE_URL || '/assets';
-
-const ASSET_DIR: string = path.join(OUTPUT_DIR, 'assets');
-const DOCS_DIR: string = path.join(OUTPUT_DIR, 'docs');
-const ROOT_NODE_TOKEN: string = process.env.ROOT_NODE_TOKEN || '';
-
 // App entry
 (async () => {
   await fetchTenantAccessToken();
 
   console.info('OUTPUT_DIR:', OUTPUT_DIR);
-  console.info('ASSET_BASE_URL:', ASSET_BASE_URL);
   console.info('FEISHU_APP_ID:', feishuConfig.appId);
   console.info('FEISHU_SPACE_ID:', feishuConfig.spaceId);
   console.info('ROOT_NODE_TOKEN:', ROOT_NODE_TOKEN);
@@ -60,8 +60,9 @@ const fetchDocAndWriteFile = async (outputDir: string, docs: FileDoc[]) => {
     let out = '';
     out += meta + '\n\n';
 
-    let { content, imageTokens } = await fetchDocBody(doc.obj_token);
-    content = await downloadFiles(content, imageTokens);
+    let { content, fileTokens } = await fetchDocBody(doc.obj_token);
+
+    content = await downloadFiles(content, fileTokens, folder);
 
     out += content;
 
@@ -77,16 +78,20 @@ const fetchDocAndWriteFile = async (outputDir: string, docs: FileDoc[]) => {
   }
 };
 
-const downloadFiles = async (content: string, imageTokens: string[]) => {
-  for (const imageToken of imageTokens) {
-    const imagePath = await feishuDownload(
-      imageToken,
-      ASSET_BASE_URL,
-      path.join(ASSET_DIR, imageToken)
+const downloadFiles = async (
+  content: string,
+  fileTokens: Record<string, FileToken>,
+  docFolder: string
+) => {
+  for (const fileToken in fileTokens) {
+    const filePath = await feishuDownload(
+      fileToken,
+      path.join(path.join(docFolder, 'assets'), fileToken)
     );
+    const extension = path.extname(filePath);
 
-    const re = new RegExp(`${imageToken}`, 'gm');
-    content = content.replace(re, imagePath);
+    const re = new RegExp(`${fileToken}`, 'gm');
+    content = content.replace(re, './assets/' + fileToken + extension);
   }
 
   return content;

diff --git a/package.json b/package.json
@@ -9,7 +9,8 @@
   "scripts": {
     "build": "yarn workspace feishu-docx build",
     "test": "yarn workspace feishu-docx test && yarn workspace feishu-pages test",
-    "dev": "yarn workspace feishu-pages dev"
+    "dev": "yarn workspace feishu-pages dev",
+    "clean": "rm -Rf dist/"
   },
   "devDependencies": {
     "autocorrect-node": "^2.8.4"

diff --git a/yarn.lock b/yarn.lock
@@ -1632,6 +1632,17 @@ fb-watchman@^2.0.0:
   dependencies:
     bser "2.1.1"
 
+feishu-docx@^0.1.3:
+  version "0.1.4"
+  resolved "https://registry.yarnpkg.com/feishu-docx/-/feishu-docx-0.1.4.tgz#53e506ec7d8acc8641522b55c9e8625d2fff0648"
+  integrity sha512-yzp/Lim9Qgg0WbaEbTTaKuYxNyPTPVCwe5KruybMM31psQbqqkl4jwVTpUYrkkDQHFTHMWHtk4hzfuAw9zEjsw==
+  dependencies:
+    "@types/node" "^20.5.7"
+    dotenv "^16.3.1"
+    feishu-docx "^0.1.3"
+    jsdom "^22.1.0"
+    typescript "^5.2.2"
+
 fill-range@^7.0.1:
   version "7.0.1"
   resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-7.0.1.tgz#1919a6a7c75fe38b2c7c77e5198535da9acdda40"
@@ -2440,7 +2451,7 @@ mime-db@1.52.0:
   resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.52.0.tgz#bbabcdc02859f4987301c856e3387ce5ec43bf70"
   integrity sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==
 
-mime-types@^2.1.12:
+mime-types@^2.1.12, mime-types@^2.1.35:
   version "2.1.35"
   resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.35.tgz#381a871b62a734450660ae3deee44813f70d959a"
   integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==