Skip to content

Commit

Permalink
Fix image streaming download sometimes may 0 kb size error.
Browse files Browse the repository at this point in the history
  • Loading branch information
@huacnlee
huacnlee committed Mar 11, 2024
1 parent d071294 commit 5e85446
Show file tree
Hide file tree
Showing 2 changed files with 118 additions and 96 deletions.
138 changes: 78 additions & 60 deletions feishu-pages/src/feishu.ts
View file
Original file line number Diff line number Diff line change
@@ -1,28 +1,28 @@
// node-sdk 使用说明：https://github.com/larksuite/node-sdk/blob/main/README.zh.md
import { Client } from '@larksuiteoapi/node-sdk';
import axios, { AxiosResponse } from 'axios';
import 'dotenv/config';
import fs from 'fs';
import mime from 'mime-types';
import path from 'path';
import { humanizeFileSize } from './utils';
import { Client } from "@larksuiteoapi/node-sdk";
import axios, { AxiosResponse } from "axios";
import "dotenv/config";
import fs from "fs";
import mime from "mime-types";
import path from "path";
import { humanizeFileSize } from "./utils";

export const OUTPUT_DIR: string = path.resolve(
process.env.OUTPUT_DIR || './dist'
process.env.OUTPUT_DIR || "./dist",
);
export const DOCS_DIR: string = path.join(OUTPUT_DIR, 'docs');
export const DOCS_DIR: string = path.join(OUTPUT_DIR, "docs");

let baseUrl = process.env.BASE_URL || process.env.URL_PREFIX || '/';
if (!baseUrl.endsWith('/')) {
baseUrl += '/';
let baseUrl = process.env.BASE_URL || process.env.URL_PREFIX || "/";
if (!baseUrl.endsWith("/")) {
baseUrl += "/";
}
export const BASE_URL: string = baseUrl;
export const ROOT_NODE_TOKEN: string = process.env.ROOT_NODE_TOKEN || '';
export const ROOT_NODE_TOKEN: string = process.env.ROOT_NODE_TOKEN || "";
export const CACHE_DIR = path.resolve(
process.env.CACHE_DIR || path.join(OUTPUT_DIR, '.cache')
process.env.CACHE_DIR || path.join(OUTPUT_DIR, ".cache"),
);
export const TMP_DIR = path.resolve(
process.env.TMP_DIR || path.join(OUTPUT_DIR, '.tmp')
process.env.TMP_DIR || path.join(OUTPUT_DIR, ".tmp"),
);

/**
Expand All @@ -31,14 +31,14 @@ export const TMP_DIR = path.resolve(
* raw: /G5JwdLWUkopngoxfQtIc7EFSnIg
* nested: /slug1/slug2/slug3
*/
export const URL_STYLE = process.env.URL_STYLE || 'nested';
export const URL_STYLE = process.env.URL_STYLE || "nested";
/**
* Skip download assets, for debug speed up.
*/
export const SKIP_ASSETS = process.env.SKIP_ASSETS || false;

const feishuConfig = {
endpoint: process.env.FEISHU_ENDPOINT || 'https://open.feishu.cn',
endpoint: process.env.FEISHU_ENDPOINT || "https://open.feishu.cn",
/**
* App Id of Feishu App
*
Expand Down Expand Up @@ -67,20 +67,20 @@ const feishuConfig = {
* env: `FEISHU_SPACE_ID`
*/
spaceId: process.env.FEISHU_SPACE_ID,
logLevel: process.env.FEISHU_LOG_LEVEL || '2',
logLevel: process.env.FEISHU_LOG_LEVEL || "2",
};

const checkEnv = () => {
if (!feishuConfig.appId) {
throw new Error('FEISHU_APP_ID is required');
throw new Error("FEISHU_APP_ID is required");
}

if (!feishuConfig.appSecret) {
throw new Error('FEISHU_APP_SECRET is required');
throw new Error("FEISHU_APP_SECRET is required");
}

if (!feishuConfig.spaceId) {
throw new Error('FEISHU_SPACE_ID is required');
throw new Error("FEISHU_SPACE_ID is required");
}
};

Expand All @@ -101,16 +101,16 @@ const feishuClient = new Client({
* @returns
*/
export const fetchTenantAccessToken = async () => {
console.log('Fetching tenantAccessToken...');
console.log("Fetching tenantAccessToken...");
const res: Record<string, any> =
await feishuClient.auth.tenantAccessToken.internal({
data: {
app_id: feishuConfig.appId,
app_secret: feishuConfig.appSecret,
},
});
const access_token = res?.tenant_access_token || '';
console.info('TENANT_ACCESS_TOKEN:', maskToken(access_token));
const access_token = res?.tenant_access_token || "";
console.info("TENANT_ACCESS_TOKEN:", maskToken(access_token));
feishuConfig.tenantAccessToken = access_token;
};

Expand All @@ -124,7 +124,7 @@ export const maskToken = (token) => {
const mashLen = len * 0.6;
return (
token.substring(0, len - mashLen + 5) +
'*'.repeat(mashLen) +
"*".repeat(mashLen) +
token.substring(len - 5)
);
};
Expand All @@ -148,7 +148,7 @@ export const requestWait = async (ms?: number) => {
// If overload 100 times/min, wait 1 minute
if (RATE_LIMITS[minuteLockKey] >= 100) {
console.warn(
'[RATE LIMIT] Overload request 100 times/min, wait 1 minute...'
"[RATE LIMIT] Overload request 100 times/min, wait 1 minute...",
);
await await new Promise((resolve) => setTimeout(resolve, 60 * 1000));
RATE_LIMITS[minuteLockKey] = 0;
Expand All @@ -167,12 +167,12 @@ axios.interceptors.response.use(

// Rate Limit code: 99991400, delay to retry
if (data?.code === 99991400) {
const rateLimitResetSeconds = headers['x-ogw-ratelimit-reset'];
const rateLimitResetSeconds = headers["x-ogw-ratelimit-reset"];
console.warn(
'[RATE LIMIT]',
"[RATE LIMIT]",
data.code,
data.msg,
`delay ${rateLimitResetSeconds}s to retry...`
`delay ${rateLimitResetSeconds}s to retry...`,
);

// Delay to retry
Expand All @@ -181,7 +181,7 @@ axios.interceptors.response.use(
}

throw error;
}
},
);

/**
Expand All @@ -195,8 +195,8 @@ export const feishuFetch = async (method, path, payload): Promise<any> => {
const authorization = `Bearer ${feishuConfig.tenantAccessToken}`;
const headers = {
Authorization: authorization,
'Content-Type': 'application/json; charset=utf-8',
'User-Agent': 'feishu-pages',
"Content-Type": "application/json; charset=utf-8",
"User-Agent": "feishu-pages",
};

const url = `${feishuConfig.endpoint}${path}`;
Expand All @@ -211,7 +211,7 @@ export const feishuFetch = async (method, path, payload): Promise<any> => {
.then((res) => res.data);

if (code !== 0) {
console.warn('feishuFetch code:', code, 'msg:', msg);
console.warn("feishuFetch code:", code, "msg:", msg);
return null;
}

Expand All @@ -234,53 +234,62 @@ export const feishuDownload = async (fileToken: string, localPath: string) => {

let res: { data?: fs.ReadStream; headers?: Record<string, any> } = {};
let hasCache = false;
let fileSize = 0;

if (fs.existsSync(cacheFilePath) && fs.existsSync(cacheFileMetaPath)) {
hasCache = true;
res.headers = JSON.parse(fs.readFileSync(cacheFileMetaPath, 'utf-8'));
console.info(' -> Cache hit:', fileToken);
res.headers = JSON.parse(fs.readFileSync(cacheFileMetaPath, "utf-8"));
console.info(" -> Cache hit:", fileToken);
} else {
console.info('Download file', fileToken, '...');
console.info("Downloading file", fileToken, "...");
res = await axios
.get(
`${feishuConfig.endpoint}/open-apis/drive/v1/medias/${fileToken}/download`,
{
responseType: 'stream',
responseType: "stream",
headers: {
Authorization: `Bearer ${feishuConfig.tenantAccessToken}`,
'User-Agent': 'feishu-pages',
"User-Agent": "feishu-pages",
},
}
},
)
.then((res: AxiosResponse) => {
if (res.status !== 200) {
console.error(' -> Failed to download image:', fileToken, res.status);
console.error(
" -> ERROR: Failed to download image:",
fileToken,
res.status,
);
return null;
}

// Write cache info
fs.writeFileSync(cacheFileMetaPath, JSON.stringify(res.headers));
res.data.pipe(fs.createWriteStream(cacheFilePath));

fileSize = res.data.readableLength;

return new Promise((resolve: any) => {
res.data.on('end', () => {
return new Promise((resolve: any, reject: any) => {
const writer = fs.createWriteStream(cacheFilePath);
res.data.pipe(writer);
writer.on("finish", () => {
resolve({
headers: res.headers,
});
});
writer.on("error", (e) => {
reject(e);
});
});
})
.catch((err) => {
const { message } = err;
console.error(' -> Failed to download image:', fileToken, message);
console.error(
" -> ERROR: Failed to download image:",
fileToken,
message,
);
// If status is 403
// https://open.feishu.cn/document/server-docs/docs/drive-v1/faq#6e38a6de
if (message.includes('403')) {
if (message.includes("403")) {
console.error(
`无文件下载权限时接口将返回 403 的 HTTP 状态码。\nhttps://open.feishu.cn/document/server-docs/docs/drive-v1/faq#6e38a6de\nhttps://open.feishu.cn/document/server-docs/docs/drive-v1/download/download`
`无文件下载权限时接口将返回 403 的 HTTP 状态码。\nhttps://open.feishu.cn/document/server-docs/docs/drive-v1/faq#6e38a6de\nhttps://open.feishu.cn/document/server-docs/docs/drive-v1/download/download`,
);
return null;
}
Expand All @@ -291,25 +300,34 @@ export const feishuDownload = async (fileToken: string, localPath: string) => {
return null;
}

let extension = mime.extension(res.headers['content-type']);
// Check cacheFile size, if is 0kb remove it
const stats = fs.statSync(cacheFilePath);
if (stats.size === 0) {
fs.rmSync(cacheFilePath);
console.error(" -> ERROR: Invalid image, size is 0, token:", fileToken);
return null;
}

let extension = mime.extension(res.headers["content-type"]);
let fileSize = res.headers["content-length"];
if (!hasCache) {
console.info(
' =>',
res.headers['content-type'],
humanizeFileSize(fileSize)
" =>",
res.headers["content-type"],
humanizeFileSize(fileSize),
);
}

if (extension) {
localPath = localPath + '.' + extension;
localPath = localPath + "." + extension;
}
const dir = path.dirname(localPath);
fs.mkdirSync(dir, { recursive: true });
if (!hasCache) {
console.info(' -> Writing file:', localPath);
console.info(" -> Writing file:", localPath);
}
if (!fs.existsSync(cacheFilePath)) {
console.warn('file not found,', cacheFilePath, 'may be is download 404');
console.warn("file not found,", cacheFilePath, "may be is download 404");
return null;
}

Expand All @@ -329,9 +347,9 @@ export const feishuDownload = async (fileToken: string, localPath: string) => {
export const feishuFetchWithIterator = async (
method: string,
path: string,
payload: Record<string, any> = {}
payload: Record<string, any> = {},
): Promise<WikiNode[]> => {
let pageToken = '';
let pageToken = "";
let hasMore = true;
let results: any[] = [];

Expand Down Expand Up @@ -380,7 +398,7 @@ export interface WikiNode {
*/
node_token: string;
obj_token: string;
obj_type: 'doc' | 'docx' | 'sheet' | 'file';
obj_type: "doc" | "docx" | "sheet" | "file";
/**
* 父节点 token。若当前节点为一级节点，父节点 token 为空
*/
Expand All @@ -391,7 +409,7 @@ export interface WikiNode {
* origin：实体
* shortcut：快捷方式
*/
node_type: 'origin' | 'shortcut';
node_type: "origin" | "shortcut";
origin_node_token: string;
origin_space_id: string;
has_child: boolean;
Expand Down
Loading

0 comments on commit 5e85446

Please sign in to comment.