diff --git a/api/api.ts b/api/api.ts index 14c79fc0..6b8f9852 100644 --- a/api/api.ts +++ b/api/api.ts @@ -199,12 +199,12 @@ router.get('/playerDiffRank', async ctx => { }) router.post('/dispatch', ctx => { - ctx.body = { url: dispatch() } + ctx.body = { key: dispatch() } }) router.post('/receipt', koaBody(), ctx => { - const { url, data } = ctx.request.body - ctx.body = { result: receipt(url, JSON.parse(data)) } + const { key, rank } = ctx.request.body + ctx.body = { result: receipt(key, rank) } }) app.use(router.routes()) diff --git a/api/common.ts b/api/common.ts index d3598fb5..fb6b75e7 100644 --- a/api/common.ts +++ b/api/common.ts @@ -5,6 +5,11 @@ import { PlayerValue } from './type.js' type SearchType = AbstractSublevel type PlayerType = AbstractSublevel +export const platforms = { + mobile: 'leaderboard', + pc: 'pcleaderboard' +} as const + export const wait = (ms: number): Promise => new Promise(resolve => setTimeout(resolve, ms)) export const resultWithHistory = ({ result, current }: { result: T[], current: U[] }): U[] => { diff --git a/api/dispatcher.ts b/api/dispatcher.ts index 0d492bcd..2e435d94 100644 --- a/api/dispatcher.ts +++ b/api/dispatcher.ts @@ -1,30 +1,44 @@ -import { Agent, fetch as f, setGlobalDispatcher } from 'undici' - -import { wait } from './common.js' - -setGlobalDispatcher(new Agent({ connect: { timeout: 120_000 } })) +import { platforms, wait } from './common.js' +import { APIResult, RankKey, RawAPI } from './type.js' type Job = { - url: string - p: (data: unknown) => void + key: RankKey + p: (data: APIResult[]) => void } let jobs: Job[] = [] -export const fetch = async (url: string) => { - if (url.includes('music_tag')) { - return f(url).then(res => res.json()) - } - const promise = new Promise(resolve => jobs.push({ url, p: resolve })) +export const downloadCore = async (key: RankKey) => { + const promise = new Promise(resolve => jobs.push({ key, p: resolve })) return promise } -export const dispatch = () => jobs[Math.floor(Math.random() * jobs.length)]?.url +const down = async (url: string) => { + const result = await fetch(url).then(w => w.json()) + return result as T +} -export const receipt = (url: string, data: any) => { - if (data.code === 0) { +export const download = async ({ uid, difficulty, platform }: RankKey) => { + const api = platforms[platform] + const { result: firstPage, total, code, msg } = await down(`https://prpr-muse-dash.peropero.net/musedash/v1/${api}/top?music_uid=${uid}&music_difficulty=${difficulty + 1}&limit=100&offset=0&version=2.11.0&platform=musedash.moe`) //platform=ios_overseas + if (code !== 0) { + console.error(`Error: ${msg}`) + throw new Error(msg) + } + const pageNumber = Math.max(Math.ceil(total / 100) - 1, 0) + const urls = Array(pageNumber).fill(undefined).map((_, i) => i + 1).map(i => i * 100 - 1).map(limit => `https://prpr-muse-dash.peropero.net/musedash/v1/${api}/top?music_uid=${uid}&music_difficulty=${difficulty + 1}&limit=${limit}&offset=1&version=2.11.0&platform=musedash.moe`) + return [firstPage, ...await Promise.all(urls.map(url => down(url).then(({ result }) => result)))] + .flat() + .filter(r => r?.play?.score != undefined && r?.user?.user_id != undefined) + .filter(r => r.play.acc <= 100) +} + +export const dispatch = () => jobs[Math.floor(Math.random() * jobs.length)]?.key + +export const receipt = (key: RankKey, data: APIResult[]) => { + if (Array.isArray(data)) { jobs = jobs.filter(job => { - if (job.url === url) { + if (job.key === key) { job.p(data) return false } @@ -40,12 +54,13 @@ export const receipt = (url: string, data: any) => { (async () => { while (true) { - await wait(1000 * 10) - const url = dispatch() - if (url) { - const res = await f(url) - const data = await res.json() as any - receipt(url, data) + const key = dispatch() + if (key) { + const data = await download(key).catch(() => { }) + if (data) { + receipt(key, data) + } } + await wait(1000 * 10) } })() diff --git a/api/spider.ts b/api/spider.ts index e7fa88b6..5717377c 100644 --- a/api/spider.ts +++ b/api/spider.ts @@ -1,12 +1,15 @@ -import { fetch } from './dispatcher.js' +import { Agent, setGlobalDispatcher } from 'undici' -import { MusicData, MusicCore, PlayerValue, RawAPI, RankKey, MusicTagList, genKey } from './type.js' + +import { downloadCore } from './dispatcher.js' + +import { MusicData, MusicCore, PlayerValue, RankKey, MusicTagList, genKey } from './type.js' import { rank, player, search, rankUpdateTime, playerUpdateTime, putTag, checkNewSong, isNewSong, saveRaw, playerDataOld, updatePlayerData, setPlayerNumer } from './database.js' import { albums, AvailableLocales, musics } from './albumParser.js' import { log, error, reloadAlbums } from './api.js' -import { resultWithHistory, wait } from './common.js' +import { platforms, resultWithHistory, wait } from './common.js' import { diffdiff, diffPlayer } from './diffdiff.js' @@ -18,10 +21,7 @@ let spiders = SPIDER_PARALLEL let musicList: MusicData[] = [] -const platforms = { - mobile: 'leaderboard', - pc: 'pcleaderboard' -} as const +setGlobalDispatcher(new Agent({ connect: { timeout: 120_000 } })) const sumMutexMap = new Map>() @@ -37,27 +37,12 @@ const sumLock = ({ uid, difficulty, platform }: RankKey) => { } const down = async (url: string) => { - const result = await fetch(url) + const result = await fetch(url).then(w => w.json()) return result as T } const downloadTag = () => down('https://prpr-muse-dash.peropero.net/musedash/v1/music_tag?platform=pc') -const downloadCore = async ({ uid, difficulty, platform }: RankKey) => { - const api = platforms[platform] - const { result: firstPage, total, code, msg } = await down(`https://prpr-muse-dash.peropero.net/musedash/v1/${api}/top?music_uid=${uid}&music_difficulty=${difficulty + 1}&limit=100&offset=0&version=2.11.0&platform=musedash.moe`) //platform=ios_overseas - if (code !== 0) { - console.error(`Error: ${msg}`) - throw new Error(msg) - } - const pageNumber = Math.max(Math.ceil(total / 100) - 1, 0) - const urls = Array(pageNumber).fill(undefined).map((_, i) => i + 1).map(i => i * 100 - 1).map(limit => `https://prpr-muse-dash.peropero.net/musedash/v1/${api}/top?music_uid=${uid}&music_difficulty=${difficulty + 1}&limit=${limit}&offset=1&version=2.11.0&platform=musedash.moe`) - return [firstPage, ...await Promise.all(urls.map(url => down(url).then(({ result }) => result)))] - .flat() - .filter(r => r?.play?.score != undefined && r?.user?.user_id != undefined) - .filter(r => r.play.acc <= 100) -} - const toSum = ({ uid, difficulty }: RankKey) => !Object.keys(platforms) .map(otherPlatform => genKey({ uid, difficulty, platform: otherPlatform })) .map(k => waits.get(k)) @@ -429,10 +414,10 @@ const mal = async (musicData: MusicData[]) => { const players = await analyze([...waits.values()].map(({ core }) => core)) await makeSearch(players) log('Search Cached') - // await diffdiff(musicData) - // log('Difficulty ranked') - // await diffPlayer() - // log('Players Analyzed') + await diffdiff(musicData) + log('Difficulty ranked') + await diffPlayer() + log('Players Analyzed') await reloadAlbums() await updatePlayerData() }