rank download

simon300000 · Sep 7, 2023 · ed0e221 · ed0e221
1 parent 091229c
commit ed0e221
Show file tree

Hide file tree

Showing 4 changed files with 57 additions and 52 deletions.
diff --git a/api/api.ts b/api/api.ts
@@ -199,12 +199,12 @@ router.get('/playerDiffRank', async ctx => {
 })
 
 router.post('/dispatch', ctx => {
-  ctx.body = { url: dispatch() }
+  ctx.body = { key: dispatch() }
 })
 
 router.post('/receipt', koaBody(), ctx => {
-  const { url, data } = ctx.request.body
-  ctx.body = { result: receipt(url, JSON.parse(data)) }
+  const { key, rank } = ctx.request.body
+  ctx.body = { result: receipt(key, rank) }
 })
 
 app.use(router.routes())

diff --git a/api/common.ts b/api/common.ts
@@ -5,6 +5,11 @@ import { PlayerValue } from './type.js'
 type SearchType = AbstractSublevel<any, any, string, string>
 type PlayerType = AbstractSublevel<any, any, string, PlayerValue>
 
+export const platforms = {
+  mobile: 'leaderboard',
+  pc: 'pcleaderboard'
+} as const
+
 export const wait = (ms: number): Promise<void> => new Promise(resolve => setTimeout(resolve, ms))
 
 export const resultWithHistory = <U extends { user: { user_id: string } }, T extends U,>({ result, current }: { result: T[], current: U[] }): U[] => {

diff --git a/api/dispatcher.ts b/api/dispatcher.ts
@@ -1,30 +1,44 @@
-import { Agent, fetch as f, setGlobalDispatcher } from 'undici'
-
-import { wait } from './common.js'
-
-setGlobalDispatcher(new Agent({ connect: { timeout: 120_000 } }))
+import { platforms, wait } from './common.js'
+import { APIResult, RankKey, RawAPI } from './type.js'
 
 type Job = {
-  url: string
-  p: (data: unknown) => void
+  key: RankKey
+  p: (data: APIResult[]) => void
 }
 
 let jobs: Job[] = []
 
-export const fetch = async (url: string) => {
-  if (url.includes('music_tag')) {
-    return f(url).then(res => res.json())
-  }
-  const promise = new Promise<unknown>(resolve => jobs.push({ url, p: resolve }))
+export const downloadCore = async (key: RankKey) => {
+  const promise = new Promise<APIResult[]>(resolve => jobs.push({ key, p: resolve }))
   return promise
 }
 
-export const dispatch = () => jobs[Math.floor(Math.random() * jobs.length)]?.url
+const down = async <T>(url: string) => {
+  const result = await fetch(url).then(w => w.json())
+  return result as T
+}
 
-export const receipt = (url: string, data: any) => {
-  if (data.code === 0) {
+export const download = async ({ uid, difficulty, platform }: RankKey) => {
+  const api = platforms[platform]
+  const { result: firstPage, total, code, msg } = await down<RawAPI>(`https://prpr-muse-dash.peropero.net/musedash/v1/${api}/top?music_uid=${uid}&music_difficulty=${difficulty + 1}&limit=100&offset=0&version=2.11.0&platform=musedash.moe`) //platform=ios_overseas
+  if (code !== 0) {
+    console.error(`Error: ${msg}`)
+    throw new Error(msg)
+  }
+  const pageNumber = Math.max(Math.ceil(total / 100) - 1, 0)
+  const urls = Array(pageNumber).fill(undefined).map((_, i) => i + 1).map(i => i * 100 - 1).map(limit => `https://prpr-muse-dash.peropero.net/musedash/v1/${api}/top?music_uid=${uid}&music_difficulty=${difficulty + 1}&limit=${limit}&offset=1&version=2.11.0&platform=musedash.moe`)
+  return [firstPage, ...await Promise.all(urls.map(url => down<RawAPI>(url).then(({ result }) => result)))]
+    .flat()
+    .filter(r => r?.play?.score != undefined && r?.user?.user_id != undefined)
+    .filter(r => r.play.acc <= 100)
+}
+
+export const dispatch = () => jobs[Math.floor(Math.random() * jobs.length)]?.key
+
+export const receipt = (key: RankKey, data: APIResult[]) => {
+  if (Array.isArray(data)) {
     jobs = jobs.filter(job => {
-      if (job.url === url) {
+      if (job.key === key) {
         job.p(data)
         return false
       }
@@ -40,12 +54,13 @@ export const receipt = (url: string, data: any) => {
 
 (async () => {
   while (true) {
-    await wait(1000 * 10)
-    const url = dispatch()
-    if (url) {
-      const res = await f(url)
-      const data = await res.json() as any
-      receipt(url, data)
+    const key = dispatch()
+    if (key) {
+      const data = await download(key).catch(() => { })
+      if (data) {
+        receipt(key, data)
+      }
     }
+    await wait(1000 * 10)
   }
 })()
diff --git a/api/spider.ts b/api/spider.ts
@@ -1,12 +1,15 @@
-import { fetch } from './dispatcher.js'
+import { Agent, setGlobalDispatcher } from 'undici'
 
-import { MusicData, MusicCore, PlayerValue, RawAPI, RankKey, MusicTagList, genKey } from './type.js'
+
+import { downloadCore } from './dispatcher.js'
+
+import { MusicData, MusicCore, PlayerValue, RankKey, MusicTagList, genKey } from './type.js'
 import { rank, player, search, rankUpdateTime, playerUpdateTime, putTag, checkNewSong, isNewSong, saveRaw, playerDataOld, updatePlayerData, setPlayerNumer } from './database.js'
 import { albums, AvailableLocales, musics } from './albumParser.js'
 
 import { log, error, reloadAlbums } from './api.js'
 
-import { resultWithHistory, wait } from './common.js'
+import { platforms, resultWithHistory, wait } from './common.js'
 
 import { diffdiff, diffPlayer } from './diffdiff.js'
 
@@ -18,10 +21,7 @@ let spiders = SPIDER_PARALLEL
 
 let musicList: MusicData[] = []
 
-const platforms = {
-  mobile: 'leaderboard',
-  pc: 'pcleaderboard'
-} as const
+setGlobalDispatcher(new Agent({ connect: { timeout: 120_000 } }))
 
 const sumMutexMap = new Map<string, Promise<void>>()
 
@@ -37,27 +37,12 @@ const sumLock = ({ uid, difficulty, platform }: RankKey) => {
 }
 
 const down = async <T>(url: string) => {
-  const result = await fetch(url)
+  const result = await fetch(url).then(w => w.json())
   return result as T
 }
 
 const downloadTag = () => down<MusicTagList>('https://prpr-muse-dash.peropero.net/musedash/v1/music_tag?platform=pc')
 
-const downloadCore = async ({ uid, difficulty, platform }: RankKey) => {
-  const api = platforms[platform]
-  const { result: firstPage, total, code, msg } = await down<RawAPI>(`https://prpr-muse-dash.peropero.net/musedash/v1/${api}/top?music_uid=${uid}&music_difficulty=${difficulty + 1}&limit=100&offset=0&version=2.11.0&platform=musedash.moe`) //platform=ios_overseas
-  if (code !== 0) {
-    console.error(`Error: ${msg}`)
-    throw new Error(msg)
-  }
-  const pageNumber = Math.max(Math.ceil(total / 100) - 1, 0)
-  const urls = Array(pageNumber).fill(undefined).map((_, i) => i + 1).map(i => i * 100 - 1).map(limit => `https://prpr-muse-dash.peropero.net/musedash/v1/${api}/top?music_uid=${uid}&music_difficulty=${difficulty + 1}&limit=${limit}&offset=1&version=2.11.0&platform=musedash.moe`)
-  return [firstPage, ...await Promise.all(urls.map(url => down<RawAPI>(url).then(({ result }) => result)))]
-    .flat()
-    .filter(r => r?.play?.score != undefined && r?.user?.user_id != undefined)
-    .filter(r => r.play.acc <= 100)
-}
-
 const toSum = ({ uid, difficulty }: RankKey) => !Object.keys(platforms)
   .map(otherPlatform => genKey({ uid, difficulty, platform: otherPlatform }))
   .map(k => waits.get(k))
@@ -429,10 +414,10 @@ const mal = async (musicData: MusicData[]) => {
   const players = await analyze([...waits.values()].map(({ core }) => core))
   await makeSearch(players)
   log('Search Cached')
-  // await diffdiff(musicData)
-  // log('Difficulty ranked')
-  // await diffPlayer()
-  // log('Players Analyzed')
+  await diffdiff(musicData)
+  log('Difficulty ranked')
+  await diffPlayer()
+  log('Players Analyzed')
   await reloadAlbums()
   await updatePlayerData()
 }