Skip to content

Commit

Permalink
feat: mirror edgewebdriver (#676)
Browse files Browse the repository at this point in the history 
closes #594

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->

## Summary by CodeRabbit

- **New Features**
- Introduced the EdgedriverBinary class to enhance the fetching and
management of Edgedriver binaries.
- Added support for the Edgedriver type in our binary management system.
- Updated the binaries configuration to include the new Edgedriver
category, complete with repository and distribution details.

- **Documentation**
- Updated documentation to reflect the addition of the Edgedriver
category and its functionalities.

- **Tests**
- Implemented new tests for the EdgedriverBinary class to ensure
functionality works as expected.

<!-- end of auto-generated comment: release notes by coderabbit.ai -->
  • Loading branch information
@fengmk2
fengmk2 authored May 11, 2024
1 parent 67f1a24 commit 21cbc18
Show file tree
Hide file tree
Showing 5 changed files with 793 additions and 0 deletions.
213 changes: 213 additions & 0 deletions app/common/adapter/binary/EdgedriverBinary.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,213 @@
import path from 'node:path';
import { SingletonProto } from '@eggjs/tegg';
import {
AbstractBinary, FetchResult, BinaryItem, BinaryAdapter,
} from './AbstractBinary';
import { BinaryType } from '../../enum/Binary';

@SingletonProto()
@BinaryAdapter(BinaryType.Edgedriver)
export class EdgedriverBinary extends AbstractBinary {
private dirItems?: {
[key: string]: BinaryItem[];
};

async initFetch() {
this.dirItems = undefined;
}

async #syncDirItems() {
this.dirItems = {};
this.dirItems['/'] = [];
const jsonApiEndpoint = 'https://edgeupdates.microsoft.com/api/products';
const { data, status, headers } = await this.httpclient.request(jsonApiEndpoint, {
dataType: 'json',
timeout: 30000,
followRedirect: true,
gzip: true,
});
if (status !== 200) {
this.logger.warn('[EdgedriverBinary.request:non-200-status] url: %s, status: %s, headers: %j, data: %j',
jsonApiEndpoint, status, headers, data);
return;
}
this.logger.info('[EdgedriverBinary] remote data length: %s', data.length);
// [
// {
// "Product": "Stable",
// "Releases": [
// {
// "ReleaseId": 73376,
// "Platform": "iOS",
// "Architecture": "arm64",
// "CVEs": [],
// "ProductVersion": "124.0.2478.89",
// "Artifacts": [],
// "PublishedTime": "2024-05-07T02:57:00",
// "ExpectedExpiryDate": "2025-05-07T02:57:00"
// },
// {
// "ReleaseId": 73629,
// "Platform": "Windows",
// "Architecture": "x86",
// "CVEs": [
// "CVE-2024-4559",
// "CVE-2024-4671"
// ],
// "ProductVersion": "124.0.2478.97",
// "Artifacts": [
// {
// "ArtifactName": "msi",
// "Location": "https://msedge.sf.dl.delivery.mp.microsoft.com/filestreamingservice/files/aa1c9fe3-bb9c-4a80-9ff7-5c109701fbfe/MicrosoftEdgeEnterpriseX86.msi",
// "Hash": "4CEF7B907D3E2371E953C41190E32C3560CEE7D3F16D7550CA156DC976EBCB80",
// "HashAlgorithm": "SHA256",
// "SizeInBytes": 162029568
// }
// ],
// "PublishedTime": "2024-05-11T06:47:00",
// "ExpectedExpiryDate": "2025-05-10T16:59:00"
// },
// {
// "ReleaseId": 73630,
// "Platform": "Linux",
// "Architecture": "x64",
// "CVEs": [
// "CVE-2024-4559"
// ],
// "ProductVersion": "124.0.2478.97",
// "Artifacts": [
// {
// "ArtifactName": "rpm",
// "Location": "https://packages.microsoft.com/yumrepos/edge/microsoft-edge-stable-124.0.2478.97-1.x86_64.rpm",
// "Hash": "32D9C333544DDD9C56FED54844E89EF00F3E5620942C07B9B68D214016687895",
// "HashAlgorithm": "SHA256",
// "SizeInBytes": 169877932
// },
// {
// "ArtifactName": "deb",
// "Location": "https://packages.microsoft.com/repos/edge/pool/main/m/microsoft-edge-stable/microsoft-edge-stable_124.0.2478.97-1_amd64.deb",
// "Hash": "85D0AD1D63847B3DD54F0F214D18A2B54462BB43291536E773AD1B8B29BBF799",
// "HashAlgorithm": "SHA256",
// "SizeInBytes": 167546042
// }
// ],
// "PublishedTime": "2024-05-10T17:01:00",
// "ExpectedExpiryDate": "2025-05-10T17:01:00"
// },
// {
// "Product": "EdgeUpdate",
// "Releases": [
// {
// "ReleaseId": 73493,
// "Platform": "Windows",
// "Architecture": "x86",
// "CVEs": [],
// "ProductVersion": "1.3.187.37",
// "Artifacts": [
// {
// "ArtifactName": "exe",
// "Location": "https://msedge.sf.dl.delivery.mp.microsoft.com/filestreamingservice/files/a2fa84fe-796b-4f80-b1cd-f4d1f5731aa8/MicrosoftEdgeUpdateSetup_X86_1.3.187.37.exe",
// "Hash": "503088D22461FEE5D7B6B011609D73FFD5869D3ACE1DBB0F00F8F3B9D122C514",
// "HashAlgorithm": "SHA256",
// "SizeInBytes": 1622072
// }
// ],
// "PublishedTime": "2024-05-08T05:44:00",
// "ExpectedExpiryDate": "2025-05-08T05:44:00"
// }
// ]
// }
const products = data as {
Product: string;
Releases: {
ReleaseId: number;
Platform: string;
Architecture: string;
CVEs: string[];
ProductVersion: string;
Artifacts: {
ArtifactName: string;
Location: string;
Hash: string;
HashAlgorithm: string;
SizeInBytes: string;
}[];
PublishedTime: string;
ExpectedExpiryDate: string;
}[];
}[];
const existsVersions = new Set<string>();
for (const product of products) {
if (product.Product === 'EdgeUpdate') continue;
for (const release of product.Releases) {
if (!release.Artifacts || release.Artifacts.length === 0) continue;
if (existsVersions.has(release.ProductVersion)) continue;
this.dirItems['/'].push({
name: `${release.ProductVersion}/`,
date: release.PublishedTime,
size: '-',
isDir: true,
url: '',
});
existsVersions.add(release.ProductVersion);
}
}
}

async fetch(dir: string): Promise<FetchResult | undefined> {
if (!this.dirItems) {
await this.#syncDirItems();
}
// fetch root dir
if (dir === '/') {
return { items: this.dirItems![dir], nextParams: null };
}

// fetch sub dir
// /foo/ => foo/
const subDir = dir.substring(1);
// https://msedgewebdriverstorage.blob.core.windows.net/edgewebdriver?prefix=124.0.2478.97/&delimiter=/&maxresults=100&restype=container&comp=list
const url = `https://msedgewebdriverstorage.blob.core.windows.net/edgewebdriver?prefix=${encodeURIComponent(subDir)}&delimiter=/&maxresults=100&restype=container&comp=list`;
const xml = await this.requestXml(url);
return { items: this.#parseItems(xml), nextParams: null };
}

#parseItems(xml: string): BinaryItem[] {
const items: BinaryItem[] = [];
// <Blob><Name>124.0.2478.97/edgedriver_arm64.zip</Name><Url>https://msedgewebdriverstorage.blob.core.windows.net/edgewebdriver/124.0.2478.97/edgedriver_arm64.zip</Url><Properties><Last-Modified>Fri, 10 May 2024 18:35:44 GMT</Last-Modified><Etag>0x8DC712000713C13</Etag><Content-Length>9191362</Content-Length><Content-Type>application/octet-stream</Content-Type><Content-Encoding /><Content-Language /><Content-MD5>1tjPTf5JU6KKB06Qf1JOGw==</Content-MD5><Cache-Control /><BlobType>BlockBlob</BlobType><LeaseStatus>unlocked</LeaseStatus></Properties></Blob>
const fileRe = /<Blob><Name>([^<]+?)<\/Name><Url>([^<]+?)<\/Url><Properties><Last\-Modified>([^<]+?)<\/Last\-Modified><Etag>(?:[^<]+?)<\/Etag><Content\-Length>(\d+)<\/Content\-Length>/g;
const matchItems = xml.matchAll(fileRe);
for (const m of matchItems) {
const fullname = m[1].trim();
// <Blob>
// <Name>124.0.2478.97/edgedriver_arm64.zip</Name>
// <Url>https://msedgewebdriverstorage.blob.core.windows.net/edgewebdriver/124.0.2478.97/edgedriver_arm64.zip</Url>
// <Properties>
// <Last-Modified>Fri, 10 May 2024 18:35:44 GMT</Last-Modified>
// <Etag>0x8DC712000713C13</Etag>
// <Content-Length>9191362</Content-Length>
// <Content-Type>application/octet-stream</Content-Type>
// <Content-Encoding/>
// <Content-Language/>
// <Content-MD5>1tjPTf5JU6KKB06Qf1JOGw==</Content-MD5>
// <Cache-Control/>
// <BlobType>BlockBlob</BlobType>
// <LeaseStatus>unlocked</LeaseStatus>
// </Properties>
// </Blob>
// ignore size = 0 dir
const name = path.basename(fullname);
const url = m[2].trim();
const date = m[3].trim();
const size = parseInt(m[4].trim());
items.push({
name,
isDir: false,
url,
size,
date,
});
}
return items;
}
}
1 change: 1 addition & 0 deletions app/common/enum/Binary.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,5 @@ export enum BinaryType {
Prisma = 'prisma',
Sqlcipher = 'sqlcipher',
ChromeForTesting = 'chromeForTesting',
Edgedriver = 'edgedriver',
}
7 changes: 7 additions & 0 deletions config/binaries.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -911,6 +911,13 @@ const binaries = {
repo: 'oven-sh/bun',
distUrl: 'https://github.com/oven-sh/bun/releases',
},
edgedriver: {
category: 'edgedriver',
description: 'Microsofts\' EdgeDriver',
type: BinaryType.Edgedriver,
repo: 'webdriverio-community/node-edgedriver',
distUrl: 'https://developer.microsoft.com/zh-cn/microsoft-edge/tools/webdriver/',
},
} as const;

export type BinaryName = keyof typeof binaries;
Expand Down
76 changes: 76 additions & 0 deletions test/common/adapter/binary/EdgedriverBinary.test.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
import assert from 'assert';
import { app } from 'egg-mock/bootstrap';
import { EdgedriverBinary } from '../../../../app/common/adapter/binary/EdgedriverBinary';
import { TestUtil } from '../../../../test/TestUtil';

describe('test/common/adapter/binary/EdgedriverBinary.test.ts', () => {
let binary: EdgedriverBinary;
beforeEach(async () => {
binary = await app.getEggObject(EdgedriverBinary);
});

describe('fetch()', () => {
it('should work', async () => {
app.mockHttpclient('https://edgeupdates.microsoft.com/api/products', 'GET', {
data: await TestUtil.readFixturesFile('edgeupdates.json'),
persist: false,
});
let result = await binary.fetch('/');
assert.deepEqual(result, {
items: [
{
name: '124.0.2478.97/',
date: '2024-05-11T06:47:00',
size: '-',
isDir: true,
url: '',
},
{
name: '125.0.2535.37/',
date: '2024-05-10T18:52:00',
size: '-',
isDir: true,
url: '',
},
{
name: '126.0.2566.1/',
date: '2024-05-07T17:30:00',
size: '-',
isDir: true,
url: '',
},
{
name: '126.0.2578.0/',
date: '2024-05-10T16:33:00',
size: '-',
isDir: true,
url: '',
},
],
nextParams: null,
});

const latestVersion = result!.items![result!.items.length - 1].name;
assert(latestVersion);
assert.equal(latestVersion, '126.0.2578.0/');
result = await binary.fetch(`/${latestVersion}`);
const items = result!.items;
assert(items.length >= 3);
for (const item of items) {
// {
// name: 'edgedriver_win64.zip',
// isDir: false,
// url: 'https://msedgewebdriverstorage.blob.core.windows.net/edgewebdriver/126.0.2578.0/edgedriver_win64.zip',
// size: 9564395,
// date: 'Fri, 10 May 2024 17:04:10 GMT'
// }
assert.equal(item.isDir, false);
assert.match(item.name, /^edgedriver_\w+.zip$/);
assert.match(item.url, /^https:\/\//);
assert(typeof item.size === 'number');
assert(item.size > 0);
assert(item.date);
}
});
});
});
Loading

0 comments on commit 21cbc18

Please sign in to comment.