Skip to content

Commit

Permalink
Support Modular Auto Generation (#8906)
Browse files Browse the repository at this point in the history 
* Add script for generating modular client

* Update

* wip

* WIP

* wip: remove sync call

* Done: modular client SDK automation

* WIP

* WIP

* minor fixes

* Updated package.json

* Improve log

* Removed former files

* Fixed typo

* Added timeout for runCommand

* Updated

* Improved logs

* Only run node test

* Removed len == 1 array

* Updated package version

* Fixed template not found bug

* Update build script

* Updated package.json

* Fixed sdk type bug

* Fixed no throw issue

* Fix

* Update package.json

* Update lock file

* Update

* a

* a

* Revert "a"

This reverts commit 8d3535c.

* Revert "a"

This reverts commit fcb860a.

* debug

* Revert "debug"

This reverts commit f7d0eb9.

* Remove dup code

* Fix merge conflict

* Update

* Normalize path

* Update apiViewArtifact declaration

* Update path

* Workaround to pass pipeline

* Update mgitignore

* Update api view json

* Remove

* Update logger

* .

* Remove useless log

* .

* Comment

* Add debug info

* .

* Silent log

* Debug

* Copy api view to somewhere else to avoid clean up by npm script

* .

* Cleanup

* Update rush script

* Update rush script

* Add missing package

* chore: Remove shx dependency and update npm packages

* Update CI creator

* Add log

* Update

* Update MC check

* Update log level

* Support emitter 0.32 in api version extraction

* Change temp folder location

* Add log

* Add debug info

* Fix package info

* Fix

* Improve logging

* Update logging

* .

* Update logging

* Update log

* RLC update

* Simplify

* Add default package name

* Fix runCommand

* Fix parameters finding

* Fix run command issue

* Fix runCommand issue

* Improve logging

* Add error keyword

* WIP on wanl/generate-mlc-in-pipeline

* .

* WIP

* Test sdk generation path

* Fix original package version restore

* .

* cleanup

* Update version

* Update package.json
  • Loading branch information
@wanlwanl
wanlwanl authored Sep 6, 2024
1 parent f9bbdd2 commit cfc4c9f
Show file tree
Hide file tree
Showing 36 changed files with 2,067 additions and 516 deletions.
751 changes: 674 additions & 77 deletions tools/js-sdk-release-tools/package-lock.json
View file

Large diffs are not rendered by default.

13 changes: 10 additions & 3 deletions tools/js-sdk-release-tools/package.json
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,16 @@
{
"name": "@azure-tools/js-sdk-release-tools",
"version": "2.7.11",
"version": "2.7.12",
"description": "",
"files": [
"dist"
],
"scripts": {
"dev": "tsx watch src/changelogToolCli.ts",
"start": "node dist/changelogToolCli.js",
"debug": "node --inspect-brk dist/changelogToolCli.js",
"build": "rimraf dist && tsc -p .",
"build": "rimraf dist && tsc -p . && npm run copy-files",
"copy-files": "copyfiles -f src/common/ciYamlTemplates/*.template.yml dist/common/ciYamlTemplates/",
"prepack": "npm run build",
"test": "vitest --run",
"test:watch": "vitest"
Expand All @@ -22,17 +26,18 @@
"license": "MIT",
"dependencies": {
"@azure-tools/openapi-tools-common": "^1.2.2",
"@npmcli/package-json": "^5.2.0",
"@ts-common/azure-js-dev-tools": "^21.1.0",
"colors": "1.4.0",
"command-line-args": "^5.1.1",
"comment-json": "^4.1.0",
"copyfiles": "^2.4.1",
"fs-extra": "^11.2.0",
"glob": "^11.0.0",
"js-yaml": "^4.1.0",
"parse-ts-to-ast": "^0.1.1",
"semver": "^7.3.5",
"shelljs": "^0.8.4",
"shx": "^0.3.4",
"simple-git": "^3.5.0",
"ts-morph": "^23.0.0",
"tslib": "^1.9.3",
Expand All @@ -42,7 +47,9 @@
"yaml": "^1.10.2"
},
"devDependencies": {
"@types/fs-extra": "^11.0.4",
"@types/node": "^20.12.12",
"@types/npmcli__package-json": "^4.0.4",
"@types/shelljs": "^0.8.15",
"@types/unixify": "^1.0.2",
"rimraf": "^3.0.2",
Expand Down
2 changes: 1 addition & 1 deletion ...packages/typescript-codegen-breaking-change-detector/src/azure/detect-breaking-changes.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ async function detectBreakingChangesCore(projectContext: ProjectContext): Promis
);
return breakingChangeResults;
} catch (err) {
logger.error(`Failed to detect breaking changes due to ${(err as Error).stack ?? err}`);
logger.error(`Failed to detect breaking changes due to ${(err as Error)?.stack ?? err}`);
return undefined;
}
}
Expand Down
167 changes: 97 additions & 70 deletions tools/js-sdk-release-tools/src/autoGenerateInPipeline.ts
View file
Original file line number Diff line number Diff line change
@@ -1,88 +1,112 @@
#!/usr/bin/env node

import * as path from 'path';
import { generateMgmt } from "./hlc/generateMgmt";
import { generateMgmt } from './hlc/generateMgmt';
import { backupNodeModules, restoreNodeModules } from './utils/backupNodeModules';
import { logger } from "./utils/logger";
import { generateRLCInPipeline } from "./llc/generateRLCInPipeline/generateRLCInPipeline";
import { RunningEnvironment } from "./utils/runningEnvironment";
import { logger } from './utils/logger';
import { generateRLCInPipeline } from './llc/generateRLCInPipeline/generateRLCInPipeline';
import { ModularClientPackageOptions, SDKType } from './common/types';
import { generateAzureSDKPackage } from './mlc/clientGenerator/modularClientPackageGenerator';
import { parseInputJson } from './utils/generateInputUtils';

const shell = require('shelljs');
const fs = require('fs');

async function automationGenerateInPipeline(inputJsonPath: string, outputJsonPath: string, use: string | undefined, typespecEmitter: string | undefined, sdkGenerationType: string | undefined) {
async function automationGenerateInPipeline(
inputJsonPath: string,
outputJsonPath: string,
use: string | undefined,
typespecEmitter: string | undefined,
sdkGenerationType: string | undefined,
local: boolean
) {
const inputJson = JSON.parse(fs.readFileSync(inputJsonPath, { encoding: 'utf-8' }));
const specFolder: string = inputJson['specFolder'];
const readmeFiles: string[] | string | undefined = inputJson['relatedReadmeMdFiles'] ? inputJson['relatedReadmeMdFiles'] : inputJson['relatedReadmeMdFile'];
const typespecProjectFolder: string[] | string | undefined = inputJson['relatedTypeSpecProjectFolder'];
const gitCommitId: string = inputJson['headSha'];
const repoHttpsUrl: string = inputJson['repoHttpsUrl'];
const autorestConfig: string | undefined = inputJson['autorestConfig'];
const downloadUrlPrefix: string | undefined = inputJson.installInstructionInput?.downloadUrlPrefix;
const skipGeneration: boolean | undefined = inputJson['skipGeneration'];
const {
sdkType,
specFolder,
readmeMd,
gitCommitId,
outputJson,
repoHttpsUrl,
downloadUrlPrefix,
skipGeneration,
runningEnvironment,
typespecProject,
autorestConfig
} = await parseInputJson(inputJson);

if (!readmeFiles && !typespecProjectFolder) {
throw new Error(`readme files and typespec project info are both undefined`);
}

if (readmeFiles && (typeof readmeFiles !== 'string') && readmeFiles.length !== 1) {
throw new Error(`get ${readmeFiles.length} readme files`);
}

if (typespecProjectFolder && (typeof typespecProjectFolder !== 'string') && typespecProjectFolder.length !== 1) {
throw new Error(`get ${typespecProjectFolder.length} typespec project`);
}

const isTypeSpecProject = !!typespecProjectFolder;

const packages: any[] = [];
const outputJson = {
packages: packages
};
const readmeMd = isTypeSpecProject ? undefined : typeof readmeFiles === 'string' ? readmeFiles : readmeFiles![0];
const typespecProject = isTypeSpecProject ? typeof typespecProjectFolder === 'string' ? typespecProjectFolder : typespecProjectFolder![0] : undefined;
const isMgmt = isTypeSpecProject ? false : readmeMd!.includes('resource-manager');
const runningEnvironment = typeof readmeFiles === 'string' || typeof typespecProjectFolder === 'string' ? RunningEnvironment.SdkGeneration : RunningEnvironment.SwaggerSdkAutomation;
try {
await backupNodeModules(String(shell.pwd()));
if (isMgmt) {
await generateMgmt({
sdkRepo: String(shell.pwd()),
swaggerRepo: specFolder,
readmeMd: readmeMd!,
gitCommitId: gitCommitId,
use: use,
outputJson: outputJson,
swaggerRepoUrl: repoHttpsUrl,
downloadUrlPrefix: downloadUrlPrefix,
skipGeneration: skipGeneration,
runningEnvironment: runningEnvironment
});
} else {
await generateRLCInPipeline({
sdkRepo: String(shell.pwd()),
swaggerRepo: path.isAbsolute(specFolder) ? specFolder : path.join(String(shell.pwd()), specFolder),
readmeMd: readmeMd,
typespecProject: typespecProject,
autorestConfig,
use: use,
typespecEmitter: !!typespecEmitter ? typespecEmitter : `@azure-tools/typespec-ts`,
outputJson: outputJson,
skipGeneration: skipGeneration,
sdkGenerationType: (sdkGenerationType === "command") ? "command" : "script",
runningEnvironment: runningEnvironment,
swaggerRepoUrl: repoHttpsUrl,
gitCommitId: gitCommitId,
})
if (!local) {
await backupNodeModules(String(shell.pwd()));
}
switch (sdkType) {
case SDKType.HighLevelClient:
await generateMgmt({
sdkRepo: String(shell.pwd()),
swaggerRepo: specFolder,
readmeMd: readmeMd!,
gitCommitId: gitCommitId,
use: use,
outputJson: outputJson,
swaggerRepoUrl: repoHttpsUrl,
downloadUrlPrefix: downloadUrlPrefix,
skipGeneration: skipGeneration,
runningEnvironment: runningEnvironment
});
break;
case SDKType.RestLevelClient:
await generateRLCInPipeline({
sdkRepo: String(shell.pwd()),
swaggerRepo: path.isAbsolute(specFolder) ? specFolder : path.join(String(shell.pwd()), specFolder),
readmeMd: readmeMd,
typespecProject: typespecProject,
autorestConfig,
use: use,
typespecEmitter: !!typespecEmitter ? typespecEmitter : `@azure-tools/typespec-ts`,
outputJson: outputJson,
skipGeneration: skipGeneration,
sdkGenerationType: sdkGenerationType === 'command' ? 'command' : 'script',
runningEnvironment: runningEnvironment,
swaggerRepoUrl: repoHttpsUrl,
gitCommitId: gitCommitId
});
break;

case SDKType.ModularClient: {
const typeSpecDirectory = path.posix.join(specFolder, typespecProject!);
const sdkRepoRoot = String(shell.pwd());
const skip = skipGeneration ?? false;
const repoUrl = repoHttpsUrl;
const options: ModularClientPackageOptions = {
sdkRepoRoot,
specRepoRoot: specFolder,
typeSpecDirectory,
gitCommitId,
skip,
repoUrl,
local,
// support MPG for now
versionPolicyName: 'management'
};
const packageResult = await generateAzureSDKPackage(options);
outputJson.packages = [packageResult];
break;
}
default:
break;
}
} catch (e) {
const packageName = outputJson.packages?.[0].packageName;
logger.error(`Failed to generate SDK for package ${"'" + packageName + "'" ?? ''} due to ${(e as Error)?.stack ?? e}.`);
const packageNameStr = `'${outputJson.packages?.[0]?.packageName}' `;
logger.error(`Failed to generate SDK for package ${packageNameStr ?? ''}due to ${(e as Error)?.stack ?? e}.`);
logger.error(`Please review the detail errors for potential fixes.`);
logger.error(`If the issue persists, contact the support channel at https://aka.ms/azsdk/js-teams-channel and include this spec pull request.`)
logger.error(
`If the issue persists, contact the support channel at https://aka.ms/azsdk/js-teams-channel and include this spec pull request.`
);
throw e;
} finally {
await restoreNodeModules(String(shell.pwd()));
if (!local) {
await restoreNodeModules(String(shell.pwd()));
}
fs.writeFileSync(outputJsonPath, JSON.stringify(outputJson, null, ' '), { encoding: 'utf-8' });
}
}
Expand All @@ -93,10 +117,13 @@ const optionDefinitions = [
{ name: 'sdkGenerationType', type: String },
{ name: 'inputJsonPath', type: String },
{ name: 'outputJsonPath', type: String },
// this option should be only used in local run, it will skip backup node modules, etc.
// do NOT set to true in sdk automation pipeline
{ name: 'local', type: Boolean, defaultValue: false }
];
const commandLineArgs = require('command-line-args');
const options = commandLineArgs(optionDefinitions);
automationGenerateInPipeline(options.inputJsonPath, options.outputJsonPath, options.use, options.typespecEmitter, options.sdkGenerationType).catch(e => {
automationGenerateInPipeline(options.inputJsonPath, options.outputJsonPath, options.use, options.typespecEmitter, options.sdkGenerationType, options.local ?? false).catch(e => {
logger.error(e.message);
process.exit(1);
});
33 changes: 33 additions & 0 deletions tools/js-sdk-release-tools/src/common/ciYamlTemplates/ci.mgmt.template.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
# NOTE: Please refer to https://aka.ms/azsdk/engsys/ci-yaml before editing this file.

trigger:
branches:
include:
- main
- feature/*
- release/*
- hotfix/*
exclude:
- feature/v4
paths:
include: null

pr:
branches:
include:
- main
- feature/*
- release/*
- hotfix/*
exclude:
- feature/v4
paths:
include: null

extends:
template: /eng/pipelines/templates/stages/archetype-sdk-client.yml
parameters:
ServiceDirectory: null
Artifacts:
- name: null
safeName: null
Loading

0 comments on commit cfc4c9f

Please sign in to comment.