refactor(storage): Migrate conformance tests to StorageTransport and enhance retry logic

handwritten/storage/CHANGELOG.md

@@ -1,6 +1,5 @@
 # Changelog
 
-
 [npm history][1]
 
 [1]: https://www.npmjs.com/package/@google-cloud/storage?activeTab=versions

handwritten/storage/SECURITY.md

@@ -0,0 +1,7 @@
+# Security Policy
+
+To report a security issue, please use [g.co/vulnz](https://g.co/vulnz).
+
+The Google Security Team will respond within 5 working days of your report on g.co/vulnz.
+
+We use g.co/vulnz for our intake, and do coordination and disclosure here using GitHub Security Advisory to privately discuss and fix the issue.

handwritten/storage/conformance-test/conformanceCommon.ts

@@ -13,14 +13,18 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+// eslint-disable-next-line @typescript-eslint/no-unused-vars
 import * as jsonToNodeApiMapping from './test-data/retryInvocationMap.json';
-import * as libraryMethods from './libraryMethods';
-import {Bucket, File, HmacKey, Notification, Storage} from '../src/';
+import * as libraryMethods from './libraryMethods.js';
+import {Bucket, File, HmacKey, Notification, Storage} from '../src';
+import * as gaxios from 'gaxios';
 import * as crypto from 'crypto';
 import * as assert from 'assert';
-import {DecorateRequestOptions} from '../src/nodejs-common';
-import fetch from 'node-fetch';
-
+import {StorageTransport} from '../src/storage-transport.js';
+import {getDirName} from '../src/util.js';
+import path from 'path';
+import * as fs from 'fs';
+import {GoogleAuth} from 'google-auth-library';
 interface RetryCase {
   instructions: String[];
 }
@@ -50,7 +54,7 @@ interface ConformanceTestResult {
 
 type LibraryMethodsModuleType = typeof import('./libraryMethods');
 const methodMap: Map<String, String[]> = new Map(
-  Object.entries(jsonToNodeApiMapping)
+  Object.entries(jsonToNodeApiMapping),
 );
 
 const DURATION_SECONDS = 600; // 10 mins.
@@ -60,6 +64,27 @@ const TESTBENCH_HOST =
 const CONF_TEST_PROJECT_ID = 'my-project-id';
 const TIMEOUT_FOR_INDIVIDUAL_TEST = 20000;
 const RETRY_MULTIPLIER_FOR_CONFORMANCE_TESTS = 0.01;
+const SERVICE_ACCOUNT = path.join(
+  getDirName(),
+  '../../../conformance-test/fixtures/signing-service-account.json',
+);
+
+const authClient = new GoogleAuth({
+  keyFilename: SERVICE_ACCOUNT,
+  scopes: ['https://www.googleapis.com/auth/devstorage.full_control'],
+}).fromJSON(JSON.parse(fs.readFileSync(SERVICE_ACCOUNT, 'utf8')));
+
+authClient.getAccessToken = async () => ({token: 'unauthenticated-test-token'});
+authClient.request = async (opts: unknown) => {
+  const options = opts as gaxios.GaxiosOptions & {
+    adapter?: (opts: unknown) => Promise<unknown>;
+  };
+  if (typeof options.adapter === 'function') {
+    return options.adapter(opts) as Promise<gaxios.GaxiosResponse>;
+  }
+  const defaultGaxios = gaxios as unknown as {instance: gaxios.Gaxios};
+  return defaultGaxios.instance.request(options);
+};
 
 export function executeScenario(testCase: RetryTestCase) {
   for (
@@ -79,84 +104,163 @@ export function executeScenario(testCase: RetryTestCase) {
         let bucket: Bucket;
         let file: File;
         let notification: Notification;
-        let creationResult: {id: string};
+        let creationResult: ConformanceTestCreationResult;
         let storage: Storage;
         let hmacKey: HmacKey;
+        let storageTransport: StorageTransport;
 
         describe(`${storageMethodString}`, async () => {
           beforeEach(async () => {
-            storage = new Storage({
+            const defaultGaxios = gaxios as unknown as {
+              instance?: gaxios.Gaxios;
+            };
+            defaultGaxios.instance?.interceptors?.request?.clear();
+
+            const rawTransport = new StorageTransport({
               apiEndpoint: TESTBENCH_HOST,
-              projectId: CONF_TEST_PROJECT_ID,
+              authClient: authClient,
+              keyFilename: SERVICE_ACCOUNT,
+              baseUrl: TESTBENCH_HOST,
+              packageJson: {name: 'test-package', version: '1.0.0'},
               retryOptions: {
                 retryDelayMultiplier: RETRY_MULTIPLIER_FOR_CONFORMANCE_TESTS,
+                maxRetries: 3,
+                maxRetryDelay: 32,
+                totalTimeout: TIMEOUT_FOR_INDIVIDUAL_TEST,
               },
+              scopes: [
+                'http://www.googleapis.com/auth/devstorage.full_control',
+              ],
+              projectId: CONF_TEST_PROJECT_ID,
+              userAgent: 'retry-test',
+              useAuthWithCustomEndpoint: true,
+              customEndpoint: true,
+              timeout: DURATION_SECONDS,
             });
+
             creationResult = await createTestBenchRetryTest(
               instructionSet.instructions,
-              jsonMethod?.name.toString()
+              jsonMethod?.name.toString(),
+              rawTransport,
             );
-            if (storageMethodString.includes('InstancePrecondition')) {
-              bucket = await createBucketForTest(
-                storage,
-                testCase.preconditionProvided,
-                storageMethodString
-              );
-              file = await createFileForTest(
-                testCase.preconditionProvided,
-                storageMethodString,
-                bucket
-              );
-            } else {
-              bucket = await createBucketForTest(
-                storage,
-                false,
-                storageMethodString
-              );
-              file = await createFileForTest(
-                false,
-                storageMethodString,
-                bucket
-              );
-            }
-            notification = bucket.notification(`${TESTS_PREFIX}`);
-            await notification.create();
 
-            [hmacKey] = await storage.createHmacKey(
-              `${TESTS_PREFIX}@email.com`
-            );
-
-            storage.interceptors.push({
-              request: requestConfig => {
-                requestConfig.headers = requestConfig.headers || {};
-                Object.assign(requestConfig.headers, {
-                  'x-retry-test-id': creationResult.id,
-                });
-                return requestConfig as DecorateRequestOptions;
+            storage = new Storage({
+              apiEndpoint: TESTBENCH_HOST,
+              projectId: CONF_TEST_PROJECT_ID,
+              keyFilename: SERVICE_ACCOUNT,
+              authClient: authClient,
+              retryOptions: {
+                retryDelayMultiplier: RETRY_MULTIPLIER_FOR_CONFORMANCE_TESTS,
               },
             });
+
+            bucket = await createBucketForTest(
+              storage,
+              testCase.preconditionProvided &&
+                !storageMethodString.includes('combine'),
+              storageMethodString,
+            );
+            file = await createFileForTest(
+              testCase.preconditionProvided,
+              storageMethodString,
+              bucket,
+            );
+            if (
+              storageMethodString !== 'createNotification' &&
+              storageMethodString !== 'notificationCreate'
+            ) {
+              notification = bucket.notification(TESTS_PREFIX);
+              await notification.create();
+            }
+
+            if (
+              storageMethodString === 'deleteHMAC' ||
+              storageMethodString === 'getHMAC' ||
+              storageMethodString === 'getMetadataHMAC' ||
+              storageMethodString === 'setMetadataHMAC'
+            ) {
+              [hmacKey] = await storage.createHmacKey(
+                `${TESTS_PREFIX}@email.com`,
+              );
+            }
+
+            storageTransport = storage.storageTransport;
           });
 
           it(`${instructionNumber}`, async () => {
             const methodParameters: libraryMethods.ConformanceTestOptions = {
-              bucket: bucket,
-              file: file,
-              notification: notification,
-              storage: storage,
-              hmacKey: hmacKey,
+              storage,
+              bucket,
+              file,
+              storageTransport,
+              notification,
+              hmacKey,
+              projectId: CONF_TEST_PROJECT_ID,
+              preconditionRequired: testCase.preconditionProvided,
             };
-            if (testCase.preconditionProvided) {
-              methodParameters.preconditionRequired = true;
-            }
-            if (testCase.expectSuccess) {
-              assert.ifError(await storageMethodObject(methodParameters));
-            } else {
-              await assert.rejects(storageMethodObject(methodParameters));
-            }
-            const testBenchResult = await getTestBenchRetryTest(
-              creationResult.id
+
+            const injectHeader = async (
+              reqOpts: gaxios.GaxiosOptionsPrepared,
+            ) => {
+              const url = reqOpts.url?.toString() || '';
+              if (url.includes('retry_test') || !creationResult?.id) {
+                return reqOpts;
+              }
+              reqOpts.headers = reqOpts.headers || {};
+              if (typeof (reqOpts.headers as Headers).set === 'function') {
+                (reqOpts.headers as Headers).set(
+                  'x-retry-test-id',
+                  creationResult.id,
+                );
+              }
+              try {
+                (reqOpts.headers as unknown as Record<string, unknown>)[
+                  'x-retry-test-id'
+                ] = creationResult.id;
+              } catch (e) {
+                /* empty */
+              }
+              return reqOpts;
+            };
+
+            const interceptor = {
+              resolved: injectHeader,
+              request: injectHeader,
+            };
+
+            const transportWithInstance =
+              storage.storageTransport as unknown as {
+                gaxiosInstance: gaxios.Gaxios;
+              };
+            const defaultGaxios = gaxios as unknown as {
+              instance: gaxios.Gaxios;
+            };
+
+            transportWithInstance.gaxiosInstance?.interceptors?.request?.clear();
+            defaultGaxios.instance?.interceptors?.request?.clear();
+
+            transportWithInstance.gaxiosInstance.interceptors.request.add(
+              interceptor,
             );
-            assert.strictEqual(testBenchResult.completed, true);
+            defaultGaxios.instance.interceptors.request.add(interceptor);
+
+            try {
+              if (testCase.expectSuccess) {
+                await storageMethodObject(methodParameters);
+                const testBenchResult = await getTestBenchRetryTest(
+                  creationResult.id,
+                  storageTransport,
+                );
+                assert.strictEqual(testBenchResult.completed, true);
+              } else {
+                await assert.rejects(async () => {
+                  await storageMethodObject(methodParameters);
+                }, undefined);
+              }
+            } finally {
+              transportWithInstance.gaxiosInstance?.interceptors?.request?.clear();
+              defaultGaxios.instance?.interceptors?.request?.clear();
+            }
           }).timeout(TIMEOUT_FOR_INDIVIDUAL_TEST);
         });
       });
@@ -166,68 +270,72 @@ export function executeScenario(testCase: RetryTestCase) {
 
 async function createBucketForTest(
   storage: Storage,
-  preconditionShouldBeOnInstance: boolean,
-  storageMethodString: String
+  withPrecondition: boolean,
+  method: String,
 ) {
-  const name = generateName(storageMethodString, 'bucket');
-  const bucket = storage.bucket(name);
-  await bucket.create();
+  const bucket = storage.bucket(generateName(method, 'bucket'));
+  const [metadata] = await bucket.create();
   await bucket.setRetentionPeriod(DURATION_SECONDS);
-
-  if (preconditionShouldBeOnInstance) {
+  if (withPrecondition) {
     return new Bucket(storage, bucket.name, {
       preconditionOpts: {
-        ifMetagenerationMatch: 2,
+        ifMetagenerationMatch: metadata.metageneration,
       },
     });
   }
   return bucket;
 }
 
 async function createFileForTest(
-  preconditionShouldBeOnInstance: boolean,
-  storageMethodString: String,
-  bucket: Bucket
+  withPrecondition: boolean,
+  method: String,
+  bucket: Bucket,
 ) {
-  const name = generateName(storageMethodString, 'file');
-  const file = bucket.file(name);
-  await file.save(name);
-  if (preconditionShouldBeOnInstance) {
+  const file = bucket.file(generateName(method, 'file'));
+  if (method === 'deleteBucket') {
+    return file;
+  }
+  await file.save('test-content');
+  if (withPrecondition) {
+    const [metadata] = await file.getMetadata();
     return new File(bucket, file.name, {
       preconditionOpts: {
-        ifMetagenerationMatch: file.metadata.metageneration,
-        ifGenerationMatch: file.metadata.generation,
+        ifMetagenerationMatch: metadata.metageneration,
+        ifGenerationMatch: metadata.generation,
       },
     });
   }
   return file;
 }
 
-function generateName(storageMethodString: String, bucketOrFile: string) {
-  return `${TESTS_PREFIX}${storageMethodString.toLowerCase()}${bucketOrFile}.${shortUUID()}`;
-}
-
 async function createTestBenchRetryTest(
   instructions: String[],
-  methodName: string
+  methodName: string,
+  transport: StorageTransport,
 ): Promise<ConformanceTestCreationResult> {
-  const requestBody = {instructions: {[methodName]: instructions}};
-  const response = await fetch(`${TESTBENCH_HOST}retry_test`, {
+  const response = await transport.makeRequest({
     method: 'POST',
-    body: JSON.stringify(requestBody),
+    url: 'retry_test',
+    body: JSON.stringify({instructions: {[methodName]: instructions}}),
     headers: {'Content-Type': 'application/json'},
   });
-  return response.json() as Promise<ConformanceTestCreationResult>;
+  return response.data as ConformanceTestCreationResult;
 }
 
 async function getTestBenchRetryTest(
-  testId: string
+  testId: string,
+  transport: StorageTransport,
 ): Promise<ConformanceTestResult> {
-  const response = await fetch(`${TESTBENCH_HOST}retry_test/${testId}`, {
+  const response = await transport.makeRequest({
+    url: `retry_test/${testId}`,
     method: 'GET',
+    headers: {'x-retry-test-id': testId},
   });
+  return response.data as ConformanceTestResult;
+}
 
-  return response.json() as Promise<ConformanceTestResult>;
+function generateName(method: String, type: string) {
+  return `${TESTS_PREFIX}${method.toLowerCase()}${type}.${shortUUID()}`;
 }
 
 function shortUUID() {