From b86435ab044bb2123afc19921e7f5d8a54fd6205 Mon Sep 17 00:00:00 2001 From: Samir Musali Date: Fri, 19 Mar 2021 15:33:46 -0400 Subject: [PATCH] feat(config): use LogDNA's env-config package Replace current configuration with [@logdna/env-logdna](https://www.npmjs.com/package/@logdna/env-config) Ref: #2 Semver: major --- doc/env.md | 254 +++++++++++++++++++++++++++++++++++ index.js | 45 ++++--- lib/config.js | 98 ++++++++++---- lib/constants.js | 16 +++ lib/logger.js | 61 ++++----- package-lock.json | 5 + package.json | 10 +- test/unit/lib/transformer.js | 48 +++++-- 8 files changed, 445 insertions(+), 92 deletions(-) create mode 100644 doc/env.md create mode 100644 lib/constants.js diff --git a/doc/env.md b/doc/env.md new file mode 100644 index 0000000..f235cdb --- /dev/null +++ b/doc/env.md @@ -0,0 +1,254 @@ +## Environment Variables + +### `BATCH_INTERVAL` + +> The number of milliseconds between sending each batch + +| Config | Value | +| --- | --- | +| Name | `batch-interval` | +| Environment Variable | `BATCH_INTERVAL` | +| Type | `number` | +| Required | no | +| Default | `50` | + +*** + +### `BATCH_LIMIT` + +> The number of lines within each batch + +| Config | Value | +| --- | --- | +| Name | `batch-limit` | +| Environment Variable | `BATCH_LIMIT` | +| Type | `number` | +| Required | no | +| Default | `25` | + +*** + +### `FREE_SOCKET_TIMEOUT` + +> The number of milliseconds to wait for inactivity before timing out + +| Config | Value | +| --- | --- | +| Name | `free-socket-timeout` | +| Environment Variable | `FREE_SOCKET_TIMEOUT` | +| Type | `number` | +| Required | no | +| Default | `300000` | + +*** + +### `HOSTNAME` + +> Optionally, use alternative host name set through the environment + +| Config | Value | +| --- | --- | +| Name | `hostname` | +| Environment Variable | `HOSTNAME` | +| Type | `string` | +| Required | no | +| Default | `(none)` | + +*** + +### `HTTP_PROXY` + +> An http:// proxy URL to pass through before going to LogDNA + +| Config | Value | +| --- | --- | +| Name | `http-proxy` | +| Environment Variable | `HTTP_PROXY` | +| Type | `string` | +| Required | no | +| Default | `(none)` | + +*** + +### `HTTPS_PROXY` + +> A secure (https://) proxy URL to pass through before going to LogDNA + +| Config | Value | +| --- | --- | +| Name | `https-proxy` | +| Environment Variable | `HTTPS_PROXY` | +| Type | `string` | +| Required | no | +| Default | `(none)` | + +*** + +### `INGESTION_ENDPOINT` + +> The endpoint for log ingestion at LogDNA + +| Config | Value | +| --- | --- | +| Name | `ingestion-endpoint` | +| Environment Variable | `INGESTION_ENDPOINT` | +| Type | `string` | +| Required | no | +| Default | `/logs/ingest` | + +*** + +### `INGESTION_HOST` + +> The host for log ingestion + +| Config | Value | +| --- | --- | +| Name | `ingestion-host` | +| Environment Variable | `INGESTION_HOST` | +| Type | `string` | +| Required | no | +| Default | `logs.logdna.com` | + +*** + +### `INGESTION_KEY` + +> LogDNA Ingestion Key to stream the logs from files + +| Config | Value | +| --- | --- | +| Name | `ingestion-key` | +| Environment Variable | `INGESTION_KEY` | +| Type | `string` | +| Required | **yes** | +| Default | `(none)` | + +*** + +### `INGESTION_PORT` + +> The port for log ingestion + +| Config | Value | +| --- | --- | +| Name | `ingestion-port` | +| Environment Variable | `INGESTION_PORT` | +| Type | `number` | +| Required | no | +| Default | `443` | + +*** + +### `MAX_REQUEST_RETRIES` + +> Maximum number of retries for sending each batch + +| Config | Value | +| --- | --- | +| Name | `max-request-retries` | +| Environment Variable | `MAX_REQUEST_RETRIES` | +| Type | `number` | +| Required | no | +| Default | `5` | + +*** + +### `MAX_REQUEST_TIMEOUT` + +> Maximum request timeout in sending each batch + +| Config | Value | +| --- | --- | +| Name | `max-request-timeout` | +| Environment Variable | `MAX_REQUEST_TIMEOUT` | +| Type | `number` | +| Required | no | +| Default | `300` | + +*** + +### `PROXY` + +> A full proxy URL (including protocol) to pass through before going to LogDNA + +| Config | Value | +| --- | --- | +| Name | `proxy` | +| Environment Variable | `PROXY` | +| Type | `string` | +| Required | no | +| Default | `(none)` | + +*** + +### `REQUEST_RETRY_INTERVAL` + +> The number of milliseconds between each retry + +| Config | Value | +| --- | --- | +| Name | `request-retry-interval` | +| Environment Variable | `REQUEST_RETRY_INTERVAL` | +| Type | `number` | +| Required | no | +| Default | `100` | + +*** + +### `SSL` + +> Use https:// for log ingestion + +| Config | Value | +| --- | --- | +| Name | `ssl` | +| Environment Variable | `SSL` | +| Type | `boolean` | +| Required | no | +| Default | `true` | + +*** + +### `TAGS` + +> Optionally, use comma-separated tags set through the environment + +| Config | Value | +| --- | --- | +| Name | `tags` | +| Environment Variable | `TAGS` | +| Type | `string` | +| Required | no | +| Default | `(none)` | + +*** + +### `URL` + +> *Combination of SSL, INGESTION_HOST, INGESTION_PORT, and INGESTION_ENDPOINT* + +| Config | Value | +| --- | --- | +| Name | `url` | +| Environment Variable | `URL` | +| Type | `string` | +| Required | no | +| Default | `https://logs.logdna.com/logs/ingest` | + +*** + +### `USER_AGENT` + +> user-agent header value to use while sending logs + +| Config | Value | +| --- | --- | +| Name | `user-agent` | +| Environment Variable | `USER_AGENT` | +| Type | `string` | +| Required | no | +| Default | `logdna-s3/2.0.0` | + +*** + diff --git a/index.js b/index.js index dbb69cc..3ac7b06 100644 --- a/index.js +++ b/index.js @@ -2,36 +2,49 @@ const async = require('async') -const {getConfig} = require('./lib/config.js') +const config = require('./lib/config.js') const {handleEvent} = require('./lib/event-handler.js') const {flush} = require('./lib/logger.js') const {getLogs, prepareLogs} = require('./lib/transformer.js') const {batchify, getProperty} = require('./lib/utils.js') -const BATCH_INTERVAL = parseInt(process.env.LOGDNA_BATCH_INTERVAL) || 50 -const BATCH_LIMIT = parseInt(process.env.LOGDNA_BATCH_LIMIT) || 25 +const DOT_REGEXP = /\./g module.exports = { handler } -function handler(event, context, callback) { - const config = getConfig(event) +async function handler(event, context, callback) { + config.validateEnvVars() + const tags = config.get('tags') + if (tags) { + config.set('tags', tags.split(',').map((tag) => { + return tag.trim() + }).join(',')) + } + const eventData = handleEvent(event) const s3params = { Bucket: getProperty(eventData, 'meta.bucket.name') , Key: getProperty(eventData, 'meta.object.key') } - return getLogs(s3params, (error, lines) => { - if (error) return callback(error) - - const logArrays = prepareLogs(lines, eventData) - const batches = batchify(logArrays, BATCH_LIMIT) - return async.everySeries(batches, (batch, next) => { - setTimeout(() => { - return flush(batch, config, next) - }, BATCH_INTERVAL) - }, callback) - }) + let lines + try { + lines = getLogs(s3params) + } catch (e) { + return callback(e) + } + + const logArrays = prepareLogs(lines, eventData) + const batches = batchify(logArrays, config.get('batch-limit')) + if (!config.get('hostname')) { + config.set('hostname', s3params.Bucket.replace(DOT_REGEXP, '_')) + } + + async.everySeries(batches, (batch, next) => { + setTimeout(() => { + return flush(batch, config, next) + }, config.get('batch-interval')) + }, callback) } diff --git a/lib/config.js b/lib/config.js index 8f41256..811c1da 100644 --- a/lib/config.js +++ b/lib/config.js @@ -1,31 +1,77 @@ 'use strict' -const {getProperty} = require('./utils.js') +const Config = require('@logdna/env-config') -const DOT_REGEXP = /\./g +const pkg = require('../package.json') -module.exports = { - getConfig -} +const config = new Config([ + Config + .number('batch-interval') + .default(50) + .desc('The number of milliseconds between sending each batch') +, Config + .number('batch-limit') + .default(25) + .desc('The number of lines within each batch') +, Config + .number('free-socket-timeout') + .default(300000) + .desc('The number of milliseconds to wait for inactivity before timing out') +, Config + .string('hostname') + .desc('Optionally, use alternative host name set through the environment') +, Config + .string('http-proxy') + .desc('An http:// proxy URL to pass through before going to LogDNA') +, Config + .string('https-proxy') + .desc('A secure (https://) proxy URL to pass through before going to LogDNA') +, Config + .string('ingestion-key') + .required() + .desc('LogDNA Ingestion Key to stream the logs from files') +, Config + .string('ingestion-endpoint') + .default('/logs/ingest') + .desc('The endpoint for log ingestion at LogDNA') +, Config + .string('ingestion-host') + .default('logs.logdna.com') + .desc('The host for log ingestion') +, Config + .number('ingestion-port') + .default(443) + .desc('The port for log ingestion') +, Config + .number('max-request-retries') + .default(5) + .desc('Maximum number of retries for sending each batch') +, Config + .number('max-request-timeout') + .default(300) + .desc('Maximum request timeout in sending each batch') +, Config + .string('proxy') + .desc('A full proxy URL (including protocol) to pass through before going to LogDNA') +, Config + .number('request-retry-interval') + .default(100) + .desc('The number of milliseconds between each retry') +, Config + .boolean('ssl') + .default(true) + .desc('Use https:// for log ingestion') +, Config + .string('tags') + .desc('Optionally, use comma-separated tags set through the environment') +, Config + .string('url') + .default('https://logs.logdna.com/logs/ingest') + .desc('*Combination of SSL, INGESTION_HOST, INGESTION_PORT, and INGESTION_ENDPOINT*') +, Config + .string('user-agent') + .default(`${pkg.name}/${pkg.version}`) + .desc('user-agent header value to use while sending logs') +]) -function getConfig(event) { - const pkg = require('../package.json') - const config = { - UserAgent: `${pkg.name}/${pkg.version}` - } - - if (process.env.LOGDNA_KEY) config.key = process.env.LOGDNA_KEY - if (process.env.LOGDNA_HOSTNAME) config.hostname = process.env.LOGDNA_HOSTNAME - if (process.env.LOGDNA_TAGS) { - config.tags = process.env.LOGDNA_TAGS.split(',').map((tag) => { - return tag.trim() - }).join(',') - } - - const bucketName = getProperty(event, 'Records.0.s3.bucket.name') - if (bucketName && !config.hostname) { - config.hostname = bucketName.replace(DOT_REGEXP, '_') - } - - return config -} +module.exports = config diff --git a/lib/constants.js b/lib/constants.js new file mode 100644 index 0000000..2a3128a --- /dev/null +++ b/lib/constants.js @@ -0,0 +1,16 @@ +'use strict' + +module.exports = { + DEFAULT_HTTP_ERRORS: [ + 'ECONNRESET' + , 'EHOSTUNREACH' + , 'ETIMEDOUT' + , 'ESOCKETTIMEDOUT' + , 'ECONNREFUSED' + , 'ENOTFOUND' + ] +, INTERNAL_SERVER_ERROR: { + statusCode: 500 + , code: 'INTERNAL_SERVER_ERROR' + } +} diff --git a/lib/logger.js b/lib/logger.js index a931bc9..b6c90fb 100644 --- a/lib/logger.js +++ b/lib/logger.js @@ -4,67 +4,52 @@ const agent = require('agentkeepalive') const async = require('async') const request = require('request') -const FREE_SOCKET_TIMEOUT = parseInt(process.env.LOGDNA_FREE_SOCKET_TIMEOUT) || 300000 -const LOGDNA_URL = process.env.LOGDNA_URL || 'https://logs.logdna.com/logs/ingest' -const MAX_REQUEST_RETRIES = parseInt(process.env.LOGDNA_MAX_REQUEST_RETRIES) || 5 -const MAX_REQUEST_TIMEOUT = parseInt(process.env.LOGDNA_MAX_REQUEST_TIMEOUT) || 30000 -const REQUEST_RETRY_INTERVAL = parseInt(process.env.LOGDNA_REQUEST_RETRY_INTERVAL) || 100 -const INTERNAL_SERVER_ERROR = 500 -const DEFAULT_HTTP_ERRORS = [ - 'ECONNRESET' -, 'EHOSTUNREACH' -, 'ETIMEDOUT' -, 'ESOCKETTIMEDOUT' -, 'ECONNREFUSED' -, 'ENOTFOUND' -] +const constants = require('./constants.js') module.exports = { flush } -function flush(payload, conf, callback) { - // Check for Ingestion Key - if (!conf.key) return callback('Missing LogDNA Ingestion Key') - - // Prepare HTTP Request Options +function flush(payload, config, callback) { const options = { - url: LOGDNA_URL - , qs: conf.tags ? { - tags: conf.tags - , hostname: conf.hostname - } : { - hostname: conf.hostname - }, method: 'POST' + url: config.get('url') + , qs: { + tags: config.tags + , hostname: config.hostname + } + , method: 'POST' , body: JSON.stringify({ e: 'ls' , ls: payload - }), auth: { - username: conf.key - }, headers: { + }) + , auth: { + username: config.get('ingestion-key') + } + , headers: { 'Content-Type': 'application/json; charset=UTF-8' - , 'user-agent': conf.UserAgent - }, timeout: MAX_REQUEST_TIMEOUT + , 'user-agent': config.get('user-agent') + } + , timeout: config.get('max-request-timeout') , withCredentials: false , agent: new agent.HttpsAgent({ - freeSocketTimeout: FREE_SOCKET_TIMEOUT + freeSocketTimeout: config.get('free-socket-timeout') }) } - // Flush the Log async.retry({ - times: MAX_REQUEST_RETRIES + times: config.get('max-request-retries') , interval: (retryCount) => { - return REQUEST_RETRY_INTERVAL * Math.pow(2, retryCount) + return config.get('request-retry-interval') * Math.pow(2, retryCount) } , errorFilter: (errCode) => { - return DEFAULT_HTTP_ERRORS.includes(errCode) || errCode === 'INTERNAL_SERVER_ERROR' + return constants.DEFAULT_HTTP_ERRORS.includes(errCode) + || errCode === constants.INTERNAL_SERVER_ERROR.code } }, (reqCallback) => { return request(options, (error, response, body) => { if (error) return reqCallback(error.code) - if (response.statusCode >= INTERNAL_SERVER_ERROR) { - return reqCallback('INTERNAL_SERVER_ERROR') + if (response.statusCode >= constants.INTERNAL_SERVER_ERROR.statusCode) { + return reqCallback(constants.INTERNAL_SERVER_ERROR.code) } return reqCallback(null, body) diff --git a/package-lock.json b/package-lock.json index 6621214..519e70c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -192,6 +192,11 @@ } } }, + "@logdna/env-config": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@logdna/env-config/-/env-config-1.0.5.tgz", + "integrity": "sha512-pK+8J1lSeWKdfiUu4Aeww8sssC0+cCFZmlyPbh8dViA0B4JIfpSnAFJ9KBgaGYljPvNRGrdYsb4xSSe0gQOkKw==" + }, "acorn": { "version": "7.4.1", "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.1.tgz", diff --git a/package.json b/package.json index c97614a..566e8b6 100644 --- a/package.json +++ b/package.json @@ -4,6 +4,7 @@ "description": "Lambda Function to Stream Logs from AWS S3 to LogDNA", "main": "index.js", "scripts": { + "docs": "config-doc lib/config.js > doc/env.md", "lint": "eslint ./", "lint:fix": "npm run lint -- --fix", "pretest": "npm run lint", @@ -45,10 +46,10 @@ "files": [ "test/unit" ], - "statements": 66, - "branches": 63, - "functions": 54, - "lines": 66 + "statements": 73, + "branches": 88, + "functions": 59, + "lines": 71 }, "husky": { "hooks": { @@ -56,6 +57,7 @@ } }, "dependencies": { + "@logdna/env-config": "^1.0.5", "agentkeepalive": "^4.0.2", "async": "^2.6.2", "request": "^2.88.0" diff --git a/test/unit/lib/transformer.js b/test/unit/lib/transformer.js index 247818f..2256cde 100644 --- a/test/unit/lib/transformer.js +++ b/test/unit/lib/transformer.js @@ -595,6 +595,36 @@ test('getLogs', async (t) => { s3.getObject = getObject }) + getLogs(params, (error, data) => { + t.match(data, [{ + line: input + , timestamp: /^[0-9]{13}$/ + }], 'Zipped JSON success') + t.strictEqual(error, null, 'JSON.parse is clear') + }) + }) + + t.test('where data is valid json', async (t) => { + const params = { + Bucket: SAMPLE_BUCKET + , Key: `${SAMPLE_OBJECT_KEY}.json` + } + + const input = JSON.stringify({ + log: LOG_LINE + }) + + const getObject = s3.getObject + s3.getObject = function(params) { + return { + Body: input + } + } + + t.tearDown(() => { + s3.getObject = getObject + }) + getLogs(params, (error, data) => { t.match(data, [{ line: input @@ -703,14 +733,15 @@ test('prepareLogs', async (t) => { } const output = input.map(function(item) { - item.timestamp = eventData.timestamp - item.file = eventData.file - item.meta = { + const line = {...item} + line.timestamp = eventData.timestamp + line.file = eventData.file + line.meta = { ...item.meta , ...eventData.meta } - return item + return line }) t.deepEqual(prepareLogs(input, eventData), output @@ -737,14 +768,15 @@ test('prepareLogs', async (t) => { } const output = input.map(function(item) { - item.timestamp = /^[0-9]{13}$/ - item.file = eventData.file - item.meta = { + const line = {...item} + line.timestamp = /^[0-9]{13}$/ + line.file = eventData.file + line.meta = { ...item.meta , ...eventData.meta } - return item + return line }) t.match(prepareLogs(input, eventData), output