diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 92977ba42f..16de3fd708 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -203,7 +203,7 @@ sql: attributes: batch_size: "${SQL_ATTRIBUTES_BATCH_SIZE:10000}" batch_max_delay: "${SQL_ATTRIBUTES_BATCH_MAX_DELAY_MS:100}" - stats_print_interval_ms: "${SQL_ATTRIBUTES_BATCH_STATS_PRINT_MS:1000}" + stats_print_interval_ms: "${SQL_ATTRIBUTES_BATCH_STATS_PRINT_MS:10000}" # Specify whether to remove null characters from strValue of attributes and timeseries before insert remove_null_chars: "${SQL_REMOVE_NULL_CHARS:true}" diff --git a/msa/js-executor/api/jsExecutor.js b/msa/js-executor/api/jsExecutor.js index 3568a6c410..7a4ffb04f1 100644 --- a/msa/js-executor/api/jsExecutor.js +++ b/msa/js-executor/api/jsExecutor.js @@ -17,11 +17,28 @@ const vm = require('vm'); -function JsExecutor() { +function JsExecutor(useSandbox) { + this.useSandbox = useSandbox; } JsExecutor.prototype.compileScript = function(code) { - return new Promise(function(resolve, reject) { + if (this.useSandbox) { + return createScript(code); + } else { + return createFunction(code); + } +} + +JsExecutor.prototype.executeScript = function(script, args, timeout) { + if (this.useSandbox) { + return invokeScript(script, args, timeout); + } else { + return invokeFunction(script, args); + } +} + +function createScript(code) { + return new Promise((resolve, reject) => { try { code = "("+code+")(...args)"; var script = new vm.Script(code); @@ -32,8 +49,8 @@ JsExecutor.prototype.compileScript = function(code) { }); } -JsExecutor.prototype.executeScript = function(script, args, timeout) { - return new Promise(function(resolve, reject) { +function invokeScript(script, args, timeout) { + return new Promise((resolve, reject) => { try { var sandbox = Object.create(null); sandbox.args = args; @@ -45,4 +62,29 @@ JsExecutor.prototype.executeScript = function(script, args, timeout) { }); } + +function createFunction(code) { + return new Promise((resolve, reject) => { + try { + code = "return ("+code+")(...args)"; + const parsingContext = vm.createContext({}); + const func = vm.compileFunction(code, ['args'], {parsingContext: parsingContext}); + resolve(func); + } catch (err) { + reject(err); + } + }); +} + +function invokeFunction(func, args) { + return new Promise((resolve, reject) => { + try { + var result = func(args); + resolve(result); + } catch (err) { + reject(err); + } + }); +} + module.exports = JsExecutor; diff --git a/msa/js-executor/api/jsInvokeMessageProcessor.js b/msa/js-executor/api/jsInvokeMessageProcessor.js index 1a1df75e09..6afc02e89c 100644 --- a/msa/js-executor/api/jsInvokeMessageProcessor.js +++ b/msa/js-executor/api/jsInvokeMessageProcessor.js @@ -15,18 +15,22 @@ */ 'use strict'; +const COMPILATION_ERROR = 0; +const RUNTIME_ERROR = 1; +const TIMEOUT_ERROR = 2; +const UNRECOGNIZED = -1; + const config = require('config'), - logger = require('../config/logger')('JsInvokeMessageProcessor'), + logger = require('../config/logger')._logger('JsInvokeMessageProcessor'), Utils = require('./utils'), - js = require('./jsinvoke.proto').js, - KeyedMessage = require('kafka-node').KeyedMessage, JsExecutor = require('./jsExecutor'); const scriptBodyTraceFrequency = Number(config.get('script.script_body_trace_frequency')); +const useSandbox = config.get('script.use_sandbox') === 'true'; function JsInvokeMessageProcessor(producer) { this.producer = producer; - this.executor = new JsExecutor(); + this.executor = new JsExecutor(useSandbox); this.scriptMap = {}; this.executedScriptsCounter = 0; } @@ -34,18 +38,22 @@ function JsInvokeMessageProcessor(producer) { JsInvokeMessageProcessor.prototype.onJsInvokeMessage = function(message) { var requestId; + var responseTopic; try { - var request = js.RemoteJsRequest.decode(message.value); - requestId = getRequestId(request); + var request = JSON.parse(message.value.toString('utf8')); + var buf = message.headers['requestId']; + requestId = Utils.UUIDFromBuffer(buf); + buf = message.headers['responseTopic']; + responseTopic = buf.toString('utf8'); - logger.debug('[%s] Received request, responseTopic: [%s]', requestId, request.responseTopic); + logger.debug('[%s] Received request, responseTopic: [%s]', requestId, responseTopic); if (request.compileRequest) { - this.processCompileRequest(requestId, request.responseTopic, request.compileRequest); + this.processCompileRequest(requestId, responseTopic, request.compileRequest); } else if (request.invokeRequest) { - this.processInvokeRequest(requestId, request.responseTopic, request.invokeRequest); + this.processInvokeRequest(requestId, responseTopic, request.invokeRequest); } else if (request.releaseRequest) { - this.processReleaseRequest(requestId, request.responseTopic, request.releaseRequest); + this.processReleaseRequest(requestId, responseTopic, request.releaseRequest); } else { logger.error('[%s] Unknown request recevied!', requestId); } @@ -68,7 +76,7 @@ JsInvokeMessageProcessor.prototype.processCompileRequest = function(requestId, r this.sendResponse(requestId, responseTopic, scriptId, compileResponse); }, (err) => { - var compileResponse = createCompileResponse(scriptId, false, js.JsInvokeErrorCode.COMPILATION_ERROR, err); + var compileResponse = createCompileResponse(scriptId, false, COMPILATION_ERROR, err); logger.debug('[%s] Sending failed compile response, scriptId: [%s]', requestId, scriptId); this.sendResponse(requestId, responseTopic, scriptId, compileResponse); } @@ -96,9 +104,9 @@ JsInvokeMessageProcessor.prototype.processInvokeRequest = function(requestId, re (err) => { var errorCode; if (err.message.includes('Script execution timed out')) { - errorCode = js.JsInvokeErrorCode.TIMEOUT_ERROR; + errorCode = TIMEOUT_ERROR; } else { - errorCode = js.JsInvokeErrorCode.RUNTIME_ERROR; + errorCode = RUNTIME_ERROR; } var invokeResponse = createInvokeResponse("", false, errorCode, err); logger.debug('[%s] Sending failed invoke response, scriptId: [%s], errorCode: [%s]', requestId, scriptId, errorCode); @@ -107,8 +115,8 @@ JsInvokeMessageProcessor.prototype.processInvokeRequest = function(requestId, re ) }, (err) => { - var invokeResponse = createInvokeResponse("", false, js.JsInvokeErrorCode.COMPILATION_ERROR, err); - logger.debug('[%s] Sending failed invoke response, scriptId: [%s], errorCode: [%s]', requestId, scriptId, js.JsInvokeErrorCode.COMPILATION_ERROR); + var invokeResponse = createInvokeResponse("", false, COMPILATION_ERROR, err); + logger.debug('[%s] Sending failed invoke response, scriptId: [%s], errorCode: [%s]', requestId, scriptId, COMPILATION_ERROR); this.sendResponse(requestId, responseTopic, scriptId, null, invokeResponse); } ); @@ -127,15 +135,26 @@ JsInvokeMessageProcessor.prototype.processReleaseRequest = function(requestId, r JsInvokeMessageProcessor.prototype.sendResponse = function (requestId, responseTopic, scriptId, compileResponse, invokeResponse, releaseResponse) { var remoteResponse = createRemoteResponse(requestId, compileResponse, invokeResponse, releaseResponse); - var rawResponse = js.RemoteJsResponse.encode(remoteResponse).finish(); - const message = new KeyedMessage(scriptId, rawResponse); - const payloads = [ { topic: responseTopic, messages: message, key: scriptId } ]; - this.producer.send(payloads, function (err, data) { - if (err) { - logger.error('[%s] Failed to send response to kafka: %s', requestId, err.message); - logger.error(err.stack); + var rawResponse = Buffer.from(JSON.stringify(remoteResponse), 'utf8'); + this.producer.send( + { + topic: responseTopic, + messages: [ + { + key: scriptId, + value: rawResponse + } + ] } - }); + ).then( + () => {}, + (err) => { + if (err) { + logger.error('[%s] Failed to send response to kafka: %s', requestId, err.message); + logger.error(err.stack); + } + } + ); } JsInvokeMessageProcessor.prototype.getOrCompileScript = function(scriptId, scriptBody) { @@ -159,50 +178,42 @@ JsInvokeMessageProcessor.prototype.getOrCompileScript = function(scriptId, scrip function createRemoteResponse(requestId, compileResponse, invokeResponse, releaseResponse) { const requestIdBits = Utils.UUIDToBits(requestId); - return js.RemoteJsResponse.create( - { + return { requestIdMSB: requestIdBits[0], requestIdLSB: requestIdBits[1], compileResponse: compileResponse, invokeResponse: invokeResponse, releaseResponse: releaseResponse - } - ); + }; } function createCompileResponse(scriptId, success, errorCode, err) { const scriptIdBits = Utils.UUIDToBits(scriptId); - return js.JsCompileResponse.create( - { + return { errorCode: errorCode, success: success, errorDetails: parseJsErrorDetails(err), scriptIdMSB: scriptIdBits[0], scriptIdLSB: scriptIdBits[1] - } - ); + }; } function createInvokeResponse(result, success, errorCode, err) { - return js.JsInvokeResponse.create( - { + return { errorCode: errorCode, success: success, errorDetails: parseJsErrorDetails(err), result: result - } - ); + }; } function createReleaseResponse(scriptId, success) { const scriptIdBits = Utils.UUIDToBits(scriptId); - return js.JsReleaseResponse.create( - { + return { success: success, scriptIdMSB: scriptIdBits[0], scriptIdLSB: scriptIdBits[1] - } - ); + }; } function parseJsErrorDetails(err) { @@ -229,8 +240,4 @@ function getScriptId(request) { return Utils.toUUIDString(request.scriptIdMSB, request.scriptIdLSB); } -function getRequestId(request) { - return Utils.toUUIDString(request.requestIdMSB, request.requestIdLSB); -} - -module.exports = JsInvokeMessageProcessor; \ No newline at end of file +module.exports = JsInvokeMessageProcessor; diff --git a/msa/js-executor/api/utils.js b/msa/js-executor/api/utils.js index 40ab74e721..2e0381086b 100644 --- a/msa/js-executor/api/utils.js +++ b/msa/js-executor/api/utils.js @@ -18,16 +18,17 @@ const Long = require('long'), uuidParse = require('uuid-parse'); -var logger = require('../config/logger')('Utils'); - exports.toUUIDString = function(mostSigBits, leastSigBits) { var msbBytes = Long.fromValue(mostSigBits, false).toBytes(false); var lsbBytes = Long.fromValue(leastSigBits, false).toBytes(false); var uuidBytes = msbBytes.concat(lsbBytes); - var buff = new Buffer(uuidBytes, 'utf8'); return uuidParse.unparse(uuidBytes); } +exports.UUIDFromBuffer = function(buf) { + return uuidParse.unparse(buf); +} + exports.UUIDToBits = function(uuidString) { const bytes = uuidParse.parse(uuidString); var msb = Long.fromBytes(bytes.slice(0,8), false, false).toString(); diff --git a/msa/js-executor/config/custom-environment-variables.yml b/msa/js-executor/config/custom-environment-variables.yml index 99f47cb5f1..c3caf15a9a 100644 --- a/msa/js-executor/config/custom-environment-variables.yml +++ b/msa/js-executor/config/custom-environment-variables.yml @@ -25,4 +25,5 @@ logger: filename: "LOGGER_FILENAME" script: + use_sandbox: "SCRIPT_USE_SANDBOX" script_body_trace_frequency: "SCRIPT_BODY_TRACE_FREQUENCY" diff --git a/msa/js-executor/config/default.yml b/msa/js-executor/config/default.yml index fb470567e7..9688722109 100644 --- a/msa/js-executor/config/default.yml +++ b/msa/js-executor/config/default.yml @@ -26,4 +26,5 @@ logger: filename: "tb-js-executor-%DATE%.log" script: + use_sandbox: "true" script_body_trace_frequency: "1000" diff --git a/msa/js-executor/config/logger.js b/msa/js-executor/config/logger.js index 4f5340c2a8..317dc0cdd5 100644 --- a/msa/js-executor/config/logger.js +++ b/msa/js-executor/config/logger.js @@ -17,9 +17,24 @@ var config = require('config'), path = require('path'), DailyRotateFile = require('winston-daily-rotate-file'); +const { logLevel } = require('kafkajs'); const { createLogger, format, transports } = require('winston'); const { combine, timestamp, label, printf, splat } = format; +const toWinstonLogLevel = level => { + switch(level) { + case logLevel.ERROR: + case logLevel.NOTHING: + return 'error' + case logLevel.WARN: + return 'warn' + case logLevel.INFO: + return 'info' + case logLevel.DEBUG: + return 'debug' + } +} + var loggerTransports = []; if (process.env.NODE_ENV !== 'production' || process.env.DOCKER_MODE === 'true') { @@ -56,4 +71,33 @@ function _logger(moduleLabel) { }); } -module.exports = _logger; \ No newline at end of file +const KafkaJsWinstonLogCreator = logLevel => { + const logger = createLogger({ + level: toWinstonLogLevel(logLevel), + format:combine( + splat(), + label({ label: 'kafkajs' }), + timestamp({format: 'YYYY-MM-DD HH:mm:ss,SSS'}), + printf(info => { + var res = `${info.timestamp} [${info.label}] ${info.level.toUpperCase()}: ${info.message}`; + if (info.extra) { + res +=`: ${JSON.stringify(info.extra)}`; + } + return res; + } + ) + ), + transports: loggerTransports + }); + + return ({ namespace, level, label, log }) => { + const { message, ...extra } = log; + logger.log({ + level: toWinstonLogLevel(level), + message, + extra, + }); + } +} + +module.exports = {_logger, KafkaJsWinstonLogCreator}; diff --git a/msa/js-executor/package-lock.json b/msa/js-executor/package-lock.json index 52ea28f0c7..96c8341882 100644 --- a/msa/js-executor/package-lock.json +++ b/msa/js-executor/package-lock.json @@ -35,60 +35,6 @@ "integrity": "sha512-shAmDyaQC4H92APFoIaVDHCx5bStIocgvbwQyxPRrbUY20V1EYTbSDchWbuwlMG3V17cprZhA6+78JfB+3DTPw==", "dev": true }, - "@protobufjs/aspromise": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", - "integrity": "sha1-m4sMxmPWaafY9vXQiToU00jzD78=" - }, - "@protobufjs/base64": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", - "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==" - }, - "@protobufjs/codegen": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", - "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==" - }, - "@protobufjs/eventemitter": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", - "integrity": "sha1-NVy8mLr61ZePntCV85diHx0Ga3A=" - }, - "@protobufjs/fetch": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", - "integrity": "sha1-upn7WYYUr2VwDBYZ/wbUVLDYTEU=", - "requires": { - "@protobufjs/aspromise": "^1.1.1", - "@protobufjs/inquire": "^1.1.0" - } - }, - "@protobufjs/float": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", - "integrity": "sha1-Xp4avctz/Ap8uLKR33jIy9l7h9E=" - }, - "@protobufjs/inquire": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", - "integrity": "sha1-/yAOPnzyQp4tyvwRQIKOjMY48Ik=" - }, - "@protobufjs/path": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", - "integrity": "sha1-bMKyDFya1q0NzP0hynZz2Nf79o0=" - }, - "@protobufjs/pool": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", - "integrity": "sha1-Cf0V8tbTq/qbZbw2ZQbWrXhG/1Q=" - }, - "@protobufjs/utf8": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", - "integrity": "sha1-p3c2C1s5oaLlEG+OhY8v0tBgxXA=" - }, "@types/events": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/@types/events/-/events-3.0.0.tgz", @@ -106,11 +52,6 @@ "@types/node": "*" } }, - "@types/long": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.0.tgz", - "integrity": "sha512-1w52Nyx4Gq47uuu0EVcsHBxZFJgurQ+rTKS3qMHxR1GY2T8c2AJYd6vZoZ9q1rupaDjU0yT+Jc2XTyXkjeMA+Q==" - }, "@types/minimatch": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.3.tgz", @@ -120,7 +61,8 @@ "@types/node": { "version": "10.12.10", "resolved": "https://registry.npmjs.org/@types/node/-/node-10.12.10.tgz", - "integrity": "sha512-8xZEYckCbUVgK8Eg7lf5Iy4COKJ5uXlnIOnePN0WUwSQggy9tolM+tDJf7wMOnT/JT/W9xDYIaYggt3mRV2O5w==" + "integrity": "sha512-8xZEYckCbUVgK8Eg7lf5Iy4COKJ5uXlnIOnePN0WUwSQggy9tolM+tDJf7wMOnT/JT/W9xDYIaYggt3mRV2O5w==", + "dev": true }, "abbrev": { "version": "1.1.1", @@ -182,11 +124,6 @@ } } }, - "ansi-regex": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", - "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=" - }, "ansi-styles": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", @@ -206,22 +143,6 @@ "normalize-path": "^2.1.1" } }, - "aproba": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz", - "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==", - "optional": true - }, - "are-we-there-yet": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.5.tgz", - "integrity": "sha512-5hYdAkZlcG8tOLujVDTgCT+uPX0VnpAH28gWsLfzpXYm7wP6mp5Q/gYyR7YQ0cKVJcXJnl3j2kpBan13PtQf6w==", - "optional": true, - "requires": { - "delegates": "^1.0.0", - "readable-stream": "^2.0.6" - } - }, "argparse": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", @@ -331,7 +252,8 @@ "balanced-match": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", - "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" + "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", + "dev": true }, "base": { "version": "0.11.2", @@ -397,39 +319,12 @@ "tweetnacl": "^0.14.3" } }, - "binary": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/binary/-/binary-0.3.0.tgz", - "integrity": "sha1-n2BVO8XOjDOG87VTz/R0Yq3sqnk=", - "requires": { - "buffers": "~0.1.1", - "chainsaw": "~0.1.0" - } - }, "binary-extensions": { "version": "1.12.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-1.12.0.tgz", "integrity": "sha512-DYWGk01lDcxeS/K9IHPGWfT8PsJmbXRtRd2Sx72Tnb8pcYZQFF1oSDb8hJtS1vhp212q1Rzi5dUf9+nq0o9UIg==", "dev": true }, - "bindings": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", - "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", - "optional": true, - "requires": { - "file-uri-to-path": "1.0.0" - } - }, - "bl": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/bl/-/bl-2.2.0.tgz", - "integrity": "sha512-wbgvOpqopSr7uq6fJrLH8EsvYMJf9gzfo2jCsL2eTy75qXPukA4pCgHamOQkZtY5vmfVtjB+P3LNlMHW5CEZXA==", - "requires": { - "readable-stream": "^2.3.5", - "safe-buffer": "^5.1.1" - } - }, "boxen": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/boxen/-/boxen-1.3.0.tgz", @@ -482,6 +377,7 @@ "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -516,53 +412,6 @@ } } }, - "buffer-alloc": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/buffer-alloc/-/buffer-alloc-1.2.0.tgz", - "integrity": "sha512-CFsHQgjtW1UChdXgbyJGtnm+O/uLQeZdtbDo8mfUgYXCHSM1wgrVxXm6bSyrUuErEb+4sYVGCzASBRot7zyrow==", - "optional": true, - "requires": { - "buffer-alloc-unsafe": "^1.1.0", - "buffer-fill": "^1.0.0" - } - }, - "buffer-alloc-unsafe": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/buffer-alloc-unsafe/-/buffer-alloc-unsafe-1.1.0.tgz", - "integrity": "sha512-TEM2iMIEQdJ2yjPJoSIsldnleVaAk1oW3DBVUykyOLsEsFmEc9kn+SFFPz+gl54KQNxlDnAwCXosOS9Okx2xAg==", - "optional": true - }, - "buffer-crc32": { - "version": "0.2.13", - "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", - "integrity": "sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI=" - }, - "buffer-fill": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/buffer-fill/-/buffer-fill-1.0.0.tgz", - "integrity": "sha1-+PeLdniYiO858gXNY39o5wISKyw=", - "optional": true - }, - "buffermaker": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/buffermaker/-/buffermaker-1.2.1.tgz", - "integrity": "sha512-IdnyU2jDHU65U63JuVQNTHiWjPRH0CS3aYd/WPaEwyX84rFdukhOduAVb1jwUScmb5X0JWPw8NZOrhoLMiyAHQ==", - "requires": { - "long": "1.1.2" - }, - "dependencies": { - "long": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/long/-/long-1.1.2.tgz", - "integrity": "sha1-6u9ZUcp1UdlpJrgtokLbnWso+1M=" - } - } - }, - "buffers": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz", - "integrity": "sha1-skV5w77U1tOWru5tmorn9Ugqt7s=" - }, "byline": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/byline/-/byline-5.0.0.tgz", @@ -610,14 +459,6 @@ "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=", "dev": true }, - "chainsaw": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/chainsaw/-/chainsaw-0.1.0.tgz", - "integrity": "sha1-XqtQsor+WAdNDVgpE4iCi15fvJg=", - "requires": { - "traverse": ">=0.3.0 <0.4" - } - }, "chalk": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.1.tgz", @@ -650,12 +491,6 @@ "upath": "^1.0.5" } }, - "chownr": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.2.tgz", - "integrity": "sha512-GkfeAQh+QNy3wquu9oIZr6SS5x7wGdSgNQvD10X3r+AZr1Oys22HW8kAmDMvNg2+Dm0TeGaEuO8gFwdBXxwO8A==", - "optional": true - }, "ci-info": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-1.6.0.tgz", @@ -691,11 +526,6 @@ "integrity": "sha1-T6kXw+WclKAEzWH47lCdplFocUM=", "dev": true }, - "code-point-at": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", - "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=" - }, "collection-visit": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/collection-visit/-/collection-visit-1.0.0.tgz", @@ -774,7 +604,8 @@ "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", + "dev": true }, "config": { "version": "3.2.2", @@ -798,11 +629,6 @@ "xdg-basedir": "^3.0.0" } }, - "console-control-strings": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz", - "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4=" - }, "copy-descriptor": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/copy-descriptor/-/copy-descriptor-0.1.1.tgz", @@ -858,6 +684,7 @@ "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dev": true, "requires": { "ms": "2.0.0" } @@ -868,19 +695,11 @@ "integrity": "sha1-6zkTMzRYd1y4TNGh+uBiEGu4dUU=", "dev": true }, - "decompress-response": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-3.3.0.tgz", - "integrity": "sha1-gKTdMjdIOEv6JICDYirt7Jgq3/M=", - "optional": true, - "requires": { - "mimic-response": "^1.0.0" - } - }, "deep-extend": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", - "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==" + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", + "dev": true }, "deep-is": { "version": "0.1.3", @@ -935,23 +754,6 @@ "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=", "dev": true }, - "delegates": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", - "integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=", - "optional": true - }, - "denque": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/denque/-/denque-1.4.1.tgz", - "integrity": "sha512-OfzPuSZKGcgr96rf1oODnfjqBFmr1DVoc/TrItj3Ohe0Ah1C5WX5Baquw/9U9KovnQ88EqmJbD66rKYUQYN1tQ==" - }, - "detect-libc": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz", - "integrity": "sha1-+hN8S9aY7fVc1c0CrFWfkaTEups=", - "optional": true - }, "diagnostics": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/diagnostics/-/diagnostics-1.1.1.tgz", @@ -1004,14 +806,6 @@ "env-variable": "0.0.x" } }, - "end-of-stream": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.1.tgz", - "integrity": "sha512-1MkrZNvWTKCaigbn+W15elq2BB/L22nqrSY5DKlo3X6+vclJm8Bb5djXJBmEX6fS3+zCh/F4VBK5Z2KxJt4s2Q==", - "requires": { - "once": "^1.4.0" - } - }, "env-variable": { "version": "0.0.5", "resolved": "https://registry.npmjs.org/env-variable/-/env-variable-0.0.5.tgz", @@ -1272,12 +1066,6 @@ "moment": "^2.11.2" } }, - "file-uri-to-path": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", - "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==", - "optional": true - }, "fill-range": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-4.0.0.tgz", @@ -1343,12 +1131,6 @@ "readable-stream": "^2.0.0" } }, - "fs-constants": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", - "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", - "optional": true - }, "fs-extra": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-6.0.1.tgz", @@ -1895,22 +1677,6 @@ } } }, - "gauge": { - "version": "2.7.4", - "resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz", - "integrity": "sha1-LANAXHU4w51+s3sxcCLjJfsBi/c=", - "optional": true, - "requires": { - "aproba": "^1.0.3", - "console-control-strings": "^1.0.0", - "has-unicode": "^2.0.0", - "object-assign": "^4.1.0", - "signal-exit": "^3.0.0", - "string-width": "^1.0.1", - "strip-ansi": "^3.0.1", - "wide-align": "^1.1.0" - } - }, "get-stream": { "version": "3.0.0", "resolved": "http://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", @@ -1932,12 +1698,6 @@ "assert-plus": "^1.0.0" } }, - "github-from-package": { - "version": "0.0.0", - "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", - "integrity": "sha1-l/tdlr/eiXMxPyDoKI75oWf6ZM4=", - "optional": true - }, "glob": { "version": "7.1.4", "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.4.tgz", @@ -2059,12 +1819,6 @@ "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", "dev": true }, - "has-unicode": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", - "integrity": "sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk=", - "optional": true - }, "has-value": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/has-value/-/has-value-1.0.0.tgz", @@ -2150,7 +1904,8 @@ "ini": { "version": "1.3.5", "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz", - "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==" + "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==", + "dev": true }, "into-stream": { "version": "5.1.0", @@ -2262,14 +2017,6 @@ "integrity": "sha1-qIwCU1eR8C7TfHahueqXc8gz+MI=", "dev": true }, - "is-fullwidth-code-point": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", - "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=", - "requires": { - "number-is-nan": "^1.0.0" - } - }, "is-glob": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.0.tgz", @@ -2453,42 +2200,12 @@ "verror": "1.10.0" } }, - "kafka-node": { - "version": "4.1.3", - "resolved": "https://registry.npmjs.org/kafka-node/-/kafka-node-4.1.3.tgz", - "integrity": "sha512-C2WHksRCr7vIKmbxYaCk2c5Q1lnHIi6C0f3AioK3ARcRHGO9DpqErcoaS9d8PP62yzTnkYras+iAlmPsZHNSfw==", + "kafkajs": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/kafkajs/-/kafkajs-1.11.0.tgz", + "integrity": "sha512-dLRCcFIBygZucR+e8U2ZqH2wgMrAu114K0szUyUseJoeOii3cG5bHZPIdqKecXxI6begPVCfGS3R0nJY4zHW2A==", "requires": { - "async": "^2.6.2", - "binary": "~0.3.0", - "bl": "^2.2.0", - "buffer-crc32": "~0.2.5", - "buffermaker": "~1.2.0", - "debug": "^2.1.3", - "denque": "^1.3.0", - "lodash": "^4.17.4", - "minimatch": "^3.0.2", - "nested-error-stacks": "^2.0.0", - "optional": "^0.1.3", - "retry": "^0.10.1", - "snappy": "^6.0.1", - "uuid": "^3.0.0" - }, - "dependencies": { - "async": { - "version": "2.6.3", - "resolved": "https://registry.npmjs.org/async/-/async-2.6.3.tgz", - "integrity": "sha512-zflvls11DCy+dQWzTW2dzuilv8Z5X/pjfmZOWba6TNIVDm+2UDaJmXSOXlasHKfNBs8oo3M0aT50fDEWfKZjXg==", - "requires": { - "lodash": "^4.17.14" - }, - "dependencies": { - "lodash": { - "version": "4.17.15", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", - "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==" - } - } - } + "long": "^4.0.0" } }, "kind-of": { @@ -2641,16 +2358,11 @@ "mime-db": "1.40.0" } }, - "mimic-response": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz", - "integrity": "sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==", - "optional": true - }, "minimatch": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "dev": true, "requires": { "brace-expansion": "^1.1.7" } @@ -2685,6 +2397,7 @@ "version": "0.5.1", "resolved": "http://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", + "dev": true, "requires": { "minimist": "0.0.8" }, @@ -2692,7 +2405,8 @@ "minimist": { "version": "0.0.8", "resolved": "http://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", - "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=" + "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=", + "dev": true } } }, @@ -2704,7 +2418,8 @@ "ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", + "dev": true }, "multistream": { "version": "2.1.1", @@ -2742,26 +2457,6 @@ "to-regex": "^3.0.1" } }, - "napi-build-utils": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.1.tgz", - "integrity": "sha512-boQj1WFgQH3v4clhu3mTNfP+vOBxorDlE8EKiMjUlLG3C4qAESnn9AxIOkFgTR2c9LtzNjPrjS60cT27ZKBhaA==", - "optional": true - }, - "nested-error-stacks": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/nested-error-stacks/-/nested-error-stacks-2.1.0.tgz", - "integrity": "sha512-AO81vsIO1k1sM4Zrd6Hu7regmJN1NSiAja10gc4bX3F0wd+9rQmcuHQaHVQCYIEC8iFXnE+mavh23GOt7wBgug==" - }, - "node-abi": { - "version": "2.9.0", - "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-2.9.0.tgz", - "integrity": "sha512-jmEOvv0eanWjhX8dX1pmjb7oJl1U1oR4FOh0b2GnvALwSYoOdU7sj+kLDSAyjo4pfC9aj/IxkloxdLJQhSSQBA==", - "optional": true, - "requires": { - "semver": "^5.4.1" - } - }, "nodemon": { "version": "1.18.7", "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-1.18.7.tgz", @@ -2797,12 +2492,6 @@ } } }, - "noop-logger": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/noop-logger/-/noop-logger-0.1.1.tgz", - "integrity": "sha1-lKKxYzxPExdVMAfYlm/Q6EG2pMI=", - "optional": true - }, "nopt": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/nopt/-/nopt-1.0.10.tgz", @@ -2830,35 +2519,12 @@ "path-key": "^2.0.0" } }, - "npmlog": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-4.1.2.tgz", - "integrity": "sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==", - "optional": true, - "requires": { - "are-we-there-yet": "~1.1.2", - "console-control-strings": "~1.1.0", - "gauge": "~2.7.3", - "set-blocking": "~2.0.0" - } - }, - "number-is-nan": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz", - "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=" - }, "oauth-sign": { "version": "0.9.0", "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz", "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==", "dev": true }, - "object-assign": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", - "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=", - "optional": true - }, "object-copy": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/object-copy/-/object-copy-0.1.0.tgz", @@ -2917,6 +2583,7 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", + "dev": true, "requires": { "wrappy": "1" } @@ -2926,11 +2593,6 @@ "resolved": "https://registry.npmjs.org/one-time/-/one-time-0.0.4.tgz", "integrity": "sha1-+M33eISCb+Tf+T46nMN7HkSAdC4=" }, - "optional": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/optional/-/optional-0.1.4.tgz", - "integrity": "sha512-gtvrrCfkE08wKcgXaVwQVgwEQ8vel2dc5DDBn9RLQZ3YtmtkBss6A2HY6BnJH4N/4Ku97Ri/SF8sNWE2225WJw==" - }, "optionator": { "version": "0.8.2", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.2.tgz", @@ -2945,12 +2607,6 @@ "wordwrap": "~1.0.0" } }, - "os-homedir": { - "version": "1.0.2", - "resolved": "http://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz", - "integrity": "sha1-/7xJiDNuDoM94MFox+8VISGqf7M=", - "optional": true - }, "os-tmpdir": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", @@ -3127,38 +2783,6 @@ "integrity": "sha1-AerA/jta9xoqbAL+q7jB/vfgDqs=", "dev": true }, - "prebuild-install": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-5.3.0.tgz", - "integrity": "sha512-aaLVANlj4HgZweKttFNUVNRxDukytuIuxeK2boIMHjagNJCiVKWFsKF4tCE3ql3GbrD2tExPQ7/pwtEJcHNZeg==", - "optional": true, - "requires": { - "detect-libc": "^1.0.3", - "expand-template": "^2.0.3", - "github-from-package": "0.0.0", - "minimist": "^1.2.0", - "mkdirp": "^0.5.1", - "napi-build-utils": "^1.0.1", - "node-abi": "^2.7.0", - "noop-logger": "^0.1.1", - "npmlog": "^4.0.1", - "os-homedir": "^1.0.1", - "pump": "^2.0.1", - "rc": "^1.2.7", - "simple-get": "^2.7.0", - "tar-fs": "^1.13.0", - "tunnel-agent": "^0.6.0", - "which-pm-runs": "^1.0.0" - }, - "dependencies": { - "expand-template": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", - "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==", - "optional": true - } - } - }, "prelude-ls": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", @@ -3182,26 +2806,6 @@ "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", "dev": true }, - "protobufjs": { - "version": "6.8.8", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.8.8.tgz", - "integrity": "sha512-AAmHtD5pXgZfi7GMpllpO3q1Xw1OYldr+dMUlAnffGTAhqkg72WdmSY71uKBF/JuyiKs8psYbtKrhi0ASCD8qw==", - "requires": { - "@protobufjs/aspromise": "^1.1.2", - "@protobufjs/base64": "^1.1.2", - "@protobufjs/codegen": "^2.0.4", - "@protobufjs/eventemitter": "^1.1.0", - "@protobufjs/fetch": "^1.1.0", - "@protobufjs/float": "^1.0.2", - "@protobufjs/inquire": "^1.1.0", - "@protobufjs/path": "^1.1.2", - "@protobufjs/pool": "^1.1.0", - "@protobufjs/utf8": "^1.1.0", - "@types/long": "^4.0.0", - "@types/node": "^10.1.0", - "long": "^4.0.0" - } - }, "pseudomap": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", @@ -3220,16 +2824,6 @@ "integrity": "sha512-vL6NLxNHzkNTjGJUpMm5PLC+94/0tTlC1vkP9bdU0pOHih+EujMjgMTwfZopZvHWRFbqJ5Y73OMoau50PewDDA==", "dev": true }, - "pump": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/pump/-/pump-2.0.1.tgz", - "integrity": "sha512-ruPMNRkN3MHP1cWJc9OWr+T/xDP0jhXYCLfJcBuX54hhfIBnaQmAUMfDcG4DM5UMWByBbJY69QSphm3jtDKIkA==", - "optional": true, - "requires": { - "end-of-stream": "^1.1.0", - "once": "^1.3.1" - } - }, "punycode": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", @@ -3246,6 +2840,7 @@ "version": "1.2.8", "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "dev": true, "requires": { "deep-extend": "^0.6.0", "ini": "~1.3.0", @@ -3389,11 +2984,6 @@ "integrity": "sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg==", "dev": true }, - "retry": { - "version": "0.10.1", - "resolved": "https://registry.npmjs.org/retry/-/retry-0.10.1.tgz", - "integrity": "sha1-52OI0heZLCUnUCQdPTlW/tmNj/Q=" - }, "safe-buffer": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", @@ -3428,12 +3018,6 @@ "semver": "^5.0.3" } }, - "set-blocking": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", - "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=", - "optional": true - }, "set-value": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/set-value/-/set-value-2.0.0.tgz", @@ -3475,24 +3059,8 @@ "signal-exit": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz", - "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=" - }, - "simple-concat": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.0.tgz", - "integrity": "sha1-c0TLuLbib7J9ZrL8hvn21Zl1IcY=", - "optional": true - }, - "simple-get": { - "version": "2.8.1", - "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-2.8.1.tgz", - "integrity": "sha512-lSSHRSw3mQNUGPAYRqo7xy9dhKmxFXIjLjp4KHpf99GEH2VH7C3AM+Qfx6du6jhfUi6Vm7XnbEVEf7Wb6N8jRw==", - "optional": true, - "requires": { - "decompress-response": "^3.3.0", - "once": "^1.3.1", - "simple-concat": "^1.0.0" - } + "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=", + "dev": true }, "simple-swizzle": { "version": "0.2.2", @@ -3615,25 +3183,6 @@ } } }, - "snappy": { - "version": "6.2.3", - "resolved": "https://registry.npmjs.org/snappy/-/snappy-6.2.3.tgz", - "integrity": "sha512-HZpVoIxMfQ4fL3iDuMdI1R5xycw1o9YDCAndTKZCY/EHRoKFvzwplttuBBVGeEg2fd1hYiwAXos/sM24W7N1LA==", - "optional": true, - "requires": { - "bindings": "^1.3.1", - "nan": "^2.14.0", - "prebuild-install": "^5.2.2" - }, - "dependencies": { - "nan": { - "version": "2.14.0", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.0.tgz", - "integrity": "sha512-INOFj37C7k3AfaNTtX8RhsTw7qRy7eLET14cROi9+5HAVbbHuIWUHEauBv5qT4Av2tWasiTY1Jw6puUNqRJXQg==", - "optional": true - } - } - }, "source-map": { "version": "0.5.7", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz", @@ -3725,16 +3274,6 @@ "readable-stream": "^2.1.4" } }, - "string-width": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", - "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", - "requires": { - "code-point-at": "^1.0.0", - "is-fullwidth-code-point": "^1.0.0", - "strip-ansi": "^3.0.0" - } - }, "string_decoder": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", @@ -3743,14 +3282,6 @@ "safe-buffer": "~5.1.0" } }, - "strip-ansi": { - "version": "3.0.1", - "resolved": "http://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", - "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", - "requires": { - "ansi-regex": "^2.0.0" - } - }, "strip-eof": { "version": "1.0.0", "resolved": "http://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz", @@ -3760,7 +3291,8 @@ "strip-json-comments": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", - "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=" + "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=", + "dev": true }, "supports-color": { "version": "5.5.0", @@ -3771,57 +3303,6 @@ "has-flag": "^3.0.0" } }, - "tar-fs": { - "version": "1.16.3", - "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-1.16.3.tgz", - "integrity": "sha512-NvCeXpYx7OsmOh8zIOP/ebG55zZmxLE0etfWRbWok+q2Qo8x/vOR/IJT1taADXPe+jsiu9axDb3X4B+iIgNlKw==", - "optional": true, - "requires": { - "chownr": "^1.0.1", - "mkdirp": "^0.5.1", - "pump": "^1.0.0", - "tar-stream": "^1.1.2" - }, - "dependencies": { - "pump": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/pump/-/pump-1.0.3.tgz", - "integrity": "sha512-8k0JupWme55+9tCVE+FS5ULT3K6AbgqrGa58lTT49RpyfwwcGedHqaC5LlQNdEAumn/wFsu6aPwkuPMioy8kqw==", - "optional": true, - "requires": { - "end-of-stream": "^1.1.0", - "once": "^1.3.1" - } - } - } - }, - "tar-stream": { - "version": "1.6.2", - "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-1.6.2.tgz", - "integrity": "sha512-rzS0heiNf8Xn7/mpdSVVSMAWAoy9bfb1WOTYC78Z0UQKeKa/CWS8FOq0lKGNa8DWKAn9gxjCvMLYc5PGXYlK2A==", - "optional": true, - "requires": { - "bl": "^1.0.0", - "buffer-alloc": "^1.2.0", - "end-of-stream": "^1.0.0", - "fs-constants": "^1.0.0", - "readable-stream": "^2.3.0", - "to-buffer": "^1.1.1", - "xtend": "^4.0.0" - }, - "dependencies": { - "bl": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/bl/-/bl-1.2.2.tgz", - "integrity": "sha512-e8tQYnZodmebYDWGH7KMRvtzKXaJHx3BbilrgZCfvyLUYdKpK1t5PSPmpkny/SgiTSCnjfLW7v5rlONXVFkQEA==", - "optional": true, - "requires": { - "readable-stream": "^2.3.5", - "safe-buffer": "^5.1.1" - } - } - } - }, "term-size": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/term-size/-/term-size-1.2.0.tgz", @@ -3848,12 +3329,6 @@ "integrity": "sha1-8y6srFoXW+ol1/q1Zas+2HQe9W8=", "dev": true }, - "to-buffer": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/to-buffer/-/to-buffer-1.1.1.tgz", - "integrity": "sha512-lx9B5iv7msuFYE3dytT+KE5tap+rNYw+K4jVkb9R/asAb+pbBSM17jtunHplhBe6RRJdZx3Pn2Jph24O32mOVg==", - "optional": true - }, "to-object-path": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/to-object-path/-/to-object-path-0.3.0.tgz", @@ -3923,11 +3398,6 @@ } } }, - "traverse": { - "version": "0.3.9", - "resolved": "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz", - "integrity": "sha1-cXuPIgzAu3tE5AUUwisui7xw2Lk=" - }, "triple-beam": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/triple-beam/-/triple-beam-1.3.0.tgz", @@ -3937,6 +3407,7 @@ "version": "0.6.0", "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", + "dev": true, "requires": { "safe-buffer": "^5.0.1" } @@ -4140,7 +3611,8 @@ "uuid": { "version": "3.3.2", "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", - "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" + "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==", + "dev": true }, "uuid-parse": { "version": "1.0.0", @@ -4167,21 +3639,6 @@ "isexe": "^2.0.0" } }, - "which-pm-runs": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/which-pm-runs/-/which-pm-runs-1.0.0.tgz", - "integrity": "sha1-Zws6+8VS4LVd9rd4DKdGFfI60cs=", - "optional": true - }, - "wide-align": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.3.tgz", - "integrity": "sha512-QGkOQc8XL6Bt5PwnsExKBPuMKBxnGxWWW3fU55Xt4feHozMUhdUMaBCk290qpm/wG5u/RSKzwdAC4i51YigihA==", - "optional": true, - "requires": { - "string-width": "^1.0.2 || 2" - } - }, "widest-line": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/widest-line/-/widest-line-2.0.1.tgz", @@ -4281,7 +3738,8 @@ "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", + "dev": true }, "write-file-atomic": { "version": "2.3.0", @@ -4300,12 +3758,6 @@ "integrity": "sha1-SWsswQnsqNus/i3HK2A8F8WHCtQ=", "dev": true }, - "xtend": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", - "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", - "optional": true - }, "yallist": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.0.3.tgz", diff --git a/msa/js-executor/package.json b/msa/js-executor/package.json index dbbfe6aea1..da4eb09b09 100644 --- a/msa/js-executor/package.json +++ b/msa/js-executor/package.json @@ -6,18 +6,16 @@ "main": "server.js", "bin": "server.js", "scripts": { - "build-proto": "pbjs -t static-module -w commonjs -o ./api/jsinvoke.proto.js ../../application/src/main/proto/jsinvoke.proto", - "install": "npm run build-proto && pkg -t node10-linux-x64,node10-win-x64 --out-path ./target . && node install.js", + "install": "pkg -t node10-linux-x64,node10-win-x64 --out-path ./target . && node install.js", "test": "echo \"Error: no test specified\" && exit 1", - "start": "npm run build-proto && nodemon server.js", - "start-prod": "npm run build-proto && NODE_ENV=production nodemon server.js" + "start": "nodemon server.js", + "start-prod": "NODE_ENV=production nodemon server.js" }, "dependencies": { "config": "^3.2.2", "js-yaml": "^3.12.0", - "kafka-node": "^4.1.3", + "kafkajs": "^1.11.0", "long": "^4.0.0", - "protobufjs": "^6.8.8", "uuid-parse": "^1.0.0", "winston": "^3.0.0", "winston-daily-rotate-file": "^3.2.1" diff --git a/msa/js-executor/server.js b/msa/js-executor/server.js index 8a0912bcc2..6f322f0892 100644 --- a/msa/js-executor/server.js +++ b/msa/js-executor/server.js @@ -13,14 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +const { logLevel, Kafka } = require('kafkajs'); + const config = require('config'), - kafka = require('kafka-node'), - ConsumerGroup = kafka.ConsumerGroup, - Producer = kafka.Producer, JsInvokeMessageProcessor = require('./api/jsInvokeMessageProcessor'), - logger = require('./config/logger')('main'); + logger = require('./config/logger')._logger('main'), + KafkaJsWinstonLogCreator = require('./config/logger').KafkaJsWinstonLogCreator; var kafkaClient; +var consumer; +var producer; (async() => { try { @@ -32,49 +35,24 @@ var kafkaClient; logger.info('Kafka Bootstrap Servers: %s', kafkaBootstrapServers); logger.info('Kafka Requests Topic: %s', kafkaRequestTopic); - kafkaClient = new kafka.KafkaClient({kafkaHost: kafkaBootstrapServers}); - - var consumer = new ConsumerGroup( - { - kafkaHost: kafkaBootstrapServers, - groupId: 'js-executor-group', - autoCommit: true, - encoding: 'buffer' - }, - kafkaRequestTopic - ); - - consumer.on('error', (err) => { - logger.error('Unexpected kafka consumer error: %s', err.message); - logger.error(err.stack); + kafkaClient = new Kafka({ + brokers: kafkaBootstrapServers.split(','), + logLevel: logLevel.INFO, + logCreator: KafkaJsWinstonLogCreator }); - consumer.on('offsetOutOfRange', (err) => { - logger.error('Offset out of range error: %s', err.message); - logger.error(err.stack); - }); + consumer = kafkaClient.consumer({ groupId: 'js-executor-group' }); + producer = kafkaClient.producer(); + const messageProcessor = new JsInvokeMessageProcessor(producer); + await consumer.connect(); + await producer.connect(); + await consumer.subscribe({ topic: kafkaRequestTopic}); - consumer.on('rebalancing', () => { - logger.info('Rebalancing event received.'); - }) - - consumer.on('rebalanced', () => { - logger.info('Rebalanced event received.'); - }); - - var producer = new Producer(kafkaClient); - producer.on('error', (err) => { - logger.error('Unexpected kafka producer error: %s', err.message); - logger.error(err.stack); - }); - - var messageProcessor = new JsInvokeMessageProcessor(producer); - - producer.on('ready', () => { - consumer.on('message', (message) => { + logger.info('Started ThingsBoard JavaScript Executor Microservice.'); + await consumer.run({ + eachMessage: async ({ topic, partition, message }) => { messageProcessor.onJsInvokeMessage(message); - }); - logger.info('Started ThingsBoard JavaScript Executor Microservice.'); + }, }); } catch (e) { @@ -84,21 +62,41 @@ var kafkaClient; } })(); -process.on('exit', function () { +process.on('exit', () => { exit(0); }); -function exit(status) { +async function exit(status) { logger.info('Exiting with status: %d ...', status); - if (kafkaClient) { - logger.info('Stopping Kafka Client...'); - var _kafkaClient = kafkaClient; - kafkaClient = null; - _kafkaClient.close(() => { - logger.info('Kafka Client stopped.'); + if (consumer) { + logger.info('Stopping Kafka Consumer...'); + var _consumer = consumer; + consumer = null; + try { + await _consumer.disconnect(); + logger.info('Kafka Consumer stopped.'); + await disconnectProducer(); process.exit(status); - }); + } catch (e) { + logger.info('Kafka Consumer stop error.'); + await disconnectProducer(); + process.exit(status); + } } else { process.exit(status); } } + +async function disconnectProducer() { + if (producer) { + logger.info('Stopping Kafka Producer...'); + var _producer = producer; + producer = null; + try { + await _producer.disconnect(); + logger.info('Kafka Producer stopped.'); + } catch (e) { + logger.info('Kafka Producer stop error.'); + } + } +}