Improve JS Executor

This commit is contained in:
Igor Kulikov 2019-12-03 14:45:59 +02:00
parent 5d64a27079
commit 562917649c
10 changed files with 237 additions and 693 deletions

View File

@ -203,7 +203,7 @@ sql:
attributes:
batch_size: "${SQL_ATTRIBUTES_BATCH_SIZE:10000}"
batch_max_delay: "${SQL_ATTRIBUTES_BATCH_MAX_DELAY_MS:100}"
stats_print_interval_ms: "${SQL_ATTRIBUTES_BATCH_STATS_PRINT_MS:1000}"
stats_print_interval_ms: "${SQL_ATTRIBUTES_BATCH_STATS_PRINT_MS:10000}"
# Specify whether to remove null characters from strValue of attributes and timeseries before insert
remove_null_chars: "${SQL_REMOVE_NULL_CHARS:true}"

View File

@ -17,11 +17,28 @@
const vm = require('vm');
function JsExecutor() {
function JsExecutor(useSandbox) {
this.useSandbox = useSandbox;
}
JsExecutor.prototype.compileScript = function(code) {
return new Promise(function(resolve, reject) {
if (this.useSandbox) {
return createScript(code);
} else {
return createFunction(code);
}
}
JsExecutor.prototype.executeScript = function(script, args, timeout) {
if (this.useSandbox) {
return invokeScript(script, args, timeout);
} else {
return invokeFunction(script, args);
}
}
function createScript(code) {
return new Promise((resolve, reject) => {
try {
code = "("+code+")(...args)";
var script = new vm.Script(code);
@ -32,8 +49,8 @@ JsExecutor.prototype.compileScript = function(code) {
});
}
JsExecutor.prototype.executeScript = function(script, args, timeout) {
return new Promise(function(resolve, reject) {
function invokeScript(script, args, timeout) {
return new Promise((resolve, reject) => {
try {
var sandbox = Object.create(null);
sandbox.args = args;
@ -45,4 +62,29 @@ JsExecutor.prototype.executeScript = function(script, args, timeout) {
});
}
function createFunction(code) {
return new Promise((resolve, reject) => {
try {
code = "return ("+code+")(...args)";
const parsingContext = vm.createContext({});
const func = vm.compileFunction(code, ['args'], {parsingContext: parsingContext});
resolve(func);
} catch (err) {
reject(err);
}
});
}
function invokeFunction(func, args) {
return new Promise((resolve, reject) => {
try {
var result = func(args);
resolve(result);
} catch (err) {
reject(err);
}
});
}
module.exports = JsExecutor;

View File

@ -15,18 +15,22 @@
*/
'use strict';
const COMPILATION_ERROR = 0;
const RUNTIME_ERROR = 1;
const TIMEOUT_ERROR = 2;
const UNRECOGNIZED = -1;
const config = require('config'),
logger = require('../config/logger')('JsInvokeMessageProcessor'),
logger = require('../config/logger')._logger('JsInvokeMessageProcessor'),
Utils = require('./utils'),
js = require('./jsinvoke.proto').js,
KeyedMessage = require('kafka-node').KeyedMessage,
JsExecutor = require('./jsExecutor');
const scriptBodyTraceFrequency = Number(config.get('script.script_body_trace_frequency'));
const useSandbox = config.get('script.use_sandbox') === 'true';
function JsInvokeMessageProcessor(producer) {
this.producer = producer;
this.executor = new JsExecutor();
this.executor = new JsExecutor(useSandbox);
this.scriptMap = {};
this.executedScriptsCounter = 0;
}
@ -34,18 +38,22 @@ function JsInvokeMessageProcessor(producer) {
JsInvokeMessageProcessor.prototype.onJsInvokeMessage = function(message) {
var requestId;
var responseTopic;
try {
var request = js.RemoteJsRequest.decode(message.value);
requestId = getRequestId(request);
var request = JSON.parse(message.value.toString('utf8'));
var buf = message.headers['requestId'];
requestId = Utils.UUIDFromBuffer(buf);
buf = message.headers['responseTopic'];
responseTopic = buf.toString('utf8');
logger.debug('[%s] Received request, responseTopic: [%s]', requestId, request.responseTopic);
logger.debug('[%s] Received request, responseTopic: [%s]', requestId, responseTopic);
if (request.compileRequest) {
this.processCompileRequest(requestId, request.responseTopic, request.compileRequest);
this.processCompileRequest(requestId, responseTopic, request.compileRequest);
} else if (request.invokeRequest) {
this.processInvokeRequest(requestId, request.responseTopic, request.invokeRequest);
this.processInvokeRequest(requestId, responseTopic, request.invokeRequest);
} else if (request.releaseRequest) {
this.processReleaseRequest(requestId, request.responseTopic, request.releaseRequest);
this.processReleaseRequest(requestId, responseTopic, request.releaseRequest);
} else {
logger.error('[%s] Unknown request recevied!', requestId);
}
@ -68,7 +76,7 @@ JsInvokeMessageProcessor.prototype.processCompileRequest = function(requestId, r
this.sendResponse(requestId, responseTopic, scriptId, compileResponse);
},
(err) => {
var compileResponse = createCompileResponse(scriptId, false, js.JsInvokeErrorCode.COMPILATION_ERROR, err);
var compileResponse = createCompileResponse(scriptId, false, COMPILATION_ERROR, err);
logger.debug('[%s] Sending failed compile response, scriptId: [%s]', requestId, scriptId);
this.sendResponse(requestId, responseTopic, scriptId, compileResponse);
}
@ -96,9 +104,9 @@ JsInvokeMessageProcessor.prototype.processInvokeRequest = function(requestId, re
(err) => {
var errorCode;
if (err.message.includes('Script execution timed out')) {
errorCode = js.JsInvokeErrorCode.TIMEOUT_ERROR;
errorCode = TIMEOUT_ERROR;
} else {
errorCode = js.JsInvokeErrorCode.RUNTIME_ERROR;
errorCode = RUNTIME_ERROR;
}
var invokeResponse = createInvokeResponse("", false, errorCode, err);
logger.debug('[%s] Sending failed invoke response, scriptId: [%s], errorCode: [%s]', requestId, scriptId, errorCode);
@ -107,8 +115,8 @@ JsInvokeMessageProcessor.prototype.processInvokeRequest = function(requestId, re
)
},
(err) => {
var invokeResponse = createInvokeResponse("", false, js.JsInvokeErrorCode.COMPILATION_ERROR, err);
logger.debug('[%s] Sending failed invoke response, scriptId: [%s], errorCode: [%s]', requestId, scriptId, js.JsInvokeErrorCode.COMPILATION_ERROR);
var invokeResponse = createInvokeResponse("", false, COMPILATION_ERROR, err);
logger.debug('[%s] Sending failed invoke response, scriptId: [%s], errorCode: [%s]', requestId, scriptId, COMPILATION_ERROR);
this.sendResponse(requestId, responseTopic, scriptId, null, invokeResponse);
}
);
@ -127,15 +135,26 @@ JsInvokeMessageProcessor.prototype.processReleaseRequest = function(requestId, r
JsInvokeMessageProcessor.prototype.sendResponse = function (requestId, responseTopic, scriptId, compileResponse, invokeResponse, releaseResponse) {
var remoteResponse = createRemoteResponse(requestId, compileResponse, invokeResponse, releaseResponse);
var rawResponse = js.RemoteJsResponse.encode(remoteResponse).finish();
const message = new KeyedMessage(scriptId, rawResponse);
const payloads = [ { topic: responseTopic, messages: message, key: scriptId } ];
this.producer.send(payloads, function (err, data) {
var rawResponse = Buffer.from(JSON.stringify(remoteResponse), 'utf8');
this.producer.send(
{
topic: responseTopic,
messages: [
{
key: scriptId,
value: rawResponse
}
]
}
).then(
() => {},
(err) => {
if (err) {
logger.error('[%s] Failed to send response to kafka: %s', requestId, err.message);
logger.error(err.stack);
}
});
}
);
}
JsInvokeMessageProcessor.prototype.getOrCompileScript = function(scriptId, scriptBody) {
@ -159,50 +178,42 @@ JsInvokeMessageProcessor.prototype.getOrCompileScript = function(scriptId, scrip
function createRemoteResponse(requestId, compileResponse, invokeResponse, releaseResponse) {
const requestIdBits = Utils.UUIDToBits(requestId);
return js.RemoteJsResponse.create(
{
return {
requestIdMSB: requestIdBits[0],
requestIdLSB: requestIdBits[1],
compileResponse: compileResponse,
invokeResponse: invokeResponse,
releaseResponse: releaseResponse
}
);
};
}
function createCompileResponse(scriptId, success, errorCode, err) {
const scriptIdBits = Utils.UUIDToBits(scriptId);
return js.JsCompileResponse.create(
{
return {
errorCode: errorCode,
success: success,
errorDetails: parseJsErrorDetails(err),
scriptIdMSB: scriptIdBits[0],
scriptIdLSB: scriptIdBits[1]
}
);
};
}
function createInvokeResponse(result, success, errorCode, err) {
return js.JsInvokeResponse.create(
{
return {
errorCode: errorCode,
success: success,
errorDetails: parseJsErrorDetails(err),
result: result
}
);
};
}
function createReleaseResponse(scriptId, success) {
const scriptIdBits = Utils.UUIDToBits(scriptId);
return js.JsReleaseResponse.create(
{
return {
success: success,
scriptIdMSB: scriptIdBits[0],
scriptIdLSB: scriptIdBits[1]
}
);
};
}
function parseJsErrorDetails(err) {
@ -229,8 +240,4 @@ function getScriptId(request) {
return Utils.toUUIDString(request.scriptIdMSB, request.scriptIdLSB);
}
function getRequestId(request) {
return Utils.toUUIDString(request.requestIdMSB, request.requestIdLSB);
}
module.exports = JsInvokeMessageProcessor;

View File

@ -18,16 +18,17 @@
const Long = require('long'),
uuidParse = require('uuid-parse');
var logger = require('../config/logger')('Utils');
exports.toUUIDString = function(mostSigBits, leastSigBits) {
var msbBytes = Long.fromValue(mostSigBits, false).toBytes(false);
var lsbBytes = Long.fromValue(leastSigBits, false).toBytes(false);
var uuidBytes = msbBytes.concat(lsbBytes);
var buff = new Buffer(uuidBytes, 'utf8');
return uuidParse.unparse(uuidBytes);
}
exports.UUIDFromBuffer = function(buf) {
return uuidParse.unparse(buf);
}
exports.UUIDToBits = function(uuidString) {
const bytes = uuidParse.parse(uuidString);
var msb = Long.fromBytes(bytes.slice(0,8), false, false).toString();

View File

@ -25,4 +25,5 @@ logger:
filename: "LOGGER_FILENAME"
script:
use_sandbox: "SCRIPT_USE_SANDBOX"
script_body_trace_frequency: "SCRIPT_BODY_TRACE_FREQUENCY"

View File

@ -26,4 +26,5 @@ logger:
filename: "tb-js-executor-%DATE%.log"
script:
use_sandbox: "true"
script_body_trace_frequency: "1000"

View File

@ -17,9 +17,24 @@ var config = require('config'),
path = require('path'),
DailyRotateFile = require('winston-daily-rotate-file');
const { logLevel } = require('kafkajs');
const { createLogger, format, transports } = require('winston');
const { combine, timestamp, label, printf, splat } = format;
const toWinstonLogLevel = level => {
switch(level) {
case logLevel.ERROR:
case logLevel.NOTHING:
return 'error'
case logLevel.WARN:
return 'warn'
case logLevel.INFO:
return 'info'
case logLevel.DEBUG:
return 'debug'
}
}
var loggerTransports = [];
if (process.env.NODE_ENV !== 'production' || process.env.DOCKER_MODE === 'true') {
@ -56,4 +71,33 @@ function _logger(moduleLabel) {
});
}
module.exports = _logger;
const KafkaJsWinstonLogCreator = logLevel => {
const logger = createLogger({
level: toWinstonLogLevel(logLevel),
format:combine(
splat(),
label({ label: 'kafkajs' }),
timestamp({format: 'YYYY-MM-DD HH:mm:ss,SSS'}),
printf(info => {
var res = `${info.timestamp} [${info.label}] ${info.level.toUpperCase()}: ${info.message}`;
if (info.extra) {
res +=`: ${JSON.stringify(info.extra)}`;
}
return res;
}
)
),
transports: loggerTransports
});
return ({ namespace, level, label, log }) => {
const { message, ...extra } = log;
logger.log({
level: toWinstonLogLevel(level),
message,
extra,
});
}
}
module.exports = {_logger, KafkaJsWinstonLogCreator};

View File

@ -35,60 +35,6 @@
"integrity": "sha512-shAmDyaQC4H92APFoIaVDHCx5bStIocgvbwQyxPRrbUY20V1EYTbSDchWbuwlMG3V17cprZhA6+78JfB+3DTPw==",
"dev": true
},
"@protobufjs/aspromise": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz",
"integrity": "sha1-m4sMxmPWaafY9vXQiToU00jzD78="
},
"@protobufjs/base64": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz",
"integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg=="
},
"@protobufjs/codegen": {
"version": "2.0.4",
"resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz",
"integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg=="
},
"@protobufjs/eventemitter": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz",
"integrity": "sha1-NVy8mLr61ZePntCV85diHx0Ga3A="
},
"@protobufjs/fetch": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz",
"integrity": "sha1-upn7WYYUr2VwDBYZ/wbUVLDYTEU=",
"requires": {
"@protobufjs/aspromise": "^1.1.1",
"@protobufjs/inquire": "^1.1.0"
}
},
"@protobufjs/float": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz",
"integrity": "sha1-Xp4avctz/Ap8uLKR33jIy9l7h9E="
},
"@protobufjs/inquire": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz",
"integrity": "sha1-/yAOPnzyQp4tyvwRQIKOjMY48Ik="
},
"@protobufjs/path": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz",
"integrity": "sha1-bMKyDFya1q0NzP0hynZz2Nf79o0="
},
"@protobufjs/pool": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz",
"integrity": "sha1-Cf0V8tbTq/qbZbw2ZQbWrXhG/1Q="
},
"@protobufjs/utf8": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz",
"integrity": "sha1-p3c2C1s5oaLlEG+OhY8v0tBgxXA="
},
"@types/events": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/@types/events/-/events-3.0.0.tgz",
@ -106,11 +52,6 @@
"@types/node": "*"
}
},
"@types/long": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.0.tgz",
"integrity": "sha512-1w52Nyx4Gq47uuu0EVcsHBxZFJgurQ+rTKS3qMHxR1GY2T8c2AJYd6vZoZ9q1rupaDjU0yT+Jc2XTyXkjeMA+Q=="
},
"@types/minimatch": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.3.tgz",
@ -120,7 +61,8 @@
"@types/node": {
"version": "10.12.10",
"resolved": "https://registry.npmjs.org/@types/node/-/node-10.12.10.tgz",
"integrity": "sha512-8xZEYckCbUVgK8Eg7lf5Iy4COKJ5uXlnIOnePN0WUwSQggy9tolM+tDJf7wMOnT/JT/W9xDYIaYggt3mRV2O5w=="
"integrity": "sha512-8xZEYckCbUVgK8Eg7lf5Iy4COKJ5uXlnIOnePN0WUwSQggy9tolM+tDJf7wMOnT/JT/W9xDYIaYggt3mRV2O5w==",
"dev": true
},
"abbrev": {
"version": "1.1.1",
@ -182,11 +124,6 @@
}
}
},
"ansi-regex": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz",
"integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8="
},
"ansi-styles": {
"version": "3.2.1",
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
@ -206,22 +143,6 @@
"normalize-path": "^2.1.1"
}
},
"aproba": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz",
"integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==",
"optional": true
},
"are-we-there-yet": {
"version": "1.1.5",
"resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.5.tgz",
"integrity": "sha512-5hYdAkZlcG8tOLujVDTgCT+uPX0VnpAH28gWsLfzpXYm7wP6mp5Q/gYyR7YQ0cKVJcXJnl3j2kpBan13PtQf6w==",
"optional": true,
"requires": {
"delegates": "^1.0.0",
"readable-stream": "^2.0.6"
}
},
"argparse": {
"version": "1.0.10",
"resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz",
@ -331,7 +252,8 @@
"balanced-match": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
"integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c="
"integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=",
"dev": true
},
"base": {
"version": "0.11.2",
@ -397,39 +319,12 @@
"tweetnacl": "^0.14.3"
}
},
"binary": {
"version": "0.3.0",
"resolved": "https://registry.npmjs.org/binary/-/binary-0.3.0.tgz",
"integrity": "sha1-n2BVO8XOjDOG87VTz/R0Yq3sqnk=",
"requires": {
"buffers": "~0.1.1",
"chainsaw": "~0.1.0"
}
},
"binary-extensions": {
"version": "1.12.0",
"resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-1.12.0.tgz",
"integrity": "sha512-DYWGk01lDcxeS/K9IHPGWfT8PsJmbXRtRd2Sx72Tnb8pcYZQFF1oSDb8hJtS1vhp212q1Rzi5dUf9+nq0o9UIg==",
"dev": true
},
"bindings": {
"version": "1.5.0",
"resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz",
"integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==",
"optional": true,
"requires": {
"file-uri-to-path": "1.0.0"
}
},
"bl": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/bl/-/bl-2.2.0.tgz",
"integrity": "sha512-wbgvOpqopSr7uq6fJrLH8EsvYMJf9gzfo2jCsL2eTy75qXPukA4pCgHamOQkZtY5vmfVtjB+P3LNlMHW5CEZXA==",
"requires": {
"readable-stream": "^2.3.5",
"safe-buffer": "^5.1.1"
}
},
"boxen": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/boxen/-/boxen-1.3.0.tgz",
@ -482,6 +377,7 @@
"version": "1.1.11",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
"integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
"dev": true,
"requires": {
"balanced-match": "^1.0.0",
"concat-map": "0.0.1"
@ -516,53 +412,6 @@
}
}
},
"buffer-alloc": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/buffer-alloc/-/buffer-alloc-1.2.0.tgz",
"integrity": "sha512-CFsHQgjtW1UChdXgbyJGtnm+O/uLQeZdtbDo8mfUgYXCHSM1wgrVxXm6bSyrUuErEb+4sYVGCzASBRot7zyrow==",
"optional": true,
"requires": {
"buffer-alloc-unsafe": "^1.1.0",
"buffer-fill": "^1.0.0"
}
},
"buffer-alloc-unsafe": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/buffer-alloc-unsafe/-/buffer-alloc-unsafe-1.1.0.tgz",
"integrity": "sha512-TEM2iMIEQdJ2yjPJoSIsldnleVaAk1oW3DBVUykyOLsEsFmEc9kn+SFFPz+gl54KQNxlDnAwCXosOS9Okx2xAg==",
"optional": true
},
"buffer-crc32": {
"version": "0.2.13",
"resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz",
"integrity": "sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI="
},
"buffer-fill": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/buffer-fill/-/buffer-fill-1.0.0.tgz",
"integrity": "sha1-+PeLdniYiO858gXNY39o5wISKyw=",
"optional": true
},
"buffermaker": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/buffermaker/-/buffermaker-1.2.1.tgz",
"integrity": "sha512-IdnyU2jDHU65U63JuVQNTHiWjPRH0CS3aYd/WPaEwyX84rFdukhOduAVb1jwUScmb5X0JWPw8NZOrhoLMiyAHQ==",
"requires": {
"long": "1.1.2"
},
"dependencies": {
"long": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/long/-/long-1.1.2.tgz",
"integrity": "sha1-6u9ZUcp1UdlpJrgtokLbnWso+1M="
}
}
},
"buffers": {
"version": "0.1.1",
"resolved": "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz",
"integrity": "sha1-skV5w77U1tOWru5tmorn9Ugqt7s="
},
"byline": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/byline/-/byline-5.0.0.tgz",
@ -610,14 +459,6 @@
"integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=",
"dev": true
},
"chainsaw": {
"version": "0.1.0",
"resolved": "https://registry.npmjs.org/chainsaw/-/chainsaw-0.1.0.tgz",
"integrity": "sha1-XqtQsor+WAdNDVgpE4iCi15fvJg=",
"requires": {
"traverse": ">=0.3.0 <0.4"
}
},
"chalk": {
"version": "2.4.1",
"resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.1.tgz",
@ -650,12 +491,6 @@
"upath": "^1.0.5"
}
},
"chownr": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.2.tgz",
"integrity": "sha512-GkfeAQh+QNy3wquu9oIZr6SS5x7wGdSgNQvD10X3r+AZr1Oys22HW8kAmDMvNg2+Dm0TeGaEuO8gFwdBXxwO8A==",
"optional": true
},
"ci-info": {
"version": "1.6.0",
"resolved": "https://registry.npmjs.org/ci-info/-/ci-info-1.6.0.tgz",
@ -691,11 +526,6 @@
"integrity": "sha1-T6kXw+WclKAEzWH47lCdplFocUM=",
"dev": true
},
"code-point-at": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz",
"integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c="
},
"collection-visit": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/collection-visit/-/collection-visit-1.0.0.tgz",
@ -774,7 +604,8 @@
"concat-map": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
"integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s="
"integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=",
"dev": true
},
"config": {
"version": "3.2.2",
@ -798,11 +629,6 @@
"xdg-basedir": "^3.0.0"
}
},
"console-control-strings": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz",
"integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4="
},
"copy-descriptor": {
"version": "0.1.1",
"resolved": "https://registry.npmjs.org/copy-descriptor/-/copy-descriptor-0.1.1.tgz",
@ -858,6 +684,7 @@
"version": "2.6.9",
"resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
"integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
"dev": true,
"requires": {
"ms": "2.0.0"
}
@ -868,19 +695,11 @@
"integrity": "sha1-6zkTMzRYd1y4TNGh+uBiEGu4dUU=",
"dev": true
},
"decompress-response": {
"version": "3.3.0",
"resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-3.3.0.tgz",
"integrity": "sha1-gKTdMjdIOEv6JICDYirt7Jgq3/M=",
"optional": true,
"requires": {
"mimic-response": "^1.0.0"
}
},
"deep-extend": {
"version": "0.6.0",
"resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz",
"integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA=="
"integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==",
"dev": true
},
"deep-is": {
"version": "0.1.3",
@ -935,23 +754,6 @@
"integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=",
"dev": true
},
"delegates": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz",
"integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=",
"optional": true
},
"denque": {
"version": "1.4.1",
"resolved": "https://registry.npmjs.org/denque/-/denque-1.4.1.tgz",
"integrity": "sha512-OfzPuSZKGcgr96rf1oODnfjqBFmr1DVoc/TrItj3Ohe0Ah1C5WX5Baquw/9U9KovnQ88EqmJbD66rKYUQYN1tQ=="
},
"detect-libc": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz",
"integrity": "sha1-+hN8S9aY7fVc1c0CrFWfkaTEups=",
"optional": true
},
"diagnostics": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/diagnostics/-/diagnostics-1.1.1.tgz",
@ -1004,14 +806,6 @@
"env-variable": "0.0.x"
}
},
"end-of-stream": {
"version": "1.4.1",
"resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.1.tgz",
"integrity": "sha512-1MkrZNvWTKCaigbn+W15elq2BB/L22nqrSY5DKlo3X6+vclJm8Bb5djXJBmEX6fS3+zCh/F4VBK5Z2KxJt4s2Q==",
"requires": {
"once": "^1.4.0"
}
},
"env-variable": {
"version": "0.0.5",
"resolved": "https://registry.npmjs.org/env-variable/-/env-variable-0.0.5.tgz",
@ -1272,12 +1066,6 @@
"moment": "^2.11.2"
}
},
"file-uri-to-path": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz",
"integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==",
"optional": true
},
"fill-range": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/fill-range/-/fill-range-4.0.0.tgz",
@ -1343,12 +1131,6 @@
"readable-stream": "^2.0.0"
}
},
"fs-constants": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz",
"integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==",
"optional": true
},
"fs-extra": {
"version": "6.0.1",
"resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-6.0.1.tgz",
@ -1895,22 +1677,6 @@
}
}
},
"gauge": {
"version": "2.7.4",
"resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz",
"integrity": "sha1-LANAXHU4w51+s3sxcCLjJfsBi/c=",
"optional": true,
"requires": {
"aproba": "^1.0.3",
"console-control-strings": "^1.0.0",
"has-unicode": "^2.0.0",
"object-assign": "^4.1.0",
"signal-exit": "^3.0.0",
"string-width": "^1.0.1",
"strip-ansi": "^3.0.1",
"wide-align": "^1.1.0"
}
},
"get-stream": {
"version": "3.0.0",
"resolved": "http://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz",
@ -1932,12 +1698,6 @@
"assert-plus": "^1.0.0"
}
},
"github-from-package": {
"version": "0.0.0",
"resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz",
"integrity": "sha1-l/tdlr/eiXMxPyDoKI75oWf6ZM4=",
"optional": true
},
"glob": {
"version": "7.1.4",
"resolved": "https://registry.npmjs.org/glob/-/glob-7.1.4.tgz",
@ -2059,12 +1819,6 @@
"integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=",
"dev": true
},
"has-unicode": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz",
"integrity": "sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk=",
"optional": true
},
"has-value": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/has-value/-/has-value-1.0.0.tgz",
@ -2150,7 +1904,8 @@
"ini": {
"version": "1.3.5",
"resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz",
"integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw=="
"integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==",
"dev": true
},
"into-stream": {
"version": "5.1.0",
@ -2262,14 +2017,6 @@
"integrity": "sha1-qIwCU1eR8C7TfHahueqXc8gz+MI=",
"dev": true
},
"is-fullwidth-code-point": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz",
"integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=",
"requires": {
"number-is-nan": "^1.0.0"
}
},
"is-glob": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.0.tgz",
@ -2453,42 +2200,12 @@
"verror": "1.10.0"
}
},
"kafka-node": {
"version": "4.1.3",
"resolved": "https://registry.npmjs.org/kafka-node/-/kafka-node-4.1.3.tgz",
"integrity": "sha512-C2WHksRCr7vIKmbxYaCk2c5Q1lnHIi6C0f3AioK3ARcRHGO9DpqErcoaS9d8PP62yzTnkYras+iAlmPsZHNSfw==",
"kafkajs": {
"version": "1.11.0",
"resolved": "https://registry.npmjs.org/kafkajs/-/kafkajs-1.11.0.tgz",
"integrity": "sha512-dLRCcFIBygZucR+e8U2ZqH2wgMrAu114K0szUyUseJoeOii3cG5bHZPIdqKecXxI6begPVCfGS3R0nJY4zHW2A==",
"requires": {
"async": "^2.6.2",
"binary": "~0.3.0",
"bl": "^2.2.0",
"buffer-crc32": "~0.2.5",
"buffermaker": "~1.2.0",
"debug": "^2.1.3",
"denque": "^1.3.0",
"lodash": "^4.17.4",
"minimatch": "^3.0.2",
"nested-error-stacks": "^2.0.0",
"optional": "^0.1.3",
"retry": "^0.10.1",
"snappy": "^6.0.1",
"uuid": "^3.0.0"
},
"dependencies": {
"async": {
"version": "2.6.3",
"resolved": "https://registry.npmjs.org/async/-/async-2.6.3.tgz",
"integrity": "sha512-zflvls11DCy+dQWzTW2dzuilv8Z5X/pjfmZOWba6TNIVDm+2UDaJmXSOXlasHKfNBs8oo3M0aT50fDEWfKZjXg==",
"requires": {
"lodash": "^4.17.14"
},
"dependencies": {
"lodash": {
"version": "4.17.15",
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz",
"integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A=="
}
}
}
"long": "^4.0.0"
}
},
"kind-of": {
@ -2641,16 +2358,11 @@
"mime-db": "1.40.0"
}
},
"mimic-response": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz",
"integrity": "sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==",
"optional": true
},
"minimatch": {
"version": "3.0.4",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
"integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
"dev": true,
"requires": {
"brace-expansion": "^1.1.7"
}
@ -2685,6 +2397,7 @@
"version": "0.5.1",
"resolved": "http://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz",
"integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=",
"dev": true,
"requires": {
"minimist": "0.0.8"
},
@ -2692,7 +2405,8 @@
"minimist": {
"version": "0.0.8",
"resolved": "http://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz",
"integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0="
"integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=",
"dev": true
}
}
},
@ -2704,7 +2418,8 @@
"ms": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
"integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
"integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=",
"dev": true
},
"multistream": {
"version": "2.1.1",
@ -2742,26 +2457,6 @@
"to-regex": "^3.0.1"
}
},
"napi-build-utils": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.1.tgz",
"integrity": "sha512-boQj1WFgQH3v4clhu3mTNfP+vOBxorDlE8EKiMjUlLG3C4qAESnn9AxIOkFgTR2c9LtzNjPrjS60cT27ZKBhaA==",
"optional": true
},
"nested-error-stacks": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/nested-error-stacks/-/nested-error-stacks-2.1.0.tgz",
"integrity": "sha512-AO81vsIO1k1sM4Zrd6Hu7regmJN1NSiAja10gc4bX3F0wd+9rQmcuHQaHVQCYIEC8iFXnE+mavh23GOt7wBgug=="
},
"node-abi": {
"version": "2.9.0",
"resolved": "https://registry.npmjs.org/node-abi/-/node-abi-2.9.0.tgz",
"integrity": "sha512-jmEOvv0eanWjhX8dX1pmjb7oJl1U1oR4FOh0b2GnvALwSYoOdU7sj+kLDSAyjo4pfC9aj/IxkloxdLJQhSSQBA==",
"optional": true,
"requires": {
"semver": "^5.4.1"
}
},
"nodemon": {
"version": "1.18.7",
"resolved": "https://registry.npmjs.org/nodemon/-/nodemon-1.18.7.tgz",
@ -2797,12 +2492,6 @@
}
}
},
"noop-logger": {
"version": "0.1.1",
"resolved": "https://registry.npmjs.org/noop-logger/-/noop-logger-0.1.1.tgz",
"integrity": "sha1-lKKxYzxPExdVMAfYlm/Q6EG2pMI=",
"optional": true
},
"nopt": {
"version": "1.0.10",
"resolved": "https://registry.npmjs.org/nopt/-/nopt-1.0.10.tgz",
@ -2830,35 +2519,12 @@
"path-key": "^2.0.0"
}
},
"npmlog": {
"version": "4.1.2",
"resolved": "https://registry.npmjs.org/npmlog/-/npmlog-4.1.2.tgz",
"integrity": "sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==",
"optional": true,
"requires": {
"are-we-there-yet": "~1.1.2",
"console-control-strings": "~1.1.0",
"gauge": "~2.7.3",
"set-blocking": "~2.0.0"
}
},
"number-is-nan": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz",
"integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0="
},
"oauth-sign": {
"version": "0.9.0",
"resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz",
"integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==",
"dev": true
},
"object-assign": {
"version": "4.1.1",
"resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
"integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=",
"optional": true
},
"object-copy": {
"version": "0.1.0",
"resolved": "https://registry.npmjs.org/object-copy/-/object-copy-0.1.0.tgz",
@ -2917,6 +2583,7 @@
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
"integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
"dev": true,
"requires": {
"wrappy": "1"
}
@ -2926,11 +2593,6 @@
"resolved": "https://registry.npmjs.org/one-time/-/one-time-0.0.4.tgz",
"integrity": "sha1-+M33eISCb+Tf+T46nMN7HkSAdC4="
},
"optional": {
"version": "0.1.4",
"resolved": "https://registry.npmjs.org/optional/-/optional-0.1.4.tgz",
"integrity": "sha512-gtvrrCfkE08wKcgXaVwQVgwEQ8vel2dc5DDBn9RLQZ3YtmtkBss6A2HY6BnJH4N/4Ku97Ri/SF8sNWE2225WJw=="
},
"optionator": {
"version": "0.8.2",
"resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.2.tgz",
@ -2945,12 +2607,6 @@
"wordwrap": "~1.0.0"
}
},
"os-homedir": {
"version": "1.0.2",
"resolved": "http://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz",
"integrity": "sha1-/7xJiDNuDoM94MFox+8VISGqf7M=",
"optional": true
},
"os-tmpdir": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz",
@ -3127,38 +2783,6 @@
"integrity": "sha1-AerA/jta9xoqbAL+q7jB/vfgDqs=",
"dev": true
},
"prebuild-install": {
"version": "5.3.0",
"resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-5.3.0.tgz",
"integrity": "sha512-aaLVANlj4HgZweKttFNUVNRxDukytuIuxeK2boIMHjagNJCiVKWFsKF4tCE3ql3GbrD2tExPQ7/pwtEJcHNZeg==",
"optional": true,
"requires": {
"detect-libc": "^1.0.3",
"expand-template": "^2.0.3",
"github-from-package": "0.0.0",
"minimist": "^1.2.0",
"mkdirp": "^0.5.1",
"napi-build-utils": "^1.0.1",
"node-abi": "^2.7.0",
"noop-logger": "^0.1.1",
"npmlog": "^4.0.1",
"os-homedir": "^1.0.1",
"pump": "^2.0.1",
"rc": "^1.2.7",
"simple-get": "^2.7.0",
"tar-fs": "^1.13.0",
"tunnel-agent": "^0.6.0",
"which-pm-runs": "^1.0.0"
},
"dependencies": {
"expand-template": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz",
"integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==",
"optional": true
}
}
},
"prelude-ls": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz",
@ -3182,26 +2806,6 @@
"integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==",
"dev": true
},
"protobufjs": {
"version": "6.8.8",
"resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.8.8.tgz",
"integrity": "sha512-AAmHtD5pXgZfi7GMpllpO3q1Xw1OYldr+dMUlAnffGTAhqkg72WdmSY71uKBF/JuyiKs8psYbtKrhi0ASCD8qw==",
"requires": {
"@protobufjs/aspromise": "^1.1.2",
"@protobufjs/base64": "^1.1.2",
"@protobufjs/codegen": "^2.0.4",
"@protobufjs/eventemitter": "^1.1.0",
"@protobufjs/fetch": "^1.1.0",
"@protobufjs/float": "^1.0.2",
"@protobufjs/inquire": "^1.1.0",
"@protobufjs/path": "^1.1.2",
"@protobufjs/pool": "^1.1.0",
"@protobufjs/utf8": "^1.1.0",
"@types/long": "^4.0.0",
"@types/node": "^10.1.0",
"long": "^4.0.0"
}
},
"pseudomap": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz",
@ -3220,16 +2824,6 @@
"integrity": "sha512-vL6NLxNHzkNTjGJUpMm5PLC+94/0tTlC1vkP9bdU0pOHih+EujMjgMTwfZopZvHWRFbqJ5Y73OMoau50PewDDA==",
"dev": true
},
"pump": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/pump/-/pump-2.0.1.tgz",
"integrity": "sha512-ruPMNRkN3MHP1cWJc9OWr+T/xDP0jhXYCLfJcBuX54hhfIBnaQmAUMfDcG4DM5UMWByBbJY69QSphm3jtDKIkA==",
"optional": true,
"requires": {
"end-of-stream": "^1.1.0",
"once": "^1.3.1"
}
},
"punycode": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz",
@ -3246,6 +2840,7 @@
"version": "1.2.8",
"resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz",
"integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==",
"dev": true,
"requires": {
"deep-extend": "^0.6.0",
"ini": "~1.3.0",
@ -3389,11 +2984,6 @@
"integrity": "sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg==",
"dev": true
},
"retry": {
"version": "0.10.1",
"resolved": "https://registry.npmjs.org/retry/-/retry-0.10.1.tgz",
"integrity": "sha1-52OI0heZLCUnUCQdPTlW/tmNj/Q="
},
"safe-buffer": {
"version": "5.1.2",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
@ -3428,12 +3018,6 @@
"semver": "^5.0.3"
}
},
"set-blocking": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz",
"integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=",
"optional": true
},
"set-value": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/set-value/-/set-value-2.0.0.tgz",
@ -3475,24 +3059,8 @@
"signal-exit": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz",
"integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0="
},
"simple-concat": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.0.tgz",
"integrity": "sha1-c0TLuLbib7J9ZrL8hvn21Zl1IcY=",
"optional": true
},
"simple-get": {
"version": "2.8.1",
"resolved": "https://registry.npmjs.org/simple-get/-/simple-get-2.8.1.tgz",
"integrity": "sha512-lSSHRSw3mQNUGPAYRqo7xy9dhKmxFXIjLjp4KHpf99GEH2VH7C3AM+Qfx6du6jhfUi6Vm7XnbEVEf7Wb6N8jRw==",
"optional": true,
"requires": {
"decompress-response": "^3.3.0",
"once": "^1.3.1",
"simple-concat": "^1.0.0"
}
"integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=",
"dev": true
},
"simple-swizzle": {
"version": "0.2.2",
@ -3615,25 +3183,6 @@
}
}
},
"snappy": {
"version": "6.2.3",
"resolved": "https://registry.npmjs.org/snappy/-/snappy-6.2.3.tgz",
"integrity": "sha512-HZpVoIxMfQ4fL3iDuMdI1R5xycw1o9YDCAndTKZCY/EHRoKFvzwplttuBBVGeEg2fd1hYiwAXos/sM24W7N1LA==",
"optional": true,
"requires": {
"bindings": "^1.3.1",
"nan": "^2.14.0",
"prebuild-install": "^5.2.2"
},
"dependencies": {
"nan": {
"version": "2.14.0",
"resolved": "https://registry.npmjs.org/nan/-/nan-2.14.0.tgz",
"integrity": "sha512-INOFj37C7k3AfaNTtX8RhsTw7qRy7eLET14cROi9+5HAVbbHuIWUHEauBv5qT4Av2tWasiTY1Jw6puUNqRJXQg==",
"optional": true
}
}
},
"source-map": {
"version": "0.5.7",
"resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz",
@ -3725,16 +3274,6 @@
"readable-stream": "^2.1.4"
}
},
"string-width": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz",
"integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=",
"requires": {
"code-point-at": "^1.0.0",
"is-fullwidth-code-point": "^1.0.0",
"strip-ansi": "^3.0.0"
}
},
"string_decoder": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
@ -3743,14 +3282,6 @@
"safe-buffer": "~5.1.0"
}
},
"strip-ansi": {
"version": "3.0.1",
"resolved": "http://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz",
"integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=",
"requires": {
"ansi-regex": "^2.0.0"
}
},
"strip-eof": {
"version": "1.0.0",
"resolved": "http://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz",
@ -3760,7 +3291,8 @@
"strip-json-comments": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz",
"integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo="
"integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=",
"dev": true
},
"supports-color": {
"version": "5.5.0",
@ -3771,57 +3303,6 @@
"has-flag": "^3.0.0"
}
},
"tar-fs": {
"version": "1.16.3",
"resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-1.16.3.tgz",
"integrity": "sha512-NvCeXpYx7OsmOh8zIOP/ebG55zZmxLE0etfWRbWok+q2Qo8x/vOR/IJT1taADXPe+jsiu9axDb3X4B+iIgNlKw==",
"optional": true,
"requires": {
"chownr": "^1.0.1",
"mkdirp": "^0.5.1",
"pump": "^1.0.0",
"tar-stream": "^1.1.2"
},
"dependencies": {
"pump": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/pump/-/pump-1.0.3.tgz",
"integrity": "sha512-8k0JupWme55+9tCVE+FS5ULT3K6AbgqrGa58lTT49RpyfwwcGedHqaC5LlQNdEAumn/wFsu6aPwkuPMioy8kqw==",
"optional": true,
"requires": {
"end-of-stream": "^1.1.0",
"once": "^1.3.1"
}
}
}
},
"tar-stream": {
"version": "1.6.2",
"resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-1.6.2.tgz",
"integrity": "sha512-rzS0heiNf8Xn7/mpdSVVSMAWAoy9bfb1WOTYC78Z0UQKeKa/CWS8FOq0lKGNa8DWKAn9gxjCvMLYc5PGXYlK2A==",
"optional": true,
"requires": {
"bl": "^1.0.0",
"buffer-alloc": "^1.2.0",
"end-of-stream": "^1.0.0",
"fs-constants": "^1.0.0",
"readable-stream": "^2.3.0",
"to-buffer": "^1.1.1",
"xtend": "^4.0.0"
},
"dependencies": {
"bl": {
"version": "1.2.2",
"resolved": "https://registry.npmjs.org/bl/-/bl-1.2.2.tgz",
"integrity": "sha512-e8tQYnZodmebYDWGH7KMRvtzKXaJHx3BbilrgZCfvyLUYdKpK1t5PSPmpkny/SgiTSCnjfLW7v5rlONXVFkQEA==",
"optional": true,
"requires": {
"readable-stream": "^2.3.5",
"safe-buffer": "^5.1.1"
}
}
}
},
"term-size": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/term-size/-/term-size-1.2.0.tgz",
@ -3848,12 +3329,6 @@
"integrity": "sha1-8y6srFoXW+ol1/q1Zas+2HQe9W8=",
"dev": true
},
"to-buffer": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/to-buffer/-/to-buffer-1.1.1.tgz",
"integrity": "sha512-lx9B5iv7msuFYE3dytT+KE5tap+rNYw+K4jVkb9R/asAb+pbBSM17jtunHplhBe6RRJdZx3Pn2Jph24O32mOVg==",
"optional": true
},
"to-object-path": {
"version": "0.3.0",
"resolved": "https://registry.npmjs.org/to-object-path/-/to-object-path-0.3.0.tgz",
@ -3923,11 +3398,6 @@
}
}
},
"traverse": {
"version": "0.3.9",
"resolved": "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz",
"integrity": "sha1-cXuPIgzAu3tE5AUUwisui7xw2Lk="
},
"triple-beam": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/triple-beam/-/triple-beam-1.3.0.tgz",
@ -3937,6 +3407,7 @@
"version": "0.6.0",
"resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz",
"integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=",
"dev": true,
"requires": {
"safe-buffer": "^5.0.1"
}
@ -4140,7 +3611,8 @@
"uuid": {
"version": "3.3.2",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz",
"integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA=="
"integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==",
"dev": true
},
"uuid-parse": {
"version": "1.0.0",
@ -4167,21 +3639,6 @@
"isexe": "^2.0.0"
}
},
"which-pm-runs": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/which-pm-runs/-/which-pm-runs-1.0.0.tgz",
"integrity": "sha1-Zws6+8VS4LVd9rd4DKdGFfI60cs=",
"optional": true
},
"wide-align": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.3.tgz",
"integrity": "sha512-QGkOQc8XL6Bt5PwnsExKBPuMKBxnGxWWW3fU55Xt4feHozMUhdUMaBCk290qpm/wG5u/RSKzwdAC4i51YigihA==",
"optional": true,
"requires": {
"string-width": "^1.0.2 || 2"
}
},
"widest-line": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/widest-line/-/widest-line-2.0.1.tgz",
@ -4281,7 +3738,8 @@
"wrappy": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
"integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8="
"integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=",
"dev": true
},
"write-file-atomic": {
"version": "2.3.0",
@ -4300,12 +3758,6 @@
"integrity": "sha1-SWsswQnsqNus/i3HK2A8F8WHCtQ=",
"dev": true
},
"xtend": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",
"integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==",
"optional": true
},
"yallist": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/yallist/-/yallist-3.0.3.tgz",

View File

@ -6,18 +6,16 @@
"main": "server.js",
"bin": "server.js",
"scripts": {
"build-proto": "pbjs -t static-module -w commonjs -o ./api/jsinvoke.proto.js ../../application/src/main/proto/jsinvoke.proto",
"install": "npm run build-proto && pkg -t node10-linux-x64,node10-win-x64 --out-path ./target . && node install.js",
"install": "pkg -t node10-linux-x64,node10-win-x64 --out-path ./target . && node install.js",
"test": "echo \"Error: no test specified\" && exit 1",
"start": "npm run build-proto && nodemon server.js",
"start-prod": "npm run build-proto && NODE_ENV=production nodemon server.js"
"start": "nodemon server.js",
"start-prod": "NODE_ENV=production nodemon server.js"
},
"dependencies": {
"config": "^3.2.2",
"js-yaml": "^3.12.0",
"kafka-node": "^4.1.3",
"kafkajs": "^1.11.0",
"long": "^4.0.0",
"protobufjs": "^6.8.8",
"uuid-parse": "^1.0.0",
"winston": "^3.0.0",
"winston-daily-rotate-file": "^3.2.1"

View File

@ -13,14 +13,17 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
const { logLevel, Kafka } = require('kafkajs');
const config = require('config'),
kafka = require('kafka-node'),
ConsumerGroup = kafka.ConsumerGroup,
Producer = kafka.Producer,
JsInvokeMessageProcessor = require('./api/jsInvokeMessageProcessor'),
logger = require('./config/logger')('main');
logger = require('./config/logger')._logger('main'),
KafkaJsWinstonLogCreator = require('./config/logger').KafkaJsWinstonLogCreator;
var kafkaClient;
var consumer;
var producer;
(async() => {
try {
@ -32,49 +35,24 @@ var kafkaClient;
logger.info('Kafka Bootstrap Servers: %s', kafkaBootstrapServers);
logger.info('Kafka Requests Topic: %s', kafkaRequestTopic);
kafkaClient = new kafka.KafkaClient({kafkaHost: kafkaBootstrapServers});
var consumer = new ConsumerGroup(
{
kafkaHost: kafkaBootstrapServers,
groupId: 'js-executor-group',
autoCommit: true,
encoding: 'buffer'
},
kafkaRequestTopic
);
consumer.on('error', (err) => {
logger.error('Unexpected kafka consumer error: %s', err.message);
logger.error(err.stack);
kafkaClient = new Kafka({
brokers: kafkaBootstrapServers.split(','),
logLevel: logLevel.INFO,
logCreator: KafkaJsWinstonLogCreator
});
consumer.on('offsetOutOfRange', (err) => {
logger.error('Offset out of range error: %s', err.message);
logger.error(err.stack);
});
consumer = kafkaClient.consumer({ groupId: 'js-executor-group' });
producer = kafkaClient.producer();
const messageProcessor = new JsInvokeMessageProcessor(producer);
await consumer.connect();
await producer.connect();
await consumer.subscribe({ topic: kafkaRequestTopic});
consumer.on('rebalancing', () => {
logger.info('Rebalancing event received.');
})
consumer.on('rebalanced', () => {
logger.info('Rebalanced event received.');
});
var producer = new Producer(kafkaClient);
producer.on('error', (err) => {
logger.error('Unexpected kafka producer error: %s', err.message);
logger.error(err.stack);
});
var messageProcessor = new JsInvokeMessageProcessor(producer);
producer.on('ready', () => {
consumer.on('message', (message) => {
messageProcessor.onJsInvokeMessage(message);
});
logger.info('Started ThingsBoard JavaScript Executor Microservice.');
await consumer.run({
eachMessage: async ({ topic, partition, message }) => {
messageProcessor.onJsInvokeMessage(message);
},
});
} catch (e) {
@ -84,21 +62,41 @@ var kafkaClient;
}
})();
process.on('exit', function () {
process.on('exit', () => {
exit(0);
});
function exit(status) {
async function exit(status) {
logger.info('Exiting with status: %d ...', status);
if (kafkaClient) {
logger.info('Stopping Kafka Client...');
var _kafkaClient = kafkaClient;
kafkaClient = null;
_kafkaClient.close(() => {
logger.info('Kafka Client stopped.');
if (consumer) {
logger.info('Stopping Kafka Consumer...');
var _consumer = consumer;
consumer = null;
try {
await _consumer.disconnect();
logger.info('Kafka Consumer stopped.');
await disconnectProducer();
process.exit(status);
});
} catch (e) {
logger.info('Kafka Consumer stop error.');
await disconnectProducer();
process.exit(status);
}
} else {
process.exit(status);
}
}
async function disconnectProducer() {
if (producer) {
logger.info('Stopping Kafka Producer...');
var _producer = producer;
producer = null;
try {
await _producer.disconnect();
logger.info('Kafka Producer stopped.');
} catch (e) {
logger.info('Kafka Producer stop error.');
}
}
}