dendi-wang/thingsboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #6768 from vvlladd28/improvement/js-executor/script-map/frequirence

Browse Source 
[3.4] JS-executor: Improvement logs and clear cache
This commit is contained in:
Igor Kulikov 2022-06-21 17:26:27 +03:00 committed by GitHub
commit 2eaa4660bf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 35 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
msa/js-executor/api/jsInvokeMessageProcessor.js
View File

@ -32,15 +32,17 @@ const maxActiveScripts = Number(config.get('script.max_active_scripts'));
const slowQueryLogMs = Number(config.get('script.slow_query_log_ms')); const slowQueryLogMs = Number(config.get('script.slow_query_log_ms'));
const slowQueryLogBody = config.get('script.slow_query_log_body') === 'true'; const slowQueryLogBody = config.get('script.slow_query_log_body') === 'true';
const {performance} = require('perf_hooks'); const { performance } = require('node:perf_hooks');
function JsInvokeMessageProcessor(producer) { function JsInvokeMessageProcessor(producer) {
this.producer = producer; this.producer = producer;
this.executor = new JsExecutor(useSandbox); this.executor = new JsExecutor(useSandbox);
this.scriptMap = new Map(); this.scriptMap = new Map();
this.scriptIds = []; this.scriptIds = [];
this.executedScriptIdsCounter = [];
this.executedScriptsCounter = 0; this.executedScriptsCounter = 0;
this.lastStatTime = performance.now(); this.lastStatTime = performance.now();
this.compilationTime = 0;
} }
JsInvokeMessageProcessor.prototype.onJsInvokeMessage = function (message) { JsInvokeMessageProcessor.prototype.onJsInvokeMessage = function (message) {
@ -125,7 +127,8 @@ JsInvokeMessageProcessor.prototype.processInvokeRequest = function (requestId, r
const msSinceLastStat = nowMs - this.lastStatTime; const msSinceLastStat = nowMs - this.lastStatTime;
const requestsPerSec = msSinceLastStat == 0 ? statFrequency : statFrequency / msSinceLastStat * 1000; const requestsPerSec = msSinceLastStat == 0 ? statFrequency : statFrequency / msSinceLastStat * 1000;
this.lastStatTime = nowMs; this.lastStatTime = nowMs;
logger.info('STAT[%s]: requests [%s], took [%s]ms, request/s [%s]', this.executedScriptsCounter, statFrequency, msSinceLastStat, requestsPerSec); logger.info('STAT[%s]: requests [%s], took [%s]ms, request/s [%s], compilation [%s]ms', this.executedScriptsCounter, statFrequency, msSinceLastStat, requestsPerSec, this.compilationTime);
this.compilationTime = 0;
} }
if (this.executedScriptsCounter % scriptBodyTraceFrequency == 0) { if (this.executedScriptsCounter % scriptBodyTraceFrequency == 0) {
@ -167,6 +170,7 @@ JsInvokeMessageProcessor.prototype.processReleaseRequest = function (requestId,
var index = this.scriptIds.indexOf(scriptId); var index = this.scriptIds.indexOf(scriptId);
if (index > -1) { if (index > -1) {
this.scriptIds.splice(index, 1); this.scriptIds.splice(index, 1);
this.executedScriptIdsCounter.splice(index, 1);
} }
this.scriptMap.delete(scriptId); this.scriptMap.delete(scriptId);
} }
@ -198,14 +202,18 @@ JsInvokeMessageProcessor.prototype.getOrCompileScript = function (scriptId, scri
return new Promise(function (resolve, reject) { return new Promise(function (resolve, reject) {
const script = self.scriptMap.get(scriptId); const script = self.scriptMap.get(scriptId);
if (script) { if (script) {
incrementUseScriptId.call(self, scriptId);
resolve(script); resolve(script);
} else { } else {
const startTime = performance.now();
self.executor.compileScript(scriptBody).then( self.executor.compileScript(scriptBody).then(
(compiledScript) => { (compiledScript) => {
self.compilationTime += (performance.now() - startTime);
self.cacheScript(scriptId, compiledScript); self.cacheScript(scriptId, compiledScript);
resolve(compiledScript); resolve(compiledScript);
}, },
(err) => { (err) => {
self.compilationTime += (performance.now() - startTime);
reject(err); reject(err);
} }
); );
@ -216,11 +224,10 @@ JsInvokeMessageProcessor.prototype.getOrCompileScript = function (scriptId, scri
JsInvokeMessageProcessor.prototype.cacheScript = function (scriptId, script) { JsInvokeMessageProcessor.prototype.cacheScript = function (scriptId, script) {
if (!this.scriptMap.has(scriptId)) { if (!this.scriptMap.has(scriptId)) {
this.scriptIds.push(scriptId); this.scriptIds.push(scriptId);
this.executedScriptIdsCounter.push(0);
while (this.scriptIds.length > maxActiveScripts) { while (this.scriptIds.length > maxActiveScripts) {
logger.info('Active scripts count [%s] exceeds maximum limit [%s]', this.scriptIds.length, maxActiveScripts); logger.info('Active scripts count [%s] exceeds maximum limit [%s]', this.scriptIds.length, maxActiveScripts);
const prevScriptId = this.scriptIds.shift(); deleteMinUsedScript.apply(this);
logger.info('Removing active script with id [%s]', prevScriptId);
this.scriptMap.delete(prevScriptId);
} }
} }
this.scriptMap.set(scriptId, script); this.scriptMap.set(scriptId, script);
@ -291,4 +298,27 @@ function getScriptId(request) {
return Utils.toUUIDString(request.scriptIdMSB, request.scriptIdLSB); return Utils.toUUIDString(request.scriptIdMSB, request.scriptIdLSB);
} }
function incrementUseScriptId(scriptId) {
const index = this.scriptIds.indexOf(scriptId);
if (this.executedScriptIdsCounter[index] < Number.MAX_SAFE_INTEGER) {
this.executedScriptIdsCounter[index]++;
}
}
function deleteMinUsedScript() {
let min = Infinity;
let minIndex = 0;
const scriptIdsLength = this.executedScriptIdsCounter.length - 1; // ignored last added script
for (let i = 0; i < scriptIdsLength; i++) {
if (this.executedScriptIdsCounter[i] < min) {
min = this.executedScriptIdsCounter[i];
minIndex = i;
}
}
const prevScriptId = this.scriptIds.splice(minIndex, 1)[0];
this.executedScriptIdsCounter.splice(minIndex, 1)
logger.info('Removing active script with id [%s]', prevScriptId);
this.scriptMap.delete(prevScriptId);
}
module.exports = JsInvokeMessageProcessor; module.exports = JsInvokeMessageProcessor;