dendi-wang/thingsboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

moved kafka from service.js to own module

Browse Source
This commit is contained in:
YevhenBondarenko 2020-04-21 13:30:11 +03:00
parent 12973cecf1
commit 4fb309c37e
7 changed files with 151 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java
View File

@ -46,17 +46,17 @@ import java.util.concurrent.atomic.AtomicInteger;
@Service
public class RemoteJsInvokeService extends AbstractJsInvokeService {
@Value("${js.remote.max_requests_timeout}")
@Value("${queue.js.max_requests_timeout}")
private long maxRequestsTimeout;
@Getter
@Value("${js.remote.max_errors}")
// @Value("${queue.js.max_errors}")
private int maxErrors;
@Value("${js.remote.max_black_list_duration_sec:60}")
@Value("${queue.js.max_black_list_duration_sec:60}")
private int maxBlackListDurationSec;
@Value("${js.remote.stats.enabled:false}")
@Value("${queue.js.stats.enabled:false}")
private boolean statsEnabled;
private final AtomicInteger kafkaPushedMsgs = new AtomicInteger(0);
@ -65,7 +65,7 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService {
private final AtomicInteger kafkaFailedMsgs = new AtomicInteger(0);
private final AtomicInteger kafkaTimeoutMsgs = new AtomicInteger(0);
@Scheduled(fixedDelayString = "${js.remote.stats.print_interval_ms}")
// @Scheduled(fixedDelayString = "${queue.js.stats.print_interval_ms}")
public void printStats() {
if (statsEnabled) {
int pushedMsgs = kafkaPushedMsgs.getAndSet(0);

6
application/src/main/resources/thingsboard.yml
View File

@ -415,7 +415,7 @@ state:
persistToTelemetry: "${PERSIST_STATE_TO_TELEMETRY:false}"
js:
evaluator: "${JS_EVALUATOR:local}" # local/remote
evaluator: "${JS_EVALUATOR:remote}" # local/remote
# Built-in JVM JavaScript environment properties
local:
# Use Sandboxed (secured) JVM JavaScript environment
@ -582,9 +582,9 @@ queue:
print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}"
js:
# JS Eval request topic
request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}"
request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js.eval.requests}"
# JS Eval responses topic prefix that is combined with node id
response_topic_prefix: "${REMOTE_JS_EVAL_RESPONSE_TOPIC:js_eval.responses}"
response_topic_prefix: "${REMOTE_JS_EVAL_RESPONSE_TOPIC:js.eval.responses}"
# JS Eval max pending requests
max_pending_requests: "${REMOTE_JS_MAX_PENDING_REQUESTS:10000}"
# JS Eval max request timeout

23
msa/js-executor/api/jsInvokeMessageProcessor.js
View File

@ -31,6 +31,7 @@ const useSandbox = config.get('script.use_sandbox') === 'true';
const maxActiveScripts = Number(config.get('script.max_active_scripts'));
function JsInvokeMessageProcessor(producer) {
console.log("Kafka Producer:", producer);
this.producer = producer;
this.executor = new JsExecutor(useSandbox);
this.scriptMap = {};
@ -144,17 +145,17 @@ JsInvokeMessageProcessor.prototype.processReleaseRequest = function(requestId, r
JsInvokeMessageProcessor.prototype.sendResponse = function (requestId, responseTopic, scriptId, compileResponse, invokeResponse, releaseResponse) {
var remoteResponse = createRemoteResponse(requestId, compileResponse, invokeResponse, releaseResponse);
var rawResponse = Buffer.from(JSON.stringify(remoteResponse), 'utf8');
this.producer.send(
{
topic: responseTopic,
messages: [
{
key: scriptId,
value: rawResponse,
headers: headers
}
]
}
this.producer.send(responseTopic, scriptId, rawResponse, headers
// {
// topic: responseTopic,
// messages: [
// {
// key: scriptId,
// value: rawResponse,
// headers: headers
// }
// ]
// }
).then(
() => {},
(err) => {

2
msa/js-executor/config/custom-environment-variables.yml
View File

@ -14,6 +14,8 @@
# limitations under the License.
#
service-type: "TB_SERVICE_TYPE"
kafka:
request_topic: "REMOTE_JS_EVAL_REQUEST_TOPIC"
bootstrap:

2
msa/js-executor/config/default.yml
View File

@ -14,6 +14,8 @@
# limitations under the License.
#
service-type: "kafka"
kafka:
request_topic: "js.eval.requests"
bootstrap:

117
msa/js-executor/queue/kafka/kafkaTemplate.js Normal file
View File

@ -0,0 +1,117 @@
/*
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
const { logLevel, Kafka } = require('kafkajs');
const config = require('config'),
JsInvokeMessageProcessor = require('../../api/jsInvokeMessageProcessor'),
logger = require('../../config/logger')._logger('main'),
KafkaJsWinstonLogCreator = require('../../config/logger').KafkaJsWinstonLogCreator;
var kafkaClient;
var consumer;
var producer;
function KafkaProducer() {
this.send = async (responseTopic, scriptId, rawResponse, headers) => {
return producer.send(
{
topic: responseTopic,
messages: [
{
key: scriptId,
value: rawResponse,
headers: headers
}
]
});
}
}
(async() => {
try {
logger.info('Starting ThingsBoard JavaScript Executor Microservice...');
const kafkaBootstrapServers = config.get('kafka.bootstrap.servers');
const kafkaRequestTopic = config.get('kafka.request_topic');
logger.info('Kafka Bootstrap Servers: %s', kafkaBootstrapServers);
logger.info('Kafka Requests Topic: %s', kafkaRequestTopic);
kafkaClient = new Kafka({
brokers: kafkaBootstrapServers.split(','),
logLevel: logLevel.INFO,
logCreator: KafkaJsWinstonLogCreator
});
consumer = kafkaClient.consumer({ groupId: 'js-executor-group' });
producer = kafkaClient.producer();
const messageProcessor = new JsInvokeMessageProcessor(new KafkaProducer());
await consumer.connect();
await producer.connect();
await consumer.subscribe({ topic: kafkaRequestTopic});
logger.info('Started ThingsBoard JavaScript Executor Microservice.');
await consumer.run({
eachMessage: async ({ topic, partition, message }) => {
messageProcessor.onJsInvokeMessage(message);
},
});
} catch (e) {
logger.error('Failed to start ThingsBoard JavaScript Executor Microservice: %s', e.message);
logger.error(e.stack);
exit(-1);
}
})();
process.on('exit', () => {
exit(0);
});
async function exit(status) {
logger.info('Exiting with status: %d ...', status);
if (consumer) {
logger.info('Stopping Kafka Consumer...');
var _consumer = consumer;
consumer = null;
try {
await _consumer.disconnect();
logger.info('Kafka Consumer stopped.');
await disconnectProducer();
process.exit(status);
} catch (e) {
logger.info('Kafka Consumer stop error.');
await disconnectProducer();
process.exit(status);
}
} else {
process.exit(status);
}
}
async function disconnectProducer() {
if (producer) {
logger.info('Stopping Kafka Producer...');
var _producer = producer;
producer = null;
try {
await _producer.disconnect();
logger.info('Kafka Producer stopped.');
} catch (e) {
logger.info('Kafka Producer stop error.');
}
}
}

92
msa/js-executor/server.js
View File

@ -13,89 +13,17 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
const { logLevel, Kafka } = require('kafkajs');
const config = require('config'),
JsInvokeMessageProcessor = require('./api/jsInvokeMessageProcessor'),
logger = require('./config/logger')._logger('main'),
KafkaJsWinstonLogCreator = require('./config/logger').KafkaJsWinstonLogCreator;
const config = require('config');
var kafkaClient;
var consumer;
var producer;
(async() => {
try {
logger.info('Starting ThingsBoard JavaScript Executor Microservice...');
const kafkaBootstrapServers = config.get('kafka.bootstrap.servers');
const kafkaRequestTopic = config.get('kafka.request_topic');
logger.info('Kafka Bootstrap Servers: %s', kafkaBootstrapServers);
logger.info('Kafka Requests Topic: %s', kafkaRequestTopic);
kafkaClient = new Kafka({
brokers: kafkaBootstrapServers.split(','),
logLevel: logLevel.INFO,
logCreator: KafkaJsWinstonLogCreator
});
consumer = kafkaClient.consumer({ groupId: 'js-executor-group' });
producer = kafkaClient.producer();
const messageProcessor = new JsInvokeMessageProcessor(producer);
await consumer.connect();
await producer.connect();
await consumer.subscribe({ topic: kafkaRequestTopic});
logger.info('Started ThingsBoard JavaScript Executor Microservice.');
await consumer.run({
eachMessage: async ({ topic, partition, message }) => {
messageProcessor.onJsInvokeMessage(message);
},
});
} catch (e) {
logger.error('Failed to start ThingsBoard JavaScript Executor Microservice: %s', e.message);
logger.error(e.stack);
exit(-1);
}
})();
process.on('exit', () => {
exit(0);
});
async function exit(status) {
logger.info('Exiting with status: %d ...', status);
if (consumer) {
logger.info('Stopping Kafka Consumer...');
var _consumer = consumer;
consumer = null;
try {
await _consumer.disconnect();
logger.info('Kafka Consumer stopped.');
await disconnectProducer();
process.exit(status);
} catch (e) {
logger.info('Kafka Consumer stop error.');
await disconnectProducer();
process.exit(status);
}
} else {
process.exit(status);
}
const serviceType = config.get('service-type');
switch (serviceType) {
case 'kafka':
require('./queue/kafka/kafkaTemplate');
console.log('Used kafka template.');
break;
default:
console.error('Unknown service type: ', serviceType);
process.exit(-1);
}
async function disconnectProducer() {
if (producer) {
logger.info('Stopping Kafka Producer...');
var _producer = producer;
producer = null;
try {
await _producer.disconnect();
logger.info('Kafka Producer stopped.');
} catch (e) {
logger.info('Kafka Producer stop error.');
}
}
}