dendi-wang/thingsboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixes and refactoring (#2761)

Browse Source 
* fix sqs js executor and refactored RemoteJsInvokeService

* added REMOTE_JS_MAX_REQUEST_TIMEOUT=20000 for aws-sqs, pubsub, service-bus docker environments

* added REMOTE_JS_MAX_REQUEST_TIMEOUT=20000 for aws-sqs, pubsub, service-bus docker environments

* refactored

* docker-compose.pubsub.yml improvements

* rabbitmq js executor improvements
This commit is contained in:
Yevhen Bondarenko 2020-05-13 14:08:54 +03:00 committed by GitHub
parent dc5eb6395a
commit 7ee2cdfe3e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 28 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
application/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java
View File

@ -59,22 +59,22 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService {
@Value("${js.remote.stats.enabled:false}")
private boolean statsEnabled;
private final AtomicInteger kafkaPushedMsgs = new AtomicInteger(0);
private final AtomicInteger kafkaInvokeMsgs = new AtomicInteger(0);
private final AtomicInteger kafkaEvalMsgs = new AtomicInteger(0);
private final AtomicInteger kafkaFailedMsgs = new AtomicInteger(0);
private final AtomicInteger kafkaTimeoutMsgs = new AtomicInteger(0);
private final AtomicInteger queuePushedMsgs = new AtomicInteger(0);
private final AtomicInteger queueInvokeMsgs = new AtomicInteger(0);
private final AtomicInteger queueEvalMsgs = new AtomicInteger(0);
private final AtomicInteger queueFailedMsgs = new AtomicInteger(0);
private final AtomicInteger queueTimeoutMsgs = new AtomicInteger(0);
@Scheduled(fixedDelayString = "${js.remote.stats.print_interval_ms}")
public void printStats() {
if (statsEnabled) {
int pushedMsgs = kafkaPushedMsgs.getAndSet(0);
int invokeMsgs = kafkaInvokeMsgs.getAndSet(0);
int evalMsgs = kafkaEvalMsgs.getAndSet(0);
int failed = kafkaFailedMsgs.getAndSet(0);
int timedOut = kafkaTimeoutMsgs.getAndSet(0);
int pushedMsgs = queuePushedMsgs.getAndSet(0);
int invokeMsgs = queueInvokeMsgs.getAndSet(0);
int evalMsgs = queueEvalMsgs.getAndSet(0);
int failed = queueFailedMsgs.getAndSet(0);
int timedOut = queueTimeoutMsgs.getAndSet(0);
if (pushedMsgs > 0 || invokeMsgs > 0 || evalMsgs > 0 || failed > 0 || timedOut > 0) {
log.info("Kafka JS Invoke Stats: pushed [{}] received [{}] invoke [{}] eval [{}] failed [{}] timedOut [{}]",
log.info("Queue JS Invoke Stats: pushed [{}] received [{}] invoke [{}] eval [{}] failed [{}] timedOut [{}]",
pushedMsgs, invokeMsgs + evalMsgs, invokeMsgs, evalMsgs, failed, timedOut);
}
}
@ -116,19 +116,19 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService {
if (maxRequestsTimeout > 0) {
future = Futures.withTimeout(future, maxRequestsTimeout, TimeUnit.MILLISECONDS, timeoutExecutorService);
}
kafkaPushedMsgs.incrementAndGet();
queuePushedMsgs.incrementAndGet();
Futures.addCallback(future, new FutureCallback<TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>>() {
@Override
public void onSuccess(@Nullable TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse> result) {
kafkaEvalMsgs.incrementAndGet();
queueEvalMsgs.incrementAndGet();
}
@Override
public void onFailure(Throwable t) {
if (t instanceof TimeoutException || (t.getCause() != null && t.getCause() instanceof TimeoutException)) {
kafkaTimeoutMsgs.incrementAndGet();
queueTimeoutMsgs.incrementAndGet();
}
kafkaFailedMsgs.incrementAndGet();
queueFailedMsgs.incrementAndGet();
}
}, MoreExecutors.directExecutor());
return Futures.transform(future, response -> {
@ -170,20 +170,20 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService {
if (maxRequestsTimeout > 0) {
future = Futures.withTimeout(future, maxRequestsTimeout, TimeUnit.MILLISECONDS, timeoutExecutorService);
}
kafkaPushedMsgs.incrementAndGet();
queuePushedMsgs.incrementAndGet();
Futures.addCallback(future, new FutureCallback<TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse>>() {
@Override
public void onSuccess(@Nullable TbProtoQueueMsg<JsInvokeProtos.RemoteJsResponse> result) {
kafkaInvokeMsgs.incrementAndGet();
queueInvokeMsgs.incrementAndGet();
}
@Override
public void onFailure(Throwable t) {
onScriptExecutionError(scriptId);
if (t instanceof TimeoutException || (t.getCause() != null && t.getCause() instanceof TimeoutException)) {
kafkaTimeoutMsgs.incrementAndGet();
queueTimeoutMsgs.incrementAndGet();
}
kafkaFailedMsgs.incrementAndGet();
queueFailedMsgs.incrementAndGet();
}
}, MoreExecutors.directExecutor());
return Futures.transform(future, response -> {

4
docker/docker-compose.pubsub.yml
View File

@ -19,10 +19,10 @@ version: '2.2'
services:
tb-js-executor:
env_file:
- queue-pubsub.env.env
- queue-pubsub.env
tb-core1:
env_file:
- queue-pubsub.env.env
- queue-pubsub.env
depends_on:
- zookeeper
- redis

1
docker/queue-aws-sqs.env
View File

@ -2,3 +2,4 @@ TB_QUEUE_TYPE=aws-sqs
TB_QUEUE_AWS_SQS_ACCESS_KEY_ID=YOUR_KEY
TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY=YOUR_SECRET
TB_QUEUE_AWS_SQS_REGION=YOUR_REGION
REMOTE_JS_MAX_REQUEST_TIMEOUT=60000

3
docker/queue-pubsub.env
View File

@ -1,3 +1,4 @@
TB_QUEUE_TYPE=pubsub
TB_QUEUE_PUBSUB_PROJECT_ID=YOUR_PROJECT_ID
TB_QUEUE_PUBSUB_SERVICE_ACCOUNT=YOUR_SERVICE_ACCOUNT
TB_QUEUE_PUBSUB_SERVICE_ACCOUNT=YOUR_SERVICE_ACCOUNT
REMOTE_JS_MAX_REQUEST_TIMEOUT=60000

3
docker/queue-service-bus.env
View File

@ -1,4 +1,5 @@
TB_QUEUE_TYPE=service-bus
TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME=YOUR_NAMESPACE_NAME
TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME=YOUR_SAS_KEY_NAME
TB_QUEUE_SERVICE_BUS_SAS_KEY=YOUR_SAS_KEY
TB_QUEUE_SERVICE_BUS_SAS_KEY=YOUR_SAS_KEY
REMOTE_JS_MAX_REQUEST_TIMEOUT=60000

2
msa/js-executor/queue/awsSqsTemplate.js
View File

@ -100,7 +100,7 @@ function AwsSqsProducer() {
const params = {
MaxNumberOfMessages: 10,
QueueUrl: requestQueueURL,
WaitTimeSeconds: poolInterval / 1000
WaitTimeSeconds: pollInterval / 1000
};
while (!stopped) {
let pollStartTs = new Date().getTime();

3
msa/js-executor/queue/rabbitmqTemplate.js
View File

@ -68,9 +68,8 @@ function RabbitMqProducer() {
(async () => {
try {
logger.info('Starting ThingsBoard JavaScript Executor Microservice...');
const url = `amqp://${host}:${port}${vhost}`;
const url = `amqp://${username}:${password}@${host}:${port}${vhost}`;
amqp.credentials.amqplain(username, password);
connection = await new Promise((resolve, reject) => {
amqp.connect(url, function (err, connection) {
if (err) {