Merge branch 'vvlladd28-bugs/js-executor/greceful-shutdows' into develop/3.4

msa/js-executor/api/httpServer.ts

@@ -16,12 +16,15 @@
 
 import express from 'express';
 import { _logger}  from '../config/logger';
+import http from 'http';
+import { Socket } from 'net';
 
 export class HttpServer {
 
   private logger = _logger('httpServer');
   private app = express();
-  private server;
+  private server: http.Server | null;
+  private connections: Socket[] = [];
 
   constructor(httpPort: number) {
     this.app.get('/livenessProbe', async (req, res) => {
@@ -32,15 +35,31 @@ export class HttpServer {
     })
 
     this.server = this.app.listen(httpPort, () => {
-      this.logger.info('Started http endpoint on port %s. Please, use /livenessProbe !', httpPort);
+      this.logger.info('Started HTTP endpoint on port %s. Please, use /livenessProbe !', httpPort);
     }).on('error', (error) => {
       this.logger.error(error);
     });
-  }
 
-  stop() {
+    this.server.on('connection', connection => {
-    this.server.close(() => {
+      this.connections.push(connection);
-      this.logger.info('Http server stop');
+      connection.on('close', () => this.connections = this.connections.filter(curr => curr !== connection));
     });
   }
+
+  async stop() {
+    if (this.server) {
+      this.logger.info('Stopping HTTP Server...');
+      const _server = this.server;
+      this.server = null;
+      this.connections.forEach(curr => curr.end(() => curr.destroy()));
+      await new Promise<void>(
+          (resolve, reject) => {
+            _server.close((err) => {
+              this.logger.info('HTTP Server stopped.');
+              resolve();
+            });
+          }
+      );
+    }
+  }
 }

msa/js-executor/docker/Dockerfile

@@ -29,6 +29,7 @@ COPY package/linux/conf ./conf
 COPY package/linux/conf ./config
 COPY src/api ./api
 COPY src/queue ./queue
+COPY src/config ./config
 COPY src/server.js ./
 
 RUN chmod a+x /tmp/*.sh \

msa/js-executor/package.json

@@ -20,7 +20,7 @@
     "config": "^3.3.7",
     "express": "^4.18.1",
     "js-yaml": "^4.1.0",
-    "kafkajs": "^2.0.2",
+    "kafkajs": "^2.1.0",
     "long": "^5.2.0",
     "uuid-parse": "^1.1.0",
     "uuid-random": "^1.3.2",

msa/js-executor/queue/awsSqsTemplate.ts

@@ -54,13 +54,12 @@ export class AwsSqsTemplate implements IQueue {
         FifoQueue: 'true'
     };
 
+    name = 'AWS SQS';
+
     constructor() {
     }
 
     async init() {
-        try {
-            this.logger.info('Starting ThingsBoard JavaScript Executor Microservice...');
-
         this.sqsClient = new SQSClient({
             apiVersion: '2012-11-05',
             credentials: {
@@ -126,11 +125,6 @@ export class AwsSqsTemplate implements IQueue {
                 }
             }
         }
-        } catch (e: any) {
-            this.logger.error('Failed to start ThingsBoard JavaScript Executor Microservice: %s', e.message);
-            this.logger.error(e.stack);
-            await this.exit(-1);
-        }
     }
 
     async send(responseTopic: string, scriptId: string, rawResponse: Buffer, headers: any): Promise<any> {
@@ -187,29 +181,21 @@ export class AwsSqsTemplate implements IQueue {
         return result.QueueUrl || '';
     }
 
-    static async build(): Promise<AwsSqsTemplate> {
+    async destroy(): Promise<void> {
-        const queue = new AwsSqsTemplate();
-        await queue.init();
-        return queue;
-    }
-
-    async exit(status: number) {
         this.stopped = true;
-        this.logger.info('Exiting with status: %d ...', status);
+        this.logger.info('Stopping AWS SQS resources...');
         if (this.sqsClient) {
-            this.logger.info('Stopping Aws Sqs client.')
+            this.logger.info('Stopping AWS SQS client...');
             try {
-                this.sqsClient.destroy();
+                const _sqsClient = this.sqsClient;
                 // @ts-ignore
                 delete this.sqsClient;
-                this.logger.info('Aws Sqs client stopped.')
+                _sqsClient.destroy();
-                process.exit(status);
+                this.logger.info('AWS SQS client stopped.');
             } catch (e: any) {
-                this.logger.info('Aws Sqs client stop error.');
+                this.logger.info('AWS SQS client stop error.');
-                process.exit(status);
-            }
-        } else {
-            process.exit(status);
             }
         }
+        this.logger.info('AWS SQS resources stopped.')
+    }
 }

msa/js-executor/queue/kafkaTemplate.ts

@@ -51,13 +51,12 @@ export class KafkaTemplate implements IQueue {
     private batchMessages: TopicMessages[] = [];
     private sendLoopInstance: NodeJS.Timeout;
 
+    name = 'Kafka';
+
     constructor() {
     }
 
     async init(): Promise<void> {
-        try {
-            this.logger.info('Starting ThingsBoard JavaScript Executor Microservice...');
-
         const kafkaBootstrapServers: string = config.get('kafka.bootstrap.servers');
         const requestTopic: string = config.get('request_topic');
         const useConfluent = config.get('kafka.use_confluent_cloud');
@@ -119,11 +118,11 @@ export class KafkaTemplate implements IQueue {
 
         const {CRASH} = this.consumer.events;
 
-            this.consumer.on(CRASH, e => {
+        this.consumer.on(CRASH, async (e) => {
             this.logger.error(`Got consumer CRASH event, should restart: ${e.payload.restart}`);
             if (!e.payload.restart) {
                 this.logger.error('Going to exit due to not retryable error!');
-                    this.exit(-1);
+                await this.destroy();
             }
         });
 
@@ -133,7 +132,6 @@ export class KafkaTemplate implements IQueue {
         this.sendLoopWithLinger();
         await this.consumer.subscribe({topic: requestTopic});
 
-            this.logger.info('Started ThingsBoard JavaScript Executor Microservice.');
         await this.consumer.run({
             partitionsConsumedConcurrently: this.partitionsConsumedConcurrently,
             eachMessage: async ({topic, partition, message}) => {
@@ -149,12 +147,6 @@ export class KafkaTemplate implements IQueue {
                 messageProcessor.onJsInvokeMessage(msg);
             },
         });
-
-        } catch (e: any) {
-            this.logger.error('Failed to start ThingsBoard JavaScript Executor Microservice: %s', e.message);
-            this.logger.error(e.stack);
-            await this.exit(-1);
-        }
 }
 
     async send(responseTopic: string, scriptId: string, rawResponse: Buffer, headers: any): Promise<any> {
@@ -235,41 +227,33 @@ export class KafkaTemplate implements IQueue {
         }, this.linger);
     }
 
-    static async build(): Promise<KafkaTemplate> {
+    async destroy(): Promise<void> {
-        const queue = new KafkaTemplate();
+        this.logger.info('Stopping Kafka resources...');
-        await queue.init();
-        return queue;
-    }
-
-
-    async exit(status: number): Promise<void> {
-        this.logger.info('Exiting with status: %d ...', status);
 
         if (this.kafkaAdmin) {
             this.logger.info('Stopping Kafka Admin...');
-            await this.kafkaAdmin.disconnect();
+            const _kafkaAdmin = this.kafkaAdmin;
             // @ts-ignore
             delete this.kafkaAdmin;
+            await _kafkaAdmin.disconnect();
             this.logger.info('Kafka Admin stopped.');
         }
 
         if (this.consumer) {
             this.logger.info('Stopping Kafka Consumer...');
             try {
-                await this.consumer.disconnect();
+                const _consumer = this.consumer;
                 // @ts-ignore
                 delete this.consumer;
+                await _consumer.disconnect();
                 this.logger.info('Kafka Consumer stopped.');
                 await this.disconnectProducer();
-                process.exit(status);
             } catch (e: any) {
                 this.logger.info('Kafka Consumer stop error.');
                 await this.disconnectProducer();
-                process.exit(status);
             }
-        } else {
-            process.exit(status);
         }
+        this.logger.info('Kafka resources stopped.');
     }
 
     private async disconnectProducer(): Promise<void> {
@@ -279,13 +263,15 @@ export class KafkaTemplate implements IQueue {
                 this.logger.info('Stopping loop...');
                 clearTimeout(this.sendLoopInstance);
                 await this.sendMessagesAsBatch();
-                await this.producer.disconnect();
+                const _producer = this.producer;
                 // @ts-ignore
                 delete this.producer;
+                await _producer.disconnect();
                 this.logger.info('Kafka Producer stopped.');
             } catch (e) {
                 this.logger.info('Kafka Producer stop error.');
             }
         }
     }
+
 }

msa/js-executor/queue/pubSubTemplate.ts

@@ -34,12 +34,12 @@ export class PubSubTemplate implements IQueue {
     private topics: string[] = [];
     private subscriptions: string[] = [];
 
+    name = 'Pub/Sub';
+
     constructor() {
     }
 
     async init() {
-        try {
-            this.logger.info('Starting ThingsBoard JavaScript Executor Microservice...');
         this.pubSubClient = new PubSub({
             projectId: this.projectId,
             credentials: this.credentials
@@ -78,12 +78,6 @@ export class PubSubTemplate implements IQueue {
         };
 
         subscription.on('message', messageHandler);
-
-        } catch (e: any) {
-            this.logger.error('Failed to start ThingsBoard JavaScript Executor Microservice: %s', e.message);
-            this.logger.error(e.stack);
-            await this.exit(-1);
-        }
     }
 
     async send(responseTopic: string, scriptId: string, rawResponse: Buffer, headers: any): Promise<any> {
@@ -147,29 +141,21 @@ export class PubSubTemplate implements IQueue {
         }
     }
 
-    static async build(): Promise<PubSubTemplate> {
+    async destroy(): Promise<void> {
-        const queue = new PubSubTemplate();
+        this.logger.info('Stopping Pub/Sub resources...');
-        await queue.init();
-        return queue;
-    }
-
-    async exit(status: number): Promise<void> {
-        this.logger.info('Exiting with status: %d ...', status);
         if (this.pubSubClient) {
-            this.logger.info('Stopping Pub/Sub client.')
+            this.logger.info('Stopping Pub/Sub client...');
             try {
-                await this.pubSubClient.close();
+                const _pubSubClient = this.pubSubClient;
                 // @ts-ignore
                 delete this.pubSubClient;
-                this.logger.info('Pub/Sub client stopped.')
+                await _pubSubClient.close();
-                process.exit(status);
+                this.logger.info('Pub/Sub client stopped.');
             } catch (e) {
                 this.logger.info('Pub/Sub client stop error.');
-                process.exit(status);
             }
-        } else {
-            process.exit(status);
         }
+        this.logger.info('Pub/Sub resources stopped.');
     }
 }
 

msa/js-executor/queue/queue.models.ts

@@ -15,7 +15,8 @@
 ///
 
 export interface IQueue {
+    name: string;
     init(): Promise<void>;
     send(responseTopic: string, scriptId: string, rawResponse: Buffer, headers: any): Promise<any>;
-    exit(status: number): Promise<void>;
+    destroy(): Promise<void>;
 }

msa/js-executor/queue/rabbitmqTemplate.ts

@@ -44,13 +44,12 @@ export class RabbitMqTemplate implements IQueue {
     private stopped = false;
     private topics: string[] = [];
 
+    name = 'RabbitMQ';
+
     constructor() {
     }
 
     async init(): Promise<void> {
-        try {
-            this.logger.info('Starting ThingsBoard JavaScript Executor Microservice...');
-
         const url = `amqp://${this.username}:${this.password}@${this.host}:${this.port}${this.vhost}`;
         this.connection = await amqp.connect(url);
         this.channel = await this.connection.createConfirmChannel();
@@ -75,11 +74,6 @@ export class RabbitMqTemplate implements IQueue {
                 }
             }
         }
-        } catch (e: any) {
-            this.logger.error('Failed to start ThingsBoard JavaScript Executor Microservice: %s', e.message);
-            this.logger.error(e.stack);
-            await this.exit(-1);
-        }
     }
 
     async send(responseTopic: string, scriptId: string, rawResponse: Buffer, headers: any): Promise<any> {
@@ -114,38 +108,31 @@ export class RabbitMqTemplate implements IQueue {
         return this.channel.assertQueue(topic, this.queueOptions);
     }
 
-    static async build(): Promise<RabbitMqTemplate> {
+    async destroy() {
-        const queue = new RabbitMqTemplate();
+        this.logger.info('Stopping RabbitMQ resources...');
-        await queue.init();
-        return queue;
-    }
-
-    async exit(status: number) {
-        this.logger.info('Exiting with status: %d ...', status);
 
         if (this.channel) {
-            this.logger.info('Stopping RabbitMq chanel.')
+            this.logger.info('Stopping RabbitMQ chanel...');
-            await this.channel.close();
+            const _channel = this.channel;
             // @ts-ignore
             delete this.channel;
-            this.logger.info('RabbitMq chanel stopped');
+            await _channel.close();
+            this.logger.info('RabbitMQ chanel stopped');
         }
 
         if (this.connection) {
-            this.logger.info('Stopping RabbitMq connection.')
+            this.logger.info('Stopping RabbitMQ connection...')
             try {
-                await this.connection.close();
+                const _connection = this.connection;
                 // @ts-ignore
                 delete this.connection;
-                this.logger.info('RabbitMq client connection.')
+                await _connection.close();
-                process.exit(status);
+                this.logger.info('RabbitMQ client connection.');
             } catch (e) {
-                this.logger.info('RabbitMq connection stop error.');
+                this.logger.info('RabbitMQ connection stop error.');
-                process.exit(status);
             }
-        } else {
-            process.exit(status);
         }
+        this.logger.info('RabbitMQ resources stopped.')
     }
 
 }

msa/js-executor/queue/serviceBusTemplate.ts

@@ -44,13 +44,12 @@ export class ServiceBusTemplate implements IQueue {
     private receiver: ServiceBusReceiver;
     private senderMap = new Map<string, ServiceBusSender>();
 
+    name = 'Azure Service Bus';
+
     constructor() {
     }
 
     async init() {
-        try {
-            this.logger.info('Starting ThingsBoard JavaScript Executor Microservice...');
-
         const connectionString = `Endpoint=sb://${this.namespaceName}.servicebus.windows.net/;SharedAccessKeyName=${this.sasKeyName};SharedAccessKey=${this.sasKey}`;
         this.sbClient = new ServiceBusClient(connectionString)
         this.serviceBusService = new ServiceBusAdministrationClient(connectionString);
@@ -81,11 +80,6 @@ export class ServiceBusTemplate implements IQueue {
             this.logger.error('Failed to receive message from queue.', error);
         };
         this.receiver.subscribe({processMessage: messageHandler, processError: errorHandler})
-        } catch (e: any) {
-            this.logger.error('Failed to start ThingsBoard JavaScript Executor Microservice: %s', e.message);
-            this.logger.error(e.stack);
-            await this.exit(-1);
-        }
     }
 
     async send(responseTopic: string, scriptId: string, rawResponse: Buffer, headers: any): Promise<any> {
@@ -135,41 +129,46 @@ export class ServiceBusTemplate implements IQueue {
         }
     }
 
-    static async build(): Promise<ServiceBusTemplate> {
+    async destroy() {
-        const queue = new ServiceBusTemplate();
-        await queue.init();
-        return queue;
-    }
-
-    async exit(status: number) {
-        this.logger.info('Exiting with status: %d ...', status);
         this.logger.info('Stopping Azure Service Bus resources...')
         if (this.receiver) {
+            this.logger.info('Stopping Service Bus Receiver...');
             try {
-                await this.receiver.close();
+                const _receiver = this.receiver;
                 // @ts-ignore
                 delete this.receiver;
+                await _receiver.close();
+                this.logger.info('Service Bus Receiver stopped.');
             } catch (e) {
+                this.logger.info('Service Bus Receiver stop error.');
             }
         }
 
-        this.senderMap.forEach(k => {
+        this.logger.info('Stopping Service Bus Senders...');
-            try {
+        const senders: Promise<void>[] = [];
-                k.close();
+        this.senderMap.forEach((sender) => {
-            } catch (e) {
+            senders.push(sender.close());
-            }
         });
         this.senderMap.clear();
+        try {
+            await Promise.all(senders);
+            this.logger.info('Service Bus Senders stopped.');
+        } catch (e) {
+            this.logger.info('Service Bus Senders stop error.');
+        }
 
         if (this.sbClient) {
+            this.logger.info('Stopping Service Bus Client...');
             try {
-                await this.sbClient.close();
+                const _sbClient = this.sbClient;
                 // @ts-ignore
                 delete this.sbClient;
+                await _sbClient.close();
+                this.logger.info('Service Bus Client stopped.');
             } catch (e) {
+                this.logger.info('Service Bus Client stop error.');
             }
         }
         this.logger.info('Azure Service Bus resources stopped.')
-        process.exit(status);
     }
 }

msa/js-executor/server.ts

@@ -30,57 +30,66 @@ logger.info('===CONFIG BEGIN===');
 logger.info(JSON.stringify(config, null, 4));
 logger.info('===CONFIG END===');
 
-const serviceType = config.get('queue_type');
+const serviceType: string = config.get('queue_type');
 const httpPort = Number(config.get('http_port'));
-let queues: IQueue;
+let queues: IQueue | null;
-let httpServer: HttpServer;
+let httpServer: HttpServer | null;
 
 (async () => {
-    switch (serviceType) {
+    logger.info('Starting ThingsBoard JavaScript Executor Microservice...');
-        case 'kafka':
+    try {
-            logger.info('Starting kafka template.');
+        queues = await createQueue(serviceType);
-            queues = await KafkaTemplate.build();
+        logger.info(`Starting ${queues.name} template...`);
-            logger.info('kafka template started.');
+        await queues.init();
-            break;
+        logger.info(`${queues.name} template started.`);
-        case 'pubsub':
+        httpServer = new HttpServer(httpPort);
-            logger.info('Starting Pub/Sub template.')
+    } catch (e: any) {
-            queues = await PubSubTemplate.build();
+        logger.error('Failed to start ThingsBoard JavaScript Executor Microservice: %s', e.message);
-            logger.info('Pub/Sub template started.')
+        logger.error(e.stack);
-            break;
+        await exit(-1);
-        case 'aws-sqs':
-            logger.info('Starting Aws Sqs template.')
-            queues = await AwsSqsTemplate.build();
-            logger.info('Aws Sqs template started.')
-            break;
-        case 'rabbitmq':
-            logger.info('Starting RabbitMq template.')
-            queues = await RabbitMqTemplate.build();
-            logger.info('RabbitMq template started.')
-            break;
-        case 'service-bus':
-            logger.info('Starting Azure Service Bus template.')
-            queues = await ServiceBusTemplate.build();
-            logger.info('Azure Service Bus template started.')
-            break;
-        default:
-            logger.error('Unknown service type: ', serviceType);
-            process.exit(-1);
     }
 
-    httpServer = new HttpServer(httpPort);
 })();
 
-process.on('SIGTERM', () => {
+async function createQueue(serviceType: string): Promise<IQueue> {
-    logger.info('SIGTERM signal received');
+    switch (serviceType) {
-    process.exit(0);
+        case 'kafka':
+            return new KafkaTemplate();
+        case 'pubsub':
+            return new PubSubTemplate();
+        case 'aws-sqs':
+            return new AwsSqsTemplate();
+        case 'rabbitmq':
+            return new RabbitMqTemplate();
+        case 'service-bus':
+            return new ServiceBusTemplate();
+        default:
+            throw new Error('Unknown service type: ' + serviceType);
+    }
+}
+
+[`SIGINT`, `SIGUSR1`, `SIGUSR2`, `uncaughtException`, `SIGTERM`].forEach((eventType) => {
+    process.on(eventType, async () => {
+        logger.info(`${eventType} signal received`);
+        await exit(0);
+    })
+})
+
+process.on('exit', (code: number) => {
+    logger.info(`ThingsBoard JavaScript Executor Microservice has been stopped. Exit code: ${code}.`);
 });
 
-process.on('exit', async () => {
+async function exit(status: number) {
+    logger.info('Exiting with status: %d ...', status);
     if (httpServer) {
-        httpServer.stop();
+        const _httpServer = httpServer;
+        httpServer = null;
+        await _httpServer.stop();
     }
     if (queues) {
-        queues.exit(0);
+        const _queues = queues;
+        queues = null;
+        await _queues.destroy();
+    }
+    process.exit(status);
 }
-    logger.info('JavaScript Executor Microservice has been stopped.');
-});

msa/js-executor/yarn.lock

@@ -2670,10 +2670,10 @@ jws@^4.0.0:
     jwa "^2.0.0"
     safe-buffer "^5.0.1"
 
-kafkajs@^2.0.2:
+kafkajs@^2.1.0:
-  version "2.0.2"
+  version "2.1.0"
-  resolved "https://registry.yarnpkg.com/kafkajs/-/kafkajs-2.0.2.tgz#cdfc8f57aa4fd69f6d9ca1cce4ee89bbc2a3a1f9"
+  resolved "https://registry.yarnpkg.com/kafkajs/-/kafkajs-2.1.0.tgz#32ede4e8080cc75586c5e4406eeb582fa73f7b1e"
-  integrity sha512-g6CM3fAenofOjR1bfOAqeZUEaSGhNtBscNokybSdW1rmIKYNwBPC9xQzwulFJm36u/xcxXUiCl/L/qfslapihA==
+  integrity sha512-6IYiOdGWvFPbSbVB+AV3feT+A7vzw5sXm7Ze4QTfP7FRNdY8pGcpiNPvD2lfgYFD8Dm9KbMgBgTt2mf8KaIkzw==
 
 keyv@^3.0.0:
   version "3.1.0"