From 9ef3445b7723a48ad2b96d6cf3d1b3830e946635 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Thu, 30 Apr 2020 16:24:12 +0300 Subject: [PATCH 01/19] refactored --- .../config/custom-environment-variables.yml | 12 ++++++------ msa/js-executor/config/default.yml | 12 ++++++------ msa/js-executor/queue/awsSqsTemplate.js | 2 +- msa/js-executor/queue/kafkaTemplate.js | 2 +- msa/js-executor/queue/pubSubTemplate.js | 2 +- msa/js-executor/queue/rabbitmqTemplate.js | 2 +- msa/js-executor/queue/serviceBusTemplate.js | 2 +- msa/js-executor/server.js | 2 +- 8 files changed, 18 insertions(+), 18 deletions(-) diff --git a/msa/js-executor/config/custom-environment-variables.yml b/msa/js-executor/config/custom-environment-variables.yml index b290719739..c573274801 100644 --- a/msa/js-executor/config/custom-environment-variables.yml +++ b/msa/js-executor/config/custom-environment-variables.yml @@ -14,7 +14,7 @@ # limitations under the License. # -service-type: "TB_SERVICE_TYPE" #kafka (Apache Kafka) or aws-sqs (AWS SQS) or pubsub (PubSub) or service-bus (Azure Service Bus) or rabbitmq (RabbitMQ) +queue_type: "TB_QUEUE_TYPE" #kafka (Apache Kafka) or aws-sqs (AWS SQS) or pubsub (PubSub) or service-bus (Azure Service Bus) or rabbitmq (RabbitMQ) request_topic: "REMOTE_JS_EVAL_REQUEST_TOPIC" js: @@ -25,18 +25,18 @@ kafka: # Kafka Bootstrap Servers servers: "TB_KAFKA_SERVERS" replication_factor: "TB_QUEUE_KAFKA_REPLICATION_FACTOR" - topic-properties: "TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES" + topic_properties: "TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES" pubsub: project_id: "TB_QUEUE_PUBSUB_PROJECT_ID" service_account: "TB_QUEUE_PUBSUB_SERVICE_ACCOUNT" - queue-properties: "TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES" + queue_properties: "TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES" aws_sqs: access_key_id: "TB_QUEUE_AWS_SQS_ACCESS_KEY_ID" secret_access_key: "TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY" region: "TB_QUEUE_AWS_SQS_REGION" - queue-properties: "TB_QUEUE_AWS_SQS_JE_QUEUE_PROPERTIES" + queue_properties: "TB_QUEUE_AWS_SQS_JE_QUEUE_PROPERTIES" rabbitmq: host: "TB_QUEUE_RABBIT_MQ_HOST" @@ -44,14 +44,14 @@ rabbitmq: virtual_host: "TB_QUEUE_RABBIT_MQ_VIRTUAL_HOST" username: "TB_QUEUE_RABBIT_MQ_USERNAME" password: "TB_QUEUE_RABBIT_MQ_PASSWORD" - queue-properties: "TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES" + queue_properties: "TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES" service_bus: namespace_name: "TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME" sas_key_name: "TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME" sas_key: "TB_QUEUE_SERVICE_BUS_SAS_KEY" max_messages: "TB_QUEUE_SERVICE_BUS_MAX_MESSAGES" - queue-properties: "TB_QUEUE_SERVICE_BUS_JE_QUEUE_PROPERTIES" + queue_properties: "TB_QUEUE_SERVICE_BUS_JE_QUEUE_PROPERTIES" logger: level: "LOGGER_LEVEL" diff --git a/msa/js-executor/config/default.yml b/msa/js-executor/config/default.yml index 3155b051dc..f42b74745f 100644 --- a/msa/js-executor/config/default.yml +++ b/msa/js-executor/config/default.yml @@ -14,7 +14,7 @@ # limitations under the License. # -service-type: "kafka" +queue_type: "kafka" request_topic: "js_eval.requests" js: @@ -25,13 +25,13 @@ kafka: # Kafka Bootstrap Servers servers: "localhost:9092" replication_factor: "1" - topic-properties: "retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600" + topic_properties: "retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600" pubsub: - queue-properties: "ackDeadlineInSec:30;messageRetentionInSec:604800" + queue_properties: "ackDeadlineInSec:30;messageRetentionInSec:604800" aws_sqs: - queue-properties: "VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800" + queue_properties: "VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800" rabbitmq: host: "localhost" @@ -39,10 +39,10 @@ rabbitmq: virtual_host: "/" username: "admin" password: "password" - queue-properties: "x-max-length-bytes:1048576000;x-message-ttl:604800000" + queue_properties: "x-max-length-bytes:1048576000;x-message-ttl:604800000" service_bus: - queue-properties: "lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800" + queue_properties: "lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800" logger: level: "info" diff --git a/msa/js-executor/queue/awsSqsTemplate.js b/msa/js-executor/queue/awsSqsTemplate.js index 0396824af1..a5338c5e73 100644 --- a/msa/js-executor/queue/awsSqsTemplate.js +++ b/msa/js-executor/queue/awsSqsTemplate.js @@ -26,7 +26,7 @@ const accessKeyId = config.get('aws_sqs.access_key_id'); const secretAccessKey = config.get('aws_sqs.secret_access_key'); const region = config.get('aws_sqs.region'); const AWS = require('aws-sdk'); -const queueProperties = config.get('aws_sqs.queue-properties'); +const queueProperties = config.get('aws_sqs.queue_properties'); const poolInterval = config.get('js.response_poll_interval'); let queueAttributes = {FifoQueue: 'true', ContentBasedDeduplication: 'true'}; diff --git a/msa/js-executor/queue/kafkaTemplate.js b/msa/js-executor/queue/kafkaTemplate.js index 699ca7be00..e22ec8cd71 100644 --- a/msa/js-executor/queue/kafkaTemplate.js +++ b/msa/js-executor/queue/kafkaTemplate.js @@ -20,7 +20,7 @@ const config = require('config'), logger = require('../config/logger')._logger('kafkaTemplate'), KafkaJsWinstonLogCreator = require('../config/logger').KafkaJsWinstonLogCreator; const replicationFactor = config.get('kafka.replication_factor'); -const topicProperties = config.get('kafka.topic-properties'); +const topicProperties = config.get('kafka.topic_properties'); let kafkaClient; let kafkaAdmin; diff --git a/msa/js-executor/queue/pubSubTemplate.js b/msa/js-executor/queue/pubSubTemplate.js index 17e1b56e1d..7d0b32ea34 100644 --- a/msa/js-executor/queue/pubSubTemplate.js +++ b/msa/js-executor/queue/pubSubTemplate.js @@ -24,7 +24,7 @@ const {PubSub} = require('@google-cloud/pubsub'); const projectId = config.get('pubsub.project_id'); const credentials = JSON.parse(config.get('pubsub.service_account')); const requestTopic = config.get('request_topic'); -const queueProperties = config.get('pubsub.queue-properties'); +const queueProperties = config.get('pubsub.queue_properties'); let pubSubClient; diff --git a/msa/js-executor/queue/rabbitmqTemplate.js b/msa/js-executor/queue/rabbitmqTemplate.js index 1a2905c3a0..732206ff11 100644 --- a/msa/js-executor/queue/rabbitmqTemplate.js +++ b/msa/js-executor/queue/rabbitmqTemplate.js @@ -26,7 +26,7 @@ const port = config.get('rabbitmq.port'); const vhost = config.get('rabbitmq.virtual_host'); const username = config.get('rabbitmq.username'); const password = config.get('rabbitmq.password'); -const queueProperties = config.get('rabbitmq.queue-properties'); +const queueProperties = config.get('rabbitmq.queue_properties'); const poolInterval = config.get('js.response_poll_interval'); const amqp = require('amqplib/callback_api'); diff --git a/msa/js-executor/queue/serviceBusTemplate.js b/msa/js-executor/queue/serviceBusTemplate.js index 034921afb7..20cf664940 100644 --- a/msa/js-executor/queue/serviceBusTemplate.js +++ b/msa/js-executor/queue/serviceBusTemplate.js @@ -26,7 +26,7 @@ const requestTopic = config.get('request_topic'); const namespaceName = config.get('service_bus.namespace_name'); const sasKeyName = config.get('service_bus.sas_key_name'); const sasKey = config.get('service_bus.sas_key'); -const queueProperties = config.get('service_bus.queue-properties'); +const queueProperties = config.get('service_bus.queue_properties'); let sbClient; let receiverClient; diff --git a/msa/js-executor/server.js b/msa/js-executor/server.js index 58361016c4..e57ba62c11 100644 --- a/msa/js-executor/server.js +++ b/msa/js-executor/server.js @@ -16,7 +16,7 @@ const config = require('config'), logger = require('./config/logger')._logger('main'); -const serviceType = config.get('service-type'); +const serviceType = config.get('queue_type'); switch (serviceType) { case 'kafka': logger.info('Starting kafka template.'); From e5c5aa705fa308397536c18c0779480d849ab368 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Thu, 30 Apr 2020 20:14:34 +0300 Subject: [PATCH 02/19] refactored --- docker/tb-js-executor.env | 2 +- msa/js-executor/queue/awsSqsTemplate.js | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/docker/tb-js-executor.env b/docker/tb-js-executor.env index 0b64f43b00..b66073ea44 100644 --- a/docker/tb-js-executor.env +++ b/docker/tb-js-executor.env @@ -1,4 +1,4 @@ - +TB_QUEUE_TYPE=kafka REMOTE_JS_EVAL_REQUEST_TOPIC=js_eval.requests TB_KAFKA_SERVERS=kafka:9092 LOGGER_LEVEL=info diff --git a/msa/js-executor/queue/awsSqsTemplate.js b/msa/js-executor/queue/awsSqsTemplate.js index a5338c5e73..e0ffb55c87 100644 --- a/msa/js-executor/queue/awsSqsTemplate.js +++ b/msa/js-executor/queue/awsSqsTemplate.js @@ -74,11 +74,13 @@ function AwsSqsProducer() { const queues = await getQueues(); - queues.forEach(queueUrl => { - const delimiterPosition = queueUrl.lastIndexOf('/'); - const queueName = queueUrl.substring(delimiterPosition + 1); - queueUrls.set(queueName, queueUrl); - }) + if (queues) { + queues.forEach(queueUrl => { + const delimiterPosition = queueUrl.lastIndexOf('/'); + const queueName = queueUrl.substring(delimiterPosition + 1); + queueUrls.set(queueName, queueUrl); + }); + } parseQueueProperties(); From c7f282d39385473928f4111d015bc02ce30b07bb Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 1 May 2020 12:45:06 +0300 Subject: [PATCH 03/19] Refactoring of the Queue Consumers --- .../AbstractTbQueueConsumerTemplate.java | 136 ++++++++++++++++++ .../queue/kafka/TbKafkaConsumerTemplate.java | 114 ++++----------- 2 files changed, 163 insertions(+), 87 deletions(-) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java new file mode 100644 index 0000000000..084d10fca9 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java @@ -0,0 +1,136 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.common; + +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueMsg; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +@Slf4j +public abstract class AbstractTbQueueConsumerTemplate implements TbQueueConsumer { + + private volatile boolean subscribed; + protected volatile Set partitions; + protected final Lock consumerLock = new ReentrantLock(); + + @Getter + private final String topic; + + public AbstractTbQueueConsumerTemplate(String topic) { + this.topic = topic; + } + + @Override + public void subscribe() { + consumerLock.lock(); + try { + partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true)); + subscribed = false; + } finally { + consumerLock.unlock(); + } + } + + @Override + public void subscribe(Set partitions) { + consumerLock.lock(); + try { + this.partitions = partitions; + subscribed = false; + } finally { + consumerLock.unlock(); + } + } + + @Override + public List poll(long durationInMillis) { + if (!subscribed && partitions == null) { + try { + Thread.sleep(durationInMillis); + } catch (InterruptedException e) { + log.debug("Failed to await subscription", e); + } + } else { + consumerLock.lock(); + try { + if (!subscribed) { + doSubscribe(); + subscribed = true; + } + + List records = doPoll(durationInMillis); + if (!records.isEmpty()) { + List result = new ArrayList<>(records.size()); + records.forEach(record -> { + try { + if (record != null) { + result.add(decode(record)); + } + } catch (IOException e) { + log.error("Failed decode record: [{}]", record); + throw new RuntimeException("Failed to decode record: ", e); + } + }); + return result; + } + } finally { + consumerLock.unlock(); + } + } + return Collections.emptyList(); + } + + @Override + public void commit() { + consumerLock.lock(); + try { + doCommit(); + } finally { + consumerLock.unlock(); + } + } + + @Override + public void unsubscribe() { + consumerLock.lock(); + try { + doUnsubscribe(); + } finally { + consumerLock.unlock(); + } + } + + abstract protected List doPoll(long durationInMillis); + + abstract protected T decode(R record) throws IOException; + + abstract protected void doSubscribe(); + + abstract protected void doCommit(); + + abstract protected void doUnsubscribe(); + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java index 6b1c051eeb..fea31854df 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java @@ -16,7 +16,6 @@ package org.thingsboard.server.queue.kafka; import lombok.Builder; -import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.ConsumerRecord; @@ -24,8 +23,8 @@ import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.KafkaConsumer; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueMsg; +import org.thingsboard.server.queue.common.AbstractTbQueueConsumerTemplate; import java.io.IOException; import java.time.Duration; @@ -33,26 +32,17 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Properties; -import java.util.Set; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; import java.util.stream.Collectors; /** * Created by ashvayka on 24.09.18. */ @Slf4j -public class TbKafkaConsumerTemplate implements TbQueueConsumer { +public class TbKafkaConsumerTemplate extends AbstractTbQueueConsumerTemplate, T> { private final TbQueueAdmin admin; private final KafkaConsumer consumer; private final TbKafkaDecoder decoder; - private volatile boolean subscribed; - private volatile Set partitions; - private final Lock consumerLock; - - @Getter - private final String topic; @Builder private TbKafkaConsumerTemplate(TbKafkaSettings settings, TbKafkaDecoder decoder, @@ -60,6 +50,7 @@ public class TbKafkaConsumerTemplate implements TbQueueCon boolean autoCommit, int autoCommitIntervalMs, int maxPollRecords, TbQueueAdmin admin) { + super(topic); Properties props = settings.toProps(); props.put(ConsumerConfig.CLIENT_ID_CONFIG, clientId); if (groupId != null) { @@ -75,94 +66,43 @@ public class TbKafkaConsumerTemplate implements TbQueueCon this.admin = admin; this.consumer = new KafkaConsumer<>(props); this.decoder = decoder; - this.topic = topic; - this.consumerLock = new ReentrantLock(); } @Override - public void subscribe() { - consumerLock.lock(); - try { - partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true)); - subscribed = false; - } finally { - consumerLock.unlock(); - } + protected void doSubscribe() { + List topicNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); + topicNames.forEach(admin::createTopicIfNotExists); + consumer.subscribe(topicNames); } @Override - public void subscribe(Set partitions) { - consumerLock.lock(); - try { - this.partitions = partitions; - subscribed = false; - } finally { - consumerLock.unlock(); - } - } - - @Override - public List poll(long durationInMillis) { - if (!subscribed && partitions == null) { - try { - Thread.sleep(durationInMillis); - } catch (InterruptedException e) { - log.debug("Failed to await subscription", e); - } + protected List> doPoll(long durationInMillis) { + ConsumerRecords records = consumer.poll(Duration.ofMillis(durationInMillis)); + if (records.isEmpty()) { + return Collections.emptyList(); } else { - consumerLock.lock(); - try { - if (!subscribed) { - List topicNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); - topicNames.forEach(admin::createTopicIfNotExists); - consumer.subscribe(topicNames); - subscribed = true; - } - - ConsumerRecords records = consumer.poll(Duration.ofMillis(durationInMillis)); - if (records.count() > 0) { - List result = new ArrayList<>(); - records.forEach(record -> { - try { - result.add(decode(record)); - } catch (IOException e) { - log.error("Failed decode record: [{}]", record); - } - }); - return result; - } - } finally { - consumerLock.unlock(); - } - } - return Collections.emptyList(); - } - - @Override - public void commit() { - consumerLock.lock(); - try { - consumer.commitAsync(); - } finally { - consumerLock.unlock(); + List> recordList = new ArrayList<>(256); + records.forEach(recordList::add); + return recordList; } } @Override - public void unsubscribe() { - consumerLock.lock(); - try { - if (consumer != null) { - consumer.unsubscribe(); - consumer.close(); - } - } finally { - consumerLock.unlock(); - } - } - public T decode(ConsumerRecord record) throws IOException { return decoder.decode(new KafkaTbQueueMsg(record)); } + @Override + protected void doCommit() { + consumer.commitAsync(); + } + + @Override + protected void doUnsubscribe() { + if (consumer != null) { + consumer.unsubscribe(); + consumer.close(); + } + } + } From eedb38384536215555c97cb4fcff53ae7ee1bf72 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Fri, 1 May 2020 14:15:31 +0300 Subject: [PATCH 04/19] AWS improvements --- .../server/queue/sqs/TbAwsSqsConsumerTemplate.java | 6 +++++- .../server/queue/sqs/TbAwsSqsProducerTemplate.java | 5 ++++- .../server/queue/sqs/TbAwsSqsQueueAttributes.java | 1 - msa/js-executor/package.json | 1 + msa/js-executor/queue/awsSqsTemplate.js | 5 +++-- msa/js-executor/queue/pubSubTemplate.js | 8 +++++--- 6 files changed, 18 insertions(+), 8 deletions(-) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java index 3e71388844..b66cad1504 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java @@ -127,6 +127,11 @@ public class TbAwsSqsConsumerTemplate implements TbQueueCo if (!subscribed) { List topicNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); queueUrls = topicNames.stream().map(this::getQueueUrl).collect(Collectors.toSet()); + + if (consumerExecutor != null) { + consumerExecutor.shutdown(); + } + consumerExecutor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(queueUrls.size() * sqsSettings.getThreadsPerTopic() + 1)); subscribed = true; } @@ -172,7 +177,6 @@ public class TbAwsSqsConsumerTemplate implements TbQueueCo ReceiveMessageRequest request = new ReceiveMessageRequest(); request .withWaitTimeSeconds(waitTimeSeconds) - .withMessageAttributeNames("headers") .withQueueUrl(url) .withMaxNumberOfMessages(MAX_NUM_MSGS); return sqsClient.receiveMessage(request).getMessages(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java index 2d85539184..6110d08c5e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java @@ -37,6 +37,7 @@ import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.DefaultTbQueueMsg; import java.util.Map; +import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; @@ -80,7 +81,9 @@ public class TbAwsSqsProducerTemplate implements TbQueuePr sendMsgRequest.withQueueUrl(getQueueUrl(tpi.getFullTopicName())); sendMsgRequest.withMessageBody(gson.toJson(new DefaultTbQueueMsg(msg))); - sendMsgRequest.withMessageGroupId(msg.getKey().toString()); + sendMsgRequest.withMessageGroupId(tpi.getTopic()); + sendMsgRequest.withMessageDeduplicationId(UUID.randomUUID().toString()); + ListenableFuture future = producerExecutor.submit(() -> sqsClient.sendMessage(sendMsgRequest)); Futures.addCallback(future, new FutureCallback() { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java index 70a9587bef..c6cbbfd256 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java @@ -55,7 +55,6 @@ public class TbAwsSqsQueueAttributes { @PostConstruct private void init() { defaultAttributes.put(QueueAttributeName.FifoQueue.toString(), "true"); - defaultAttributes.put(QueueAttributeName.ContentBasedDeduplication.toString(), "true"); coreAttributes = getConfigs(coreProperties); ruleEngineAttributes = getConfigs(ruleEngineProperties); diff --git a/msa/js-executor/package.json b/msa/js-executor/package.json index 3dadac4f84..60a107061a 100644 --- a/msa/js-executor/package.json +++ b/msa/js-executor/package.json @@ -22,6 +22,7 @@ "azure-sb": "^0.11.1", "long": "^4.0.0", "uuid-parse": "^1.0.0", + "uuid-random": "^1.3.0", "winston": "^3.0.0", "winston-daily-rotate-file": "^3.2.1" }, diff --git a/msa/js-executor/queue/awsSqsTemplate.js b/msa/js-executor/queue/awsSqsTemplate.js index e0ffb55c87..5f95de7d32 100644 --- a/msa/js-executor/queue/awsSqsTemplate.js +++ b/msa/js-executor/queue/awsSqsTemplate.js @@ -19,6 +19,7 @@ const config = require('config'), JsInvokeMessageProcessor = require('../api/jsInvokeMessageProcessor'), logger = require('../config/logger')._logger('awsSqsTemplate'); +const uuid = require('uuid-random'); const requestTopic = config.get('request_topic'); @@ -29,7 +30,7 @@ const AWS = require('aws-sdk'); const queueProperties = config.get('aws_sqs.queue_properties'); const poolInterval = config.get('js.response_poll_interval'); -let queueAttributes = {FifoQueue: 'true', ContentBasedDeduplication: 'true'}; +let queueAttributes = {FifoQueue: 'true'}; let sqsClient; let requestQueueURL; const queueUrls = new Map(); @@ -51,7 +52,7 @@ function AwsSqsProducer() { queueUrls.set(responseTopic, responseQueueUrl); } - let params = {MessageBody: msgBody, QueueUrl: responseQueueUrl, MessageGroupId: scriptId}; + let params = {MessageBody: msgBody, QueueUrl: responseQueueUrl, MessageGroupId: 'js_eval', MessageDeduplicationId: uuid()}; return new Promise((resolve, reject) => { sqsClient.sendMessage(params, function (err, data) { diff --git a/msa/js-executor/queue/pubSubTemplate.js b/msa/js-executor/queue/pubSubTemplate.js index 7d0b32ea34..cc5284022d 100644 --- a/msa/js-executor/queue/pubSubTemplate.js +++ b/msa/js-executor/queue/pubSubTemplate.js @@ -60,9 +60,11 @@ function PubSubProducer() { const topicList = await pubSubClient.getTopics(); if (topicList) { - topicList[0].forEach(topic => { - topics.push(getName(topic.name)); - }); + if (topicList) { + topicList[0].forEach(topic => { + topics.push(getName(topic.name)); + }); + } } const subscriptionList = await pubSubClient.getSubscriptions(); From 8d5c38b743a91c2ad3739c25c47f93ece5480f08 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 1 May 2020 17:43:13 +0300 Subject: [PATCH 05/19] Queue refactoring --- .../processing/AbstractConsumerService.java | 1 - .../BatchTbRuleEngineSubmitStrategy.java | 5 +- .../BurstTbRuleEngineSubmitStrategy.java | 10 +- ...lByEntityIdTbRuleEngineSubmitStrategy.java | 8 - ...lByTenantIdTbRuleEngineSubmitStrategy.java | 1 - .../SequentialTbRuleEngineSubmitStrategy.java | 6 +- ...TbRuleEngineProcessingStrategyFactory.java | 8 +- .../TbServiceBusConsumerTemplate.java | 132 ++++++---------- ...stractParallelTbQueueConsumerTemplate.java | 53 +++++++ .../AbstractTbQueueConsumerTemplate.java | 20 ++- .../queue/kafka/TbKafkaConsumerTemplate.java | 3 +- .../pubsub/TbPubSubConsumerTemplate.java | 108 +++++-------- .../rabbitmq/TbRabbitMqConsumerTemplate.java | 107 ++++--------- .../queue/sqs/TbAwsSqsConsumerTemplate.java | 142 ++++++------------ 14 files changed, 243 insertions(+), 361 deletions(-) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractParallelTbQueueConsumerTemplate.java diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index c2705fcbdc..4007c9e17d 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -23,7 +23,6 @@ import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionChangeEvent; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/BatchTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/BatchTbRuleEngineSubmitStrategy.java index d0b1f7f99a..b9741d2433 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/BatchTbRuleEngineSubmitStrategy.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/BatchTbRuleEngineSubmitStrategy.java @@ -23,7 +23,6 @@ import java.util.LinkedHashMap; import java.util.Map; import java.util.UUID; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ExecutorService; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiConsumer; @@ -77,8 +76,8 @@ public class BatchTbRuleEngineSubmitStrategy extends AbstractTbRuleEngineSubmitS } } int submitSize = pendingPack.size(); - if (log.isInfoEnabled() && submitSize > 0) { - log.info("[{}] submitting [{}] messages to rule engine", queueName, submitSize); + if (log.isDebugEnabled() && submitSize > 0) { + log.debug("[{}] submitting [{}] messages to rule engine", queueName, submitSize); } pendingPack.forEach(msgConsumer); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/BurstTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/BurstTbRuleEngineSubmitStrategy.java index ffd1dd49d1..3420933d3a 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/BurstTbRuleEngineSubmitStrategy.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/BurstTbRuleEngineSubmitStrategy.java @@ -19,14 +19,8 @@ import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import java.util.ArrayList; -import java.util.List; import java.util.UUID; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ExecutorService; import java.util.function.BiConsumer; -import java.util.function.Function; -import java.util.stream.Collectors; @Slf4j public class BurstTbRuleEngineSubmitStrategy extends AbstractTbRuleEngineSubmitStrategy { @@ -37,8 +31,8 @@ public class BurstTbRuleEngineSubmitStrategy extends AbstractTbRuleEngineSubmitS @Override public void submitAttempt(BiConsumer> msgConsumer) { - if (log.isInfoEnabled()) { - log.info("[{}] submitting [{}] messages to rule engine", queueName, orderedMsgList.size()); + if (log.isDebugEnabled()) { + log.debug("[{}] submitting [{}] messages to rule engine", queueName, orderedMsgList.size()); } orderedMsgList.forEach(pair -> msgConsumer.accept(pair.uuid, pair.msg)); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByEntityIdTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByEntityIdTbRuleEngineSubmitStrategy.java index ae5993cb1c..473810b86c 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByEntityIdTbRuleEngineSubmitStrategy.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByEntityIdTbRuleEngineSubmitStrategy.java @@ -15,26 +15,18 @@ */ package org.thingsboard.server.service.queue.processing; -import com.google.protobuf.InvalidProtocolBufferException; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EntityIdFactory; -import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.gen.MsgProtos; -import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import java.util.ArrayList; import java.util.LinkedList; import java.util.List; import java.util.Queue; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiConsumer; -import java.util.stream.Collectors; @Slf4j public abstract class SequentialByEntityIdTbRuleEngineSubmitStrategy extends AbstractTbRuleEngineSubmitStrategy { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByTenantIdTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByTenantIdTbRuleEngineSubmitStrategy.java index b258c6db1b..37e9419edd 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByTenantIdTbRuleEngineSubmitStrategy.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialByTenantIdTbRuleEngineSubmitStrategy.java @@ -30,6 +30,5 @@ public class SequentialByTenantIdTbRuleEngineSubmitStrategy extends SequentialBy @Override protected EntityId getEntityId(TransportProtos.ToRuleEngineMsg msg) { return new TenantId(new UUID(msg.getTenantIdMSB(), msg.getTenantIdLSB())); - } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialTbRuleEngineSubmitStrategy.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialTbRuleEngineSubmitStrategy.java index ef45b983fc..125a1d8ef8 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialTbRuleEngineSubmitStrategy.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/SequentialTbRuleEngineSubmitStrategy.java @@ -19,8 +19,6 @@ import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import java.util.LinkedHashMap; -import java.util.Map; import java.util.UUID; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicInteger; @@ -63,8 +61,8 @@ public class SequentialTbRuleEngineSubmitStrategy extends AbstractTbRuleEngineSu if (idx < listSize) { IdMsgPair pair = orderedMsgList.get(idx); expectedMsgId = pair.uuid; - if (log.isInfoEnabled()) { - log.info("[{}] submitting [{}] message to rule engine", queueName, pair.msg); + if (log.isDebugEnabled()) { + log.debug("[{}] submitting [{}] message to rule engine", queueName, pair.msg); } msgConsumer.accept(pair.uuid, pair.msg); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java index bbf283e962..80b0523a81 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java @@ -82,10 +82,10 @@ public class TbRuleEngineProcessingStrategyFactory { retryCount++; double failedCount = result.getFailedMap().size() + result.getPendingMap().size(); if (maxRetries > 0 && retryCount > maxRetries) { - log.info("[{}] Skip reprocess of the rule engine pack due to max retries", queueName); + log.debug("[{}] Skip reprocess of the rule engine pack due to max retries", queueName); return new TbRuleEngineProcessingDecision(true, null); } else if (maxAllowedFailurePercentage > 0 && (failedCount / initialTotalCount) > maxAllowedFailurePercentage) { - log.info("[{}] Skip reprocess of the rule engine pack due to max allowed failure percentage", queueName); + log.debug("[{}] Skip reprocess of the rule engine pack due to max allowed failure percentage", queueName); return new TbRuleEngineProcessingDecision(true, null); } else { ConcurrentMap> toReprocess = new ConcurrentHashMap<>(initialTotalCount); @@ -98,7 +98,7 @@ public class TbRuleEngineProcessingStrategyFactory { if (retrySuccessful) { result.getSuccessMap().forEach(toReprocess::put); } - log.info("[{}] Going to reprocess {} messages", queueName, toReprocess.size()); + log.debug("[{}] Going to reprocess {} messages", queueName, toReprocess.size()); if (log.isTraceEnabled()) { toReprocess.forEach((id, msg) -> log.trace("Going to reprocess [{}]: {}", id, TbMsg.fromBytes(msg.getValue().getTbMsg().toByteArray(), TbMsgCallback.EMPTY))); } @@ -126,7 +126,7 @@ public class TbRuleEngineProcessingStrategyFactory { @Override public TbRuleEngineProcessingDecision analyze(TbRuleEngineProcessingResult result) { if (!result.isSuccess()) { - log.info("[{}] Reprocessing skipped for {} failed and {} timeout messages", queueName, result.getFailedMap().size(), result.getPendingMap().size()); + log.debug("[{}] Reprocessing skipped for {} failed and {} timeout messages", queueName, result.getFailedMap().size(), result.getPendingMap().size()); } if (log.isTraceEnabled()) { result.getFailedMap().forEach((id, msg) -> log.trace("Failed messages [{}]: {}", id, TbMsg.fromBytes(msg.getValue().getTbMsg().toByteArray(), TbMsgCallback.EMPTY))); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusConsumerTemplate.java index cca599d59a..4db5ade728 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusConsumerTemplate.java @@ -31,9 +31,9 @@ import org.apache.qpid.proton.amqp.transport.SenderSettleMode; import org.springframework.util.CollectionUtils; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.queue.TbQueueMsgDecoder; +import org.thingsboard.server.queue.common.AbstractTbQueueConsumerTemplate; import org.thingsboard.server.queue.common.DefaultTbQueueMsg; import java.time.Duration; @@ -50,100 +50,70 @@ import java.util.stream.Collectors; import java.util.stream.Stream; @Slf4j -public class TbServiceBusConsumerTemplate implements TbQueueConsumer { +public class TbServiceBusConsumerTemplate extends AbstractTbQueueConsumerTemplate { private final TbQueueAdmin admin; - private final String topic; private final TbQueueMsgDecoder decoder; private final TbServiceBusSettings serviceBusSettings; private final Gson gson = new Gson(); private Set receivers; - private volatile Set partitions; - private volatile boolean subscribed; - private volatile boolean stopped = false; private Map> pendingMessages = new ConcurrentHashMap<>(); private volatile int messagesPerQueue; public TbServiceBusConsumerTemplate(TbQueueAdmin admin, TbServiceBusSettings serviceBusSettings, String topic, TbQueueMsgDecoder decoder) { + super(topic); this.admin = admin; this.decoder = decoder; - this.topic = topic; this.serviceBusSettings = serviceBusSettings; } @Override - public String getTopic() { - return topic; - } - - @Override - public void subscribe() { - partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true)); - subscribed = false; - } - - @Override - public void subscribe(Set partitions) { - this.partitions = partitions; - subscribed = false; - } - - @Override - public void unsubscribe() { - stopped = true; - receivers.forEach(CoreMessageReceiver::closeAsync); - } - - @Override - public List poll(long durationInMillis) { - if (!subscribed && partitions == null) { - try { - Thread.sleep(durationInMillis); - } catch (InterruptedException e) { - log.debug("Failed to await subscription", e); - } - } else { - if (!subscribed) { - createReceivers(); - messagesPerQueue = receivers.size() / partitions.size(); - subscribed = true; - } - - List>> messageFutures = - receivers.stream() - .map(receiver -> receiver - .receiveAsync(messagesPerQueue, Duration.ofMillis(durationInMillis)) - .whenComplete((messages, err) -> { - if (!CollectionUtils.isEmpty(messages)) { - pendingMessages.put(receiver, messages); - } else if (err != null) { - log.error("Failed to receive messages.", err); - } - })) - .collect(Collectors.toList()); - try { - return fromList(messageFutures) - .get() - .stream() - .flatMap(messages -> CollectionUtils.isEmpty(messages) ? Stream.empty() : messages.stream()) - .map(message -> { - try { - return decode(message); - } catch (InvalidProtocolBufferException e) { - log.error("Failed to parse message.", e); - throw new RuntimeException("Failed to parse message.", e); - } - }).collect(Collectors.toList()); - } catch (InterruptedException | ExecutionException e) { - if (stopped) { - log.info("[{}] Service Bus consumer is stopped.", topic); - } else { - log.error("Failed to receive messages", e); - } + protected List doPoll(long durationInMillis) { + List>> messageFutures = + receivers.stream() + .map(receiver -> receiver + .receiveAsync(messagesPerQueue, Duration.ofMillis(durationInMillis)) + .whenComplete((messages, err) -> { + if (!CollectionUtils.isEmpty(messages)) { + pendingMessages.put(receiver, messages); + } else if (err != null) { + log.error("Failed to receive messages.", err); + } + })) + .collect(Collectors.toList()); + try { + return fromList(messageFutures) + .get() + .stream() + .flatMap(messages -> CollectionUtils.isEmpty(messages) ? Stream.empty() : messages.stream()) + .collect(Collectors.toList()); + } catch (InterruptedException | ExecutionException e) { + if (stopped) { + log.info("[{}] Service Bus consumer is stopped.", getTopic()); + } else { + log.error("Failed to receive messages", e); } + return Collections.emptyList(); } - return Collections.emptyList(); + } + + @Override + protected void doSubscribe(List topicNames) { + createReceivers(); + messagesPerQueue = receivers.size() / partitions.size(); + } + + @Override + protected void doCommit() { + pendingMessages.forEach((receiver, msgs) -> + msgs.forEach(msg -> receiver.completeMessageAsync(msg.getDeliveryTag(), TransactionContext.NULL_TXN))); + pendingMessages.clear(); + } + + @Override + protected void doUnsubscribe() { + receivers.forEach(CoreMessageReceiver::closeAsync); } private void createReceivers() { @@ -167,7 +137,7 @@ public class TbServiceBusConsumerTemplate implements TbQue receivers = new HashSet<>(fromList(receiverFutures).get()); } catch (InterruptedException | ExecutionException e) { if (stopped) { - log.info("[{}] Service Bus consumer is stopped.", topic); + log.info("[{}] Service Bus consumer is stopped.", getTopic()); } else { log.error("Failed to create receivers", e); } @@ -196,13 +166,7 @@ public class TbServiceBusConsumerTemplate implements TbQue } @Override - public void commit() { - pendingMessages.forEach((receiver, msgs) -> - msgs.forEach(msg -> receiver.completeMessageAsync(msg.getDeliveryTag(), TransactionContext.NULL_TXN))); - pendingMessages.clear(); - } - - private T decode(MessageWithDeliveryTag data) throws InvalidProtocolBufferException { + protected T decode(MessageWithDeliveryTag data) throws InvalidProtocolBufferException { DefaultTbQueueMsg msg = gson.fromJson(new String(((Data) data.getMessage().getBody()).getValue().getArray()), DefaultTbQueueMsg.class); return decoder.decode(msg); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractParallelTbQueueConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractParallelTbQueueConsumerTemplate.java new file mode 100644 index 0000000000..bb83a79250 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractParallelTbQueueConsumerTemplate.java @@ -0,0 +1,53 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.common; + +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.queue.TbQueueMsg; + +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +@Slf4j +public abstract class AbstractParallelTbQueueConsumerTemplate extends AbstractTbQueueConsumerTemplate { + + protected ListeningExecutorService consumerExecutor; + + public AbstractParallelTbQueueConsumerTemplate(String topic) { + super(topic); + } + + protected void initNewExecutor(int threadPoolSize) { + if (consumerExecutor != null) { + consumerExecutor.shutdown(); + try { + consumerExecutor.awaitTermination(1, TimeUnit.MINUTES); + } catch (InterruptedException e) { + log.trace("Interrupted while waiting for consumer executor to stop"); + } + } + consumerExecutor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(threadPoolSize)); + } + + protected void shutdownExecutor() { + if (consumerExecutor != null) { + consumerExecutor.shutdownNow(); + } + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java index 084d10fca9..c8cc545601 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java @@ -28,11 +28,13 @@ import java.util.List; import java.util.Set; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; +import java.util.stream.Collectors; @Slf4j public abstract class AbstractTbQueueConsumerTemplate implements TbQueueConsumer { private volatile boolean subscribed; + protected volatile boolean stopped = false; protected volatile Set partitions; protected final Lock consumerLock = new ReentrantLock(); @@ -74,10 +76,12 @@ public abstract class AbstractTbQueueConsumerTemplate i log.debug("Failed to await subscription", e); } } else { + long pollStartTs = System.currentTimeMillis(); consumerLock.lock(); try { if (!subscribed) { - doSubscribe(); + List topicNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); + doSubscribe(topicNames); subscribed = true; } @@ -95,6 +99,17 @@ public abstract class AbstractTbQueueConsumerTemplate i } }); return result; + } else { + long pollDuration = System.currentTimeMillis() - pollStartTs; + if (pollDuration < durationInMillis) { + try { + Thread.sleep(durationInMillis - pollDuration); + } catch (InterruptedException e) { + if (!stopped) { + log.error("Failed to wait.", e); + } + } + } } } finally { consumerLock.unlock(); @@ -115,6 +130,7 @@ public abstract class AbstractTbQueueConsumerTemplate i @Override public void unsubscribe() { + stopped = true; consumerLock.lock(); try { doUnsubscribe(); @@ -127,7 +143,7 @@ public abstract class AbstractTbQueueConsumerTemplate i abstract protected T decode(R record) throws IOException; - abstract protected void doSubscribe(); + abstract protected void doSubscribe(List topicNames); abstract protected void doCommit(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java index fea31854df..75635de7a4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java @@ -69,8 +69,7 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue } @Override - protected void doSubscribe() { - List topicNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); + protected void doSubscribe( List topicNames) { topicNames.forEach(admin::createTopicIfNotExists); consumer.subscribe(topicNames); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java index 5dc795739a..7302d19ff7 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.queue.pubsub; +import com.amazonaws.services.sqs.model.Message; import com.google.api.core.ApiFuture; import com.google.api.core.ApiFutures; import com.google.cloud.pubsub.v1.stub.GrpcSubscriberStub; @@ -35,11 +36,14 @@ import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.queue.TbQueueMsgDecoder; +import org.thingsboard.server.queue.common.AbstractParallelTbQueueConsumerTemplate; +import org.thingsboard.server.queue.common.AbstractTbQueueConsumerTemplate; import org.thingsboard.server.queue.common.DefaultTbQueueMsg; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; +import java.util.LinkedHashSet; import java.util.List; import java.util.Objects; import java.util.Set; @@ -47,10 +51,11 @@ import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @Slf4j -public class TbPubSubConsumerTemplate implements TbQueueConsumer { +public class TbPubSubConsumerTemplate extends AbstractParallelTbQueueConsumerTemplate { private final Gson gson = new Gson(); private final TbQueueAdmin admin; @@ -58,23 +63,18 @@ public class TbPubSubConsumerTemplate implements TbQueueCo private final TbQueueMsgDecoder decoder; private final TbPubSubSettings pubSubSettings; - private volatile boolean subscribed; - private volatile Set partitions; private volatile Set subscriptionNames; private final List acknowledgeRequests = new CopyOnWriteArrayList<>(); - private ExecutorService consumerExecutor; private final SubscriberStub subscriber; - private volatile boolean stopped; - private volatile int messagesPerTopic; public TbPubSubConsumerTemplate(TbQueueAdmin admin, TbPubSubSettings pubSubSettings, String topic, TbQueueMsgDecoder decoder) { + super(topic); this.admin = admin; this.pubSubSettings = pubSubSettings; this.topic = topic; this.decoder = decoder; - try { SubscriberStubSettings subscriberStubSettings = SubscriberStubSettings.newBuilder() @@ -84,91 +84,52 @@ public class TbPubSubConsumerTemplate implements TbQueueCo .setMaxInboundMessageSize(pubSubSettings.getMaxMsgSize()) .build()) .build(); - this.subscriber = GrpcSubscriberStub.create(subscriberStubSettings); } catch (IOException e) { log.error("Failed to create subscriber.", e); throw new RuntimeException("Failed to create subscriber.", e); } - stopped = false; } @Override - public String getTopic() { - return topic; - } - - @Override - public void subscribe() { - partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true)); - subscribed = false; - } - - @Override - public void subscribe(Set partitions) { - this.partitions = partitions; - subscribed = false; - } - - @Override - public void unsubscribe() { - stopped = true; - if (consumerExecutor != null) { - consumerExecutor.shutdownNow(); - } - - if (subscriber != null) { - subscriber.close(); - } - } - - @Override - public List poll(long durationInMillis) { - if (!subscribed && partitions == null) { - try { - Thread.sleep(durationInMillis); - } catch (InterruptedException e) { - log.debug("Failed to await subscription", e); + protected List doPoll(long durationInMillis) { + try { + List messages = receiveMessages(); + if (!messages.isEmpty()) { + return messages.stream().map(ReceivedMessage::getMessage).collect(Collectors.toList()); } - } else { - if (!subscribed) { - subscriptionNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toSet()); - subscriptionNames.forEach(admin::createTopicIfNotExists); - consumerExecutor = Executors.newFixedThreadPool(subscriptionNames.size()); - messagesPerTopic = pubSubSettings.getMaxMessages() / subscriptionNames.size(); - subscribed = true; - } - List messages; - try { - messages = receiveMessages(); - if (!messages.isEmpty()) { - List result = new ArrayList<>(); - messages.forEach(msg -> { - try { - result.add(decode(msg.getMessage())); - } catch (InvalidProtocolBufferException e) { - log.error("Failed decode record: [{}]", msg); - } - }); - return result; - } - } catch (ExecutionException | InterruptedException e) { - if (stopped) { - log.info("[{}] Pub/Sub consumer is stopped.", topic); - } else { - log.error("Failed to receive messages", e); - } + } catch (ExecutionException | InterruptedException e) { + if (stopped) { + log.info("[{}] Pub/Sub consumer is stopped.", topic); + } else { + log.error("Failed to receive messages", e); } } return Collections.emptyList(); } @Override - public void commit() { + protected void doSubscribe(List topicNames) { + subscriptionNames = new LinkedHashSet<>(topicNames); + subscriptionNames.forEach(admin::createTopicIfNotExists); + initNewExecutor(subscriptionNames.size() + 1); + messagesPerTopic = pubSubSettings.getMaxMessages() / subscriptionNames.size(); + } + + @Override + protected void doCommit() { acknowledgeRequests.forEach(subscriber.acknowledgeCallable()::futureCall); acknowledgeRequests.clear(); } + @Override + protected void doUnsubscribe() { + if (subscriber != null) { + subscriber.close(); + } + shutdownExecutor(); + } + private List receiveMessages() throws ExecutionException, InterruptedException { List>> result = subscriptionNames.stream().map(subscriptionId -> { String subscriptionName = ProjectSubscriptionName.format(pubSubSettings.getProjectId(), subscriptionId); @@ -211,6 +172,7 @@ public class TbPubSubConsumerTemplate implements TbQueueCo return transform.get(); } + @Override public T decode(PubsubMessage message) throws InvalidProtocolBufferException { DefaultTbQueueMsg msg = gson.fromJson(message.getData().toStringUtf8(), DefaultTbQueueMsg.class); return decoder.decode(msg); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqConsumerTemplate.java index 25d7719163..45dc9d6a05 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqConsumerTemplate.java @@ -23,9 +23,9 @@ import com.rabbitmq.client.GetResponse; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.queue.TbQueueMsgDecoder; +import org.thingsboard.server.queue.common.AbstractTbQueueConsumerTemplate; import org.thingsboard.server.queue.common.DefaultTbQueueMsg; import java.io.IOException; @@ -37,33 +37,26 @@ import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; @Slf4j -public class TbRabbitMqConsumerTemplate implements TbQueueConsumer { +public class TbRabbitMqConsumerTemplate extends AbstractTbQueueConsumerTemplate { private final Gson gson = new Gson(); private final TbQueueAdmin admin; - private final String topic; private final TbQueueMsgDecoder decoder; - private final TbRabbitMqSettings rabbitMqSettings; private final Channel channel; private final Connection connection; - private volatile Set partitions; - private volatile boolean subscribed; private volatile Set queues; - private volatile boolean stopped; public TbRabbitMqConsumerTemplate(TbQueueAdmin admin, TbRabbitMqSettings rabbitMqSettings, String topic, TbQueueMsgDecoder decoder) { + super(topic); this.admin = admin; this.decoder = decoder; - this.topic = topic; - this.rabbitMqSettings = rabbitMqSettings; try { connection = rabbitMqSettings.getConnectionFactory().newConnection(); } catch (IOException | TimeoutException e) { log.error("Failed to create connection.", e); throw new RuntimeException("Failed to create connection.", e); } - try { channel = connection.createChannel(); } catch (IOException e) { @@ -74,25 +67,42 @@ public class TbRabbitMqConsumerTemplate implements TbQueue } @Override - public String getTopic() { - return topic; + protected List doPoll(long durationInMillis) { + List result = queues.stream() + .map(queue -> { + try { + return channel.basicGet(queue, false); + } catch (IOException e) { + log.error("Failed to get messages from queue: [{}]", queue); + throw new RuntimeException("Failed to get messages from queue.", e); + } + }).filter(Objects::nonNull).collect(Collectors.toList()); + if (result.size() > 0) { + return result; + } else { + return Collections.emptyList(); + } } @Override - public void subscribe() { - partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true)); - subscribed = false; + protected void doSubscribe(List topicNames) { + queues = partitions.stream() + .map(TopicPartitionInfo::getFullTopicName) + .collect(Collectors.toSet()); + queues.forEach(admin::createTopicIfNotExists); } @Override - public void subscribe(Set partitions) { - this.partitions = partitions; - subscribed = false; + protected void doCommit() { + try { + channel.basicAck(0, true); + } catch (IOException e) { + log.error("Failed to ack messages.", e); + } } @Override - public void unsubscribe() { - stopped = true; + protected void doUnsubscribe() { if (channel != null) { try { channel.close(); @@ -109,63 +119,6 @@ public class TbRabbitMqConsumerTemplate implements TbQueue } } - @Override - public List poll(long durationInMillis) { - if (!subscribed && partitions == null) { - try { - Thread.sleep(durationInMillis); - } catch (InterruptedException e) { - log.debug("Failed to await subscription", e); - } - } else { - if (!subscribed) { - queues = partitions.stream() - .map(TopicPartitionInfo::getFullTopicName) - .collect(Collectors.toSet()); - - queues.forEach(admin::createTopicIfNotExists); - subscribed = true; - } - - List result = queues.stream() - .map(queue -> { - try { - return channel.basicGet(queue, false); - } catch (IOException e) { - log.error("Failed to get messages from queue: [{}]", queue); - throw new RuntimeException("Failed to get messages from queue.", e); - } - }).filter(Objects::nonNull).map(message -> { - try { - return decode(message); - } catch (InvalidProtocolBufferException e) { - log.error("Failed to decode message: [{}].", message); - throw new RuntimeException("Failed to decode message.", e); - } - }).collect(Collectors.toList()); - if (result.size() > 0) { - return result; - } - } - try { - Thread.sleep(durationInMillis); - } catch (InterruptedException e) { - if (!stopped) { - log.error("Failed to wait.", e); - } - } - return Collections.emptyList(); - } - - @Override - public void commit() { - try { - channel.basicAck(0, true); - } catch (IOException e) { - log.error("Failed to ack messages.", e); - } - } - public T decode(GetResponse message) throws InvalidProtocolBufferException { DefaultTbQueueMsg msg = gson.fromJson(new String(message.getBody()), DefaultTbQueueMsg.class); return decoder.decode(msg); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java index 3e71388844..317dd93902 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java @@ -25,21 +25,17 @@ import com.amazonaws.services.sqs.model.Message; import com.amazonaws.services.sqs.model.ReceiveMessageRequest; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.ListeningExecutorService; -import com.google.common.util.concurrent.MoreExecutors; import com.google.gson.Gson; import com.google.protobuf.InvalidProtocolBufferException; import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.springframework.util.CollectionUtils; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueMsg; import org.thingsboard.server.queue.TbQueueMsgDecoder; +import org.thingsboard.server.queue.common.AbstractParallelTbQueueConsumerTemplate; import org.thingsboard.server.queue.common.DefaultTbQueueMsg; -import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -47,34 +43,28 @@ import java.util.Objects; import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ExecutionException; -import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import java.util.stream.Stream; @Slf4j -public class TbAwsSqsConsumerTemplate implements TbQueueConsumer { +public class TbAwsSqsConsumerTemplate extends AbstractParallelTbQueueConsumerTemplate { private static final int MAX_NUM_MSGS = 10; private final Gson gson = new Gson(); private final TbQueueAdmin admin; private final AmazonSQS sqsClient; - private final String topic; private final TbQueueMsgDecoder decoder; private final TbAwsSqsSettings sqsSettings; private final List pendingMessages = new CopyOnWriteArrayList<>(); private volatile Set queueUrls; - private volatile Set partitions; - private ListeningExecutorService consumerExecutor; - private volatile boolean subscribed; - private volatile boolean stopped = false; public TbAwsSqsConsumerTemplate(TbQueueAdmin admin, TbAwsSqsSettings sqsSettings, String topic, TbQueueMsgDecoder decoder) { + super(topic); this.admin = admin; this.decoder = decoder; - this.topic = topic; this.sqsSettings = sqsSettings; AWSCredentials awsCredentials = new BasicAWSCredentials(sqsSettings.getAccessKeyId(), sqsSettings.getSecretAccessKey()); @@ -87,81 +77,64 @@ public class TbAwsSqsConsumerTemplate implements TbQueueCo } @Override - public String getTopic() { - return topic; + protected void doSubscribe(List topicNames) { + queueUrls = topicNames.stream().map(this::getQueueUrl).collect(Collectors.toSet()); + initNewExecutor(queueUrls.size() * sqsSettings.getThreadsPerTopic() + 1); } @Override - public void subscribe() { - partitions = Collections.singleton(new TopicPartitionInfo(topic, null, null, true)); - subscribed = false; + protected List doPoll(long durationInMillis) { + if (!pendingMessages.isEmpty()) { + log.warn("Present {} non committed messages.", pendingMessages.size()); + return Collections.emptyList(); + } + int duration = (int) TimeUnit.MILLISECONDS.toSeconds(durationInMillis); + List>> futureList = queueUrls + .stream() + .map(url -> poll(url, duration)) + .collect(Collectors.toList()); + ListenableFuture>> futureResult = Futures.allAsList(futureList); + try { + return futureResult.get().stream() + .flatMap(List::stream) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } catch (InterruptedException | ExecutionException e) { + if (stopped) { + log.info("[{}] Aws SQS consumer is stopped.", getTopic()); + } else { + log.error("Failed to pool messages.", e); + } + return Collections.emptyList(); + } } @Override - public void subscribe(Set partitions) { - this.partitions = partitions; - subscribed = false; + public T decode(Message message) throws InvalidProtocolBufferException { + DefaultTbQueueMsg msg = gson.fromJson(message.getBody(), DefaultTbQueueMsg.class); + return decoder.decode(msg); } @Override - public void unsubscribe() { + protected void doCommit() { + pendingMessages.forEach(msg -> + consumerExecutor.submit(() -> { + List entries = msg.getMessages() + .stream() + .map(message -> new DeleteMessageBatchRequestEntry(message.getMessageId(), message.getReceiptHandle())) + .collect(Collectors.toList()); + sqsClient.deleteMessageBatch(msg.getUrl(), entries); + })); + pendingMessages.clear(); + } + + @Override + protected void doUnsubscribe() { stopped = true; - if (sqsClient != null) { sqsClient.shutdown(); } - if (consumerExecutor != null) { - consumerExecutor.shutdownNow(); - } - } - - @Override - public List poll(long durationInMillis) { - if (!subscribed && partitions == null) { - try { - Thread.sleep(durationInMillis); - } catch (InterruptedException e) { - log.debug("Failed to await subscription", e); - } - } else { - if (!subscribed) { - List topicNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); - queueUrls = topicNames.stream().map(this::getQueueUrl).collect(Collectors.toSet()); - consumerExecutor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(queueUrls.size() * sqsSettings.getThreadsPerTopic() + 1)); - subscribed = true; - } - - if (!pendingMessages.isEmpty()) { - log.warn("Present {} non committed messages.", pendingMessages.size()); - return Collections.emptyList(); - } - - List>> futureList = queueUrls - .stream() - .map(url -> poll(url, (int) TimeUnit.MILLISECONDS.toSeconds(durationInMillis))) - .collect(Collectors.toList()); - ListenableFuture>> futureResult = Futures.allAsList(futureList); - try { - return futureResult.get().stream() - .flatMap(List::stream) - .map(msg -> { - try { - return decode(msg); - } catch (IOException e) { - log.error("Failed to decode message: [{}]", msg); - return null; - } - }).filter(Objects::nonNull) - .collect(Collectors.toList()); - } catch (InterruptedException | ExecutionException e) { - if (stopped) { - log.info("[{}] Aws SQS consumer is stopped.", topic); - } else { - log.error("Failed to pool messages.", e); - } - } - } - return Collections.emptyList(); + shutdownExecutor(); } private ListenableFuture> poll(String url, int waitTimeSeconds) { @@ -194,25 +167,6 @@ public class TbAwsSqsConsumerTemplate implements TbQueueCo }, consumerExecutor); } - @Override - public void commit() { - pendingMessages.forEach(msg -> - consumerExecutor.submit(() -> { - List entries = msg.getMessages() - .stream() - .map(message -> new DeleteMessageBatchRequestEntry(message.getMessageId(), message.getReceiptHandle())) - .collect(Collectors.toList()); - sqsClient.deleteMessageBatch(msg.getUrl(), entries); - })); - - pendingMessages.clear(); - } - - public T decode(Message message) throws InvalidProtocolBufferException { - DefaultTbQueueMsg msg = gson.fromJson(message.getBody(), DefaultTbQueueMsg.class); - return decoder.decode(msg); - } - @Data private static class AwsSqsMsgWrapper { private final String url; From 06c3caf082ee48eed660927ff453d530e965bfb6 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Sat, 2 May 2020 13:34:11 +0300 Subject: [PATCH 06/19] refactored --- .../server/queue/pubsub/TbPubSubAdmin.java | 32 ++++++++++++----- .../pubsub/TbPubSubConsumerTemplate.java | 5 +++ .../pubsub/TbPubSubProducerTemplate.java | 4 +-- msa/js-executor/queue/pubSubTemplate.js | 35 +++++++++++-------- pom.xml | 2 +- 5 files changed, 52 insertions(+), 26 deletions(-) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java index 1241370d05..d0a514ffd3 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.queue.pubsub; +import com.google.api.gax.rpc.AlreadyExistsException; import com.google.cloud.pubsub.v1.SubscriptionAdminClient; import com.google.cloud.pubsub.v1.SubscriptionAdminSettings; import com.google.cloud.pubsub.v1.TopicAdminClient; @@ -24,9 +25,9 @@ import com.google.pubsub.v1.ListSubscriptionsRequest; import com.google.pubsub.v1.ListTopicsRequest; import com.google.pubsub.v1.ProjectName; import com.google.pubsub.v1.ProjectSubscriptionName; -import com.google.pubsub.v1.ProjectTopicName; import com.google.pubsub.v1.Subscription; import com.google.pubsub.v1.Topic; +import com.google.pubsub.v1.TopicName; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.queue.TbQueueAdmin; @@ -103,7 +104,10 @@ public class TbPubSubAdmin implements TbQueueAdmin { @Override public void createTopicIfNotExists(String partition) { - ProjectTopicName topicName = ProjectTopicName.of(pubSubSettings.getProjectId(), partition); + TopicName topicName = TopicName.newBuilder() + .setTopic(partition) + .setProject(pubSubSettings.getProjectId()) + .build(); if (topicSet.contains(topicName.toString())) { createSubscriptionIfNotExists(partition, topicName); @@ -121,13 +125,18 @@ public class TbPubSubAdmin implements TbQueueAdmin { } } - topicAdminClient.createTopic(topicName); - topicSet.add(topicName.toString()); - log.info("Created new topic: [{}]", topicName.toString()); + try { + topicAdminClient.createTopic(topicName); + log.info("Created new topic: [{}]", topicName.toString()); + } catch (AlreadyExistsException e) { + log.info("[{}] Topic already exist.", topicName.toString()); + } finally { + topicSet.add(topicName.toString()); + } createSubscriptionIfNotExists(partition, topicName); } - private void createSubscriptionIfNotExists(String partition, ProjectTopicName topicName) { + private void createSubscriptionIfNotExists(String partition, TopicName topicName) { ProjectSubscriptionName subscriptionName = ProjectSubscriptionName.of(pubSubSettings.getProjectId(), partition); @@ -153,9 +162,14 @@ public class TbPubSubAdmin implements TbQueueAdmin { setAckDeadline(subscriptionBuilder); setMessageRetention(subscriptionBuilder); - subscriptionAdminClient.createSubscription(subscriptionBuilder.build()); - subscriptionSet.add(subscriptionName.toString()); - log.info("Created new subscription: [{}]", subscriptionName.toString()); + try { + subscriptionAdminClient.createSubscription(subscriptionBuilder.build()); + log.info("Created new subscription: [{}]", subscriptionName.toString()); + } catch (AlreadyExistsException e) { + log.info("[{}] Subscription already exist.", subscriptionName.toString()); + } finally { + subscriptionSet.add(subscriptionName.toString()); + } } private void setAckDeadline(Subscription.Builder builder) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java index 5dc795739a..495c895a91 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubConsumerTemplate.java @@ -134,6 +134,11 @@ public class TbPubSubConsumerTemplate implements TbQueueCo if (!subscribed) { subscriptionNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toSet()); subscriptionNames.forEach(admin::createTopicIfNotExists); + + if (consumerExecutor != null) { + consumerExecutor.shutdown(); + } + consumerExecutor = Executors.newFixedThreadPool(subscriptionNames.size()); messagesPerTopic = pubSubSettings.getMaxMessages() / subscriptionNames.size(); subscribed = true; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java index 2cd2e1054e..7a073616fd 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubProducerTemplate.java @@ -124,8 +124,8 @@ public class TbPubSubProducerTemplate implements TbQueuePr publisherMap.put(topic, publisher); return publisher; } catch (IOException e) { - log.error("Failed to create topic [{}].", topic, e); - throw new RuntimeException("Failed to create topic.", e); + log.error("Failed to create Publisher for the topic [{}].", topic, e); + throw new RuntimeException("Failed to create Publisher for the topic.", e); } } diff --git a/msa/js-executor/queue/pubSubTemplate.js b/msa/js-executor/queue/pubSubTemplate.js index cc5284022d..8cb264de64 100644 --- a/msa/js-executor/queue/pubSubTemplate.js +++ b/msa/js-executor/queue/pubSubTemplate.js @@ -60,11 +60,9 @@ function PubSubProducer() { const topicList = await pubSubClient.getTopics(); if (topicList) { - if (topicList) { - topicList[0].forEach(topic => { - topics.push(getName(topic.name)); - }); - } + topicList[0].forEach(topic => { + topics.push(getName(topic.name)); + }); } const subscriptionList = await pubSubClient.getSubscriptions(); @@ -100,23 +98,32 @@ function PubSubProducer() { async function createTopic(topic) { if (!topics.includes(topic)) { - await pubSubClient.createTopic(topic); + try { + await pubSubClient.createTopic(topic); + logger.info('Created new Pub/Sub topic: %s', topic); + } catch (e) { + logger.info('Pub/Sub topic already exists'); + } topics.push(topic); - logger.info('Created new Pub/Sub topic: %s', topic); } await createSubscription(topic) } async function createSubscription(topic) { if (!subscriptions.includes(topic)) { - await pubSubClient.createSubscription(topic, topic, { - topic: topic, - subscription: topic, - ackDeadlineSeconds: queueProps['ackDeadlineInSec'], - messageRetentionDuration: {seconds: queueProps['messageRetentionInSec']} - }); + try { + await pubSubClient.createSubscription(topic, topic, { + topic: topic, + subscription: topic, + ackDeadlineSeconds: queueProps['ackDeadlineInSec'], + messageRetentionDuration: {seconds: queueProps['messageRetentionInSec']} + }); + logger.info('Created new Pub/Sub subscription: %s', topic); + } catch (e) { + logger.info('Pub/Sub subscription already exists.'); + } + subscriptions.push(topic); - logger.info('Created new Pub/Sub subscription: %s', topic); } } diff --git a/pom.xml b/pom.xml index 494a43cb56..a1dbc2d8f5 100755 --- a/pom.xml +++ b/pom.xml @@ -95,7 +95,7 @@ 1.25 1.3.10 1.11.747 - 1.84.0 + 1.105.0 3.2.0 1.5.0 1.4.3 From e251bc575036753744daa321bd2ca6017665b0bd Mon Sep 17 00:00:00 2001 From: Andrew Shvayka Date: Sat, 2 May 2020 18:12:43 +0300 Subject: [PATCH 07/19] Improvement to deserialization and remove debug on dashboards --- .../data/json/demo/rule_chains/root_rule_chain.json | 4 ++-- .../json/demo/rule_chains/thermostat_alarms.json | 4 ++-- .../org/thingsboard/server/common/msg/TbMsg.java | 12 +++++++++++- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/application/src/main/data/json/demo/rule_chains/root_rule_chain.json b/application/src/main/data/json/demo/rule_chains/root_rule_chain.json index 32cecf6e4e..9805c6f996 100644 --- a/application/src/main/data/json/demo/rule_chains/root_rule_chain.json +++ b/application/src/main/data/json/demo/rule_chains/root_rule_chain.json @@ -17,7 +17,7 @@ }, "type": "org.thingsboard.rule.engine.filter.TbJsFilterNode", "name": "Is Thermostat?", - "debugMode": true, + "debugMode": false, "configuration": { "jsScript": "return msg.id.entityType === \"DEVICE\" && msg.type === \"thermostat\";" } @@ -113,7 +113,7 @@ }, "type": "org.thingsboard.rule.engine.action.TbCreateRelationNode", "name": "Relate to Asset", - "debugMode": true, + "debugMode": false, "configuration": { "direction": "FROM", "relationType": "ToAlarmPropagationAsset", diff --git a/application/src/main/data/json/demo/rule_chains/thermostat_alarms.json b/application/src/main/data/json/demo/rule_chains/thermostat_alarms.json index 8fd10c88f1..d67052cbc5 100644 --- a/application/src/main/data/json/demo/rule_chains/thermostat_alarms.json +++ b/application/src/main/data/json/demo/rule_chains/thermostat_alarms.json @@ -81,7 +81,7 @@ }, "type": "org.thingsboard.rule.engine.filter.TbJsSwitchNode", "name": "Check Alarms", - "debugMode": true, + "debugMode": false, "configuration": { "jsScript": "var relations = [];\nif(metadata[\"ss_alarmTemperature\"] === \"true\"){\n if(msg.temperature > metadata[\"ss_thresholdTemperature\"]){\n relations.push(\"NewTempAlarm\");\n } else {\n relations.push(\"ClearTempAlarm\");\n }\n}\nif(metadata[\"ss_alarmHumidity\"] === \"true\"){\n if(msg.humidity < metadata[\"ss_thresholdHumidity\"]){\n relations.push(\"NewHumidityAlarm\");\n } else {\n relations.push(\"ClearHumidityAlarm\");\n }\n}\n\nreturn relations;" } @@ -93,7 +93,7 @@ }, "type": "org.thingsboard.rule.engine.metadata.TbGetAttributesNode", "name": "Fetch Configuration", - "debugMode": true, + "debugMode": false, "configuration": { "clientAttributeNames": [], "sharedAttributeNames": [], diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java index 3edf4e5061..9da7407552 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java @@ -27,6 +27,7 @@ import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.msg.gen.MsgProtos; import org.thingsboard.server.common.msg.queue.TbMsgCallback; +import java.io.IOException; import java.io.Serializable; import java.util.UUID; @@ -47,7 +48,7 @@ public final class TbMsg implements Serializable { private final RuleChainId ruleChainId; private final RuleNodeId ruleNodeId; //This field is not serialized because we use queues and there is no need to do it - private final TbMsgCallback callback; + transient private final TbMsgCallback callback; public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data) { return new TbMsg(UUID.randomUUID(), type, originator, metaData.copy(), TbMsgDataType.JSON, data, null, null, TbMsgCallback.EMPTY); @@ -156,4 +157,13 @@ public final class TbMsg implements Serializable { public TbMsg copyWithRuleNodeId(RuleChainId ruleChainId, RuleNodeId ruleNodeId) { return new TbMsg(this.id, this.type, this.originator, this.metaData, this.dataType, this.data, ruleChainId, ruleNodeId, callback); } + + public TbMsgCallback getCallback() { + //May be null in case of deserialization; + if (callback != null) { + return callback; + } else { + return TbMsgCallback.EMPTY; + } + } } From d2919ba30e02df8af283d0397f133281ae50a088 Mon Sep 17 00:00:00 2001 From: Andrew Shvayka Date: Sun, 3 May 2020 02:17:17 +0300 Subject: [PATCH 08/19] Improvement to Clear Alarm Node --- .../org/thingsboard/rule/engine/action/TbClearAlarmNode.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbClearAlarmNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbClearAlarmNode.java index 63fb59bed5..b0413c6dd7 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbClearAlarmNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbClearAlarmNode.java @@ -81,8 +81,8 @@ public class TbClearAlarmNode extends TbAbstractAlarmNode Date: Mon, 4 May 2020 00:13:14 +0300 Subject: [PATCH 09/19] Better logging of Rule Engine errors --- .../RuleChainActorMessageProcessor.java | 29 ++++++++++++++----- .../actors/ruleChain/RuleNodeActor.java | 12 +++++--- .../RuleNodeActorMessageProcessor.java | 24 +++++++-------- .../actors/shared/ComponentMsgProcessor.java | 13 +++++++-- .../common/msg/queue/RuleNodeException.java | 12 ++++++-- 5 files changed, 61 insertions(+), 29 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java index 4e3d69c2b7..61c5c0da28 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java @@ -163,7 +163,7 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor relationTypes, String failureMessage) { try { - checkActive(); + checkActive(msg); EntityId entityId = msg.getOriginator(); TopicPartitionInfo tpi = systemContext.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); List relations = nodeRoutes.get(originatorNodeId).stream() @@ -272,6 +278,8 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor { + private final String ruleChainName; private final RuleChainId ruleChainId; - private RuleNodeActor(ActorSystemContext systemContext, TenantId tenantId, RuleChainId ruleChainId, RuleNodeId ruleNodeId) { + private RuleNodeActor(ActorSystemContext systemContext, TenantId tenantId, RuleChainId ruleChainId, String ruleChainName, RuleNodeId ruleNodeId) { super(systemContext, tenantId, ruleNodeId); + this.ruleChainName = ruleChainName; this.ruleChainId = ruleChainId; - setProcessor(new RuleNodeActorMessageProcessor(tenantId, ruleChainId, ruleNodeId, systemContext, + setProcessor(new RuleNodeActorMessageProcessor(tenantId, this.ruleChainName, ruleNodeId, systemContext, context().parent(), context().self())); } @@ -96,19 +98,21 @@ public class RuleNodeActor extends ComponentActor { - private final ActorRef parent; + private final String ruleChainName; private final ActorRef self; - private final RuleChainService service; private RuleNode ruleNode; private TbNode tbNode; private DefaultTbContext defaultCtx; - RuleNodeActorMessageProcessor(TenantId tenantId, RuleChainId ruleChainId, RuleNodeId ruleNodeId, ActorSystemContext systemContext + RuleNodeActorMessageProcessor(TenantId tenantId, String ruleChainName, RuleNodeId ruleNodeId, ActorSystemContext systemContext , ActorRef parent, ActorRef self) { super(systemContext, tenantId, ruleNodeId); - this.parent = parent; + this.ruleChainName = ruleChainName; this.self = self; - this.service = systemContext.getRuleChainService(); this.ruleNode = systemContext.getRuleChainService().findRuleNodeById(tenantId, entityId); this.defaultCtx = new DefaultTbContext(systemContext, new RuleNodeCtx(tenantId, parent, self, ruleNode)); } @@ -63,8 +59,8 @@ public class RuleNodeActorMessageProcessor extends ComponentMsgProcessor extends AbstractContextAwareMsgProcessor { @@ -74,11 +77,17 @@ public abstract class ComponentMsgProcessor extends Abstract schedulePeriodicMsgWithDelay(context, new StatsPersistTick(), statsPersistFrequency, statsPersistFrequency); } - protected void checkActive() { + protected void checkActive(TbMsg tbMsg) throws RuleNodeException { if (state != ComponentLifecycleState.ACTIVE) { log.debug("Component is not active. Current state [{}] for processor [{}][{}] tenant [{}]", state, entityId.getEntityType(), entityId, tenantId); - throw new IllegalStateException("Rule chain is not active! " + entityId + " - " + tenantId); + RuleNodeException ruleNodeException = getInactiveException(); + if (tbMsg != null) { + tbMsg.getCallback().onFailure(ruleNodeException); + } + throw ruleNodeException; } } + abstract protected RuleNodeException getInactiveException(); + } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleNodeException.java b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleNodeException.java index 3f437dd1d7..288e1b6002 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleNodeException.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/queue/RuleNodeException.java @@ -36,9 +36,15 @@ public class RuleNodeException extends RuleEngineException { public RuleNodeException(String message, String ruleChainName, RuleNode ruleNode) { super(message); this.ruleChainName = ruleChainName; - this.ruleNodeName = ruleNode.getName(); - this.ruleChainId = ruleNode.getRuleChainId(); - this.ruleNodeId = ruleNode.getId(); + if (ruleNode != null) { + this.ruleNodeName = ruleNode.getName(); + this.ruleChainId = ruleNode.getRuleChainId(); + this.ruleNodeId = ruleNode.getId(); + } else { + ruleNodeName = "Unknown"; + ruleChainId = new RuleChainId(RuleChainId.NULL_UUID); + ruleNodeId = new RuleNodeId(RuleNodeId.NULL_UUID); + } } public String toJsonString() { From 7fc46010b790f0453a14a70ffba77383351e8734 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Wed, 29 Apr 2020 16:37:03 +0300 Subject: [PATCH 10/19] Added base impl for OAuth-2 --- application/pom.xml | 12 ++ .../ThingsboardOAuth2Configuration.java | 81 ++++++++++++ .../ThingsboardSecurityConfiguration.java | 22 ++- .../Oauth2AuthenticationSuccessHandler.java | 125 ++++++++++++++++++ ...RestAwareAuthenticationSuccessHandler.java | 2 +- application/src/main/resources/logback.xml | 1 + .../src/main/resources/thingsboard.yml | 38 ++++++ pom.xml | 15 +++ ui/src/app/login/login.tpl.html | 1 + 9 files changed, 295 insertions(+), 2 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/config/ThingsboardOAuth2Configuration.java create mode 100644 application/src/main/java/org/thingsboard/server/service/security/auth/oauth/Oauth2AuthenticationSuccessHandler.java diff --git a/application/pom.xml b/application/pom.xml index 21a319491d..55009709d1 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -132,6 +132,18 @@ org.springframework.boot spring-boot-starter-websocket + + org.springframework.cloud + spring-cloud-starter-oauth2 + + + org.springframework.security + spring-security-oauth2-client + + + org.springframework.security + spring-security-oauth2-jose + io.jsonwebtoken jjwt diff --git a/application/src/main/java/org/thingsboard/server/config/ThingsboardOAuth2Configuration.java b/application/src/main/java/org/thingsboard/server/config/ThingsboardOAuth2Configuration.java new file mode 100644 index 0000000000..45048fddf3 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/config/ThingsboardOAuth2Configuration.java @@ -0,0 +1,81 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.config; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.security.oauth2.client.registration.ClientRegistration; +import org.springframework.security.oauth2.client.registration.ClientRegistrationRepository; +import org.springframework.security.oauth2.client.registration.InMemoryClientRegistrationRepository; +import org.springframework.security.oauth2.core.AuthorizationGrantType; +import org.springframework.security.oauth2.core.ClientAuthenticationMethod; + +import java.util.Collections; + +@ConditionalOnProperty(prefix = "security.oauth2", value = "enabled", havingValue = "true") +@Configuration +public class ThingsboardOAuth2Configuration { + + @Value("${security.oauth2.registrationId}") + private String registrationId; + @Value("${security.oauth2.userNameAttributeName}") + private String userNameAttributeName; + + @Value("${security.oauth2.client.clientId}") + private String clientId; + @Value("${security.oauth2.client.clientName}") + private String clientName; + @Value("${security.oauth2.client.clientSecret}") + private String clientSecret; + @Value("${security.oauth2.client.accessTokenUri}") + private String accessTokenUri; + @Value("${security.oauth2.client.authorizationUri}") + private String authorizationUri; + @Value("${security.oauth2.client.redirectUriTemplate}") + private String redirectUriTemplate; + @Value("${security.oauth2.client.scope}") + private String scope; + @Value("${security.oauth2.client.jwkSetUri}") + private String jwkSetUri; + @Value("${security.oauth2.client.authorizationGrantType}") + private String authorizationGrantType; + @Value("${security.oauth2.client.clientAuthenticationMethod}") + private String clientAuthenticationMethod; + + @Value("${security.oauth2.resource.userInfoUri}") + private String userInfoUri; + + @Bean + public ClientRegistrationRepository clientRegistrationRepository() { + ClientRegistration registration = ClientRegistration.withRegistrationId(registrationId) + .clientId(clientId) + .authorizationUri(authorizationUri) + .clientSecret(clientSecret) + .tokenUri(accessTokenUri) + .redirectUriTemplate(redirectUriTemplate) + .scope(scope.split(",")) + .clientName(clientName) + .authorizationGrantType(new AuthorizationGrantType(authorizationGrantType)) + .userInfoUri(userInfoUri) + .userNameAttributeName(userNameAttributeName) + .jwkSetUri(jwkSetUri) + .clientAuthenticationMethod(new ClientAuthenticationMethod(clientAuthenticationMethod)) + .build(); + return new InMemoryClientRegistrationRepository(Collections.singletonList(registration)); + } +} \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java b/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java index 53876f4040..36490b5166 100644 --- a/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java +++ b/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java @@ -18,6 +18,8 @@ package org.thingsboard.server.config; import com.fasterxml.jackson.databind.ObjectMapper; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Required; +import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.autoconfigure.security.SecurityProperties; import org.springframework.context.annotation.Bean; @@ -73,12 +75,25 @@ public class ThingsboardSecurityConfiguration extends WebSecurityConfigurerAdapt public static final String WS_TOKEN_BASED_AUTH_ENTRY_POINT = "/api/ws/**"; @Autowired private ThingsboardErrorResponseHandler restAccessDeniedHandler; - @Autowired private AuthenticationSuccessHandler successHandler; + + @Autowired(required = false) + @Qualifier("oauth2AuthenticationSuccessHandler") + private AuthenticationSuccessHandler oauth2AuthenticationSuccessHandler; + + @Autowired + @Qualifier("defaultAuthenticationSuccessHandler") + private AuthenticationSuccessHandler successHandler; + @Autowired private AuthenticationFailureHandler failureHandler; @Autowired private RestAuthenticationProvider restAuthenticationProvider; @Autowired private JwtAuthenticationProvider jwtAuthenticationProvider; @Autowired private RefreshTokenAuthenticationProvider refreshTokenAuthenticationProvider; + @Value("${security.oauth2.enabled}") + private boolean oauth2Enabled; + @Value("${security.oauth2.client.loginProcessingUrl}") + private String loginProcessingUrl; + @Autowired @Qualifier("jwtHeaderTokenExtractor") private TokenExtractor jwtHeaderTokenExtractor; @@ -189,6 +204,11 @@ public class ThingsboardSecurityConfiguration extends WebSecurityConfigurerAdapt .addFilterBefore(buildRefreshTokenProcessingFilter(), UsernamePasswordAuthenticationFilter.class) .addFilterBefore(buildWsJwtTokenAuthenticationProcessingFilter(), UsernamePasswordAuthenticationFilter.class) .addFilterAfter(rateLimitProcessingFilter, UsernamePasswordAuthenticationFilter.class); + if (oauth2Enabled) { + http.oauth2Login() + .loginProcessingUrl(loginProcessingUrl) + .successHandler(oauth2AuthenticationSuccessHandler); + } } diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/oauth/Oauth2AuthenticationSuccessHandler.java b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth/Oauth2AuthenticationSuccessHandler.java new file mode 100644 index 0000000000..e99e5896e1 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth/Oauth2AuthenticationSuccessHandler.java @@ -0,0 +1,125 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.security.auth.oauth; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.security.authentication.InsufficientAuthenticationException; +import org.springframework.security.authentication.LockedException; +import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; +import org.springframework.security.core.Authentication; +import org.springframework.security.core.userdetails.UsernameNotFoundException; +import org.springframework.security.web.authentication.SimpleUrlAuthenticationSuccessHandler; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.security.UserCredentials; +import org.thingsboard.server.dao.user.UserService; +import org.thingsboard.server.service.security.auth.jwt.RefreshTokenRepository; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.security.model.UserPrincipal; +import org.thingsboard.server.service.security.model.token.JwtToken; +import org.thingsboard.server.service.security.model.token.JwtTokenFactory; +import org.thingsboard.server.service.security.system.SystemSecurityService; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +@Component(value="oauth2AuthenticationSuccessHandler") +@ConditionalOnProperty(prefix = "security.oauth2", value = "enabled", havingValue = "true") +public class Oauth2AuthenticationSuccessHandler extends SimpleUrlAuthenticationSuccessHandler { + + private final ObjectMapper mapper; + private final JwtTokenFactory tokenFactory; + private final RefreshTokenRepository refreshTokenRepository; + private final SystemSecurityService systemSecurityService; + private final UserService userService; + + @Autowired + public Oauth2AuthenticationSuccessHandler(final ObjectMapper mapper, + final JwtTokenFactory tokenFactory, + final RefreshTokenRepository refreshTokenRepository, + final UserService userService, + final SystemSecurityService systemSecurityService) { + this.mapper = mapper; + this.tokenFactory = tokenFactory; + this.refreshTokenRepository = refreshTokenRepository; + this.userService = userService; + this.systemSecurityService = systemSecurityService; + } + + @Override + public void onAuthenticationSuccess(HttpServletRequest request, HttpServletResponse response, Authentication authentication) throws IOException, ServletException { + Object object = authentication.getPrincipal(); + + System.out.println(object); + + // active user check + + UserPrincipal principal = new UserPrincipal(UserPrincipal.Type.USER_NAME, "tenant@thingsboard.org"); + SecurityUser securityUser = (SecurityUser) authenticateByUsernameAndPassword(principal,"tenant@thingsboard.org", "tenant").getPrincipal(); + + JwtToken accessToken = tokenFactory.createAccessJwtToken(securityUser); + JwtToken refreshToken = refreshTokenRepository.requestRefreshToken(securityUser); + + Map tokenMap = new HashMap(); + tokenMap.put("token", accessToken.getToken()); + tokenMap.put("refreshToken", refreshToken.getToken()); + +// response.setStatus(HttpStatus.OK.value()); +// response.setContentType(MediaType.APPLICATION_JSON_VALUE); +// mapper.writeValue(response.getWriter(), tokenMap); + + request.setAttribute("token", accessToken.getToken()); + response.addHeader("token", accessToken.getToken()); + + getRedirectStrategy().sendRedirect(request, response, "http://localhost:4200/?accessToken=" + accessToken.getToken() + "&refreshToken=" + refreshToken.getToken()); + } + + private Authentication authenticateByUsernameAndPassword(UserPrincipal userPrincipal, String username, String password) { + User user = userService.findUserByEmail(TenantId.SYS_TENANT_ID, username); + if (user == null) { + throw new UsernameNotFoundException("User not found: " + username); + } + + try { + + UserCredentials userCredentials = userService.findUserCredentialsByUserId(TenantId.SYS_TENANT_ID, user.getId()); + if (userCredentials == null) { + throw new UsernameNotFoundException("User credentials not found"); + } + + try { + systemSecurityService.validateUserCredentials(user.getTenantId(), userCredentials, username, password); + } catch (LockedException e) { + throw e; + } + + if (user.getAuthority() == null) + throw new InsufficientAuthenticationException("User has no authority assigned"); + + SecurityUser securityUser = new SecurityUser(user, userCredentials.isEnabled(), userPrincipal); + return new UsernamePasswordAuthenticationToken(securityUser, null, securityUser.getAuthorities()); + } catch (Exception e) { + throw e; + } + } +} \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAwareAuthenticationSuccessHandler.java b/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAwareAuthenticationSuccessHandler.java index aa55818084..d26b02174e 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAwareAuthenticationSuccessHandler.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAwareAuthenticationSuccessHandler.java @@ -36,7 +36,7 @@ import java.io.IOException; import java.util.HashMap; import java.util.Map; -@Component +@Component(value="defaultAuthenticationSuccessHandler") public class RestAwareAuthenticationSuccessHandler implements AuthenticationSuccessHandler { private final ObjectMapper mapper; private final JwtTokenFactory tokenFactory; diff --git a/application/src/main/resources/logback.xml b/application/src/main/resources/logback.xml index e25fb72ccb..9b14ff06f5 100644 --- a/application/src/main/resources/logback.xml +++ b/application/src/main/resources/logback.xml @@ -26,6 +26,7 @@ + diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index bf82c586c3..97e22fa2c6 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -97,6 +97,44 @@ security: allowClaimingByDefault: "${SECURITY_CLAIM_ALLOW_CLAIMING_BY_DEFAULT:true}" # Time allowed to claim the device in milliseconds duration: "${SECURITY_CLAIM_DURATION:60000}" # 1 minute, note this value must equal claimDevices.timeToLiveInMinutes value + basic: + enabled: false + # oauth2: + # enabled: true + # registrationId: A + # userNameAttributeName: email + # client: + # clientName: Thingsboard Dev Test Q + # clientId: 5f5c0998-1d9b-4679-9610-6108fb91af2a + # clientSecret: h_kXVb7Ee1LgDDinix_nkAh_owWX7YCO783NNteF9AIOqlTWu2L03YoFjv5KL8yRVyx4uYAE-r_N3tFbupE8Kw + # accessTokenUri: https://federation-q.auth.schwarz/nidp/oauth/nam/token + # authorizationUri: https://federation-q.auth.schwarz/nidp/oauth/nam/authz + # scope: openid,profile,email,siam + # redirectUriTemplate: http://localhost:8080/login/oauth2/code/ + # loginProcessingUrl: /login/oauth2/code/ + # jwkSetUri: https://federation-q.auth.schwarz/nidp/oauth/nam/keys + # authorizationGrantType: authorization_code # authorization_code, implicit, refresh_token, client_credentials + # clientAuthenticationMethod: post # basic, post + # resource: + # userInfoUri: https://federation-q.auth.schwarz/nidp/oauth/nam/userinfo + oauth2: + enabled: true + registrationId: A + userNameAttributeName: email + client: + clientName: Test app + clientId: dVH9reqyqiXIG7M2wmamb0ySue8zaM4g + clientSecret: EYAfAGxwkwoeYnb2o2cDgaWZB5k97OStpZQPPvcMMD-SVH2BuughTGeBazXtF5I6 + accessTokenUri: https://dev-r9m8ht0k.auth0.com/oauth/token + authorizationUri: https://dev-r9m8ht0k.auth0.com/authorize + scope: openid,profile,email + redirectUriTemplate: http://localhost:8080/login/oauth2/code/ + loginProcessingUrl: /login/oauth2/code/ + jwkSetUri: https://dev-r9m8ht0k.auth0.com/.well-known/jwks.json + authorizationGrantType: authorization_code # authorization_code, implicit, refresh_token, client_credentials + clientAuthenticationMethod: post # basic, post + resource: + userInfoUri: https://dev-r9m8ht0k.auth0.com/userinfo # Dashboard parameters dashboard: diff --git a/pom.xml b/pom.xml index 494a43cb56..f263307538 100755 --- a/pom.xml +++ b/pom.xml @@ -458,6 +458,21 @@ spring-boot-starter-security ${spring-boot.version} + + org.springframework.cloud + spring-cloud-starter-oauth2 + ${spring-boot.version} + + + org.springframework.security + spring-security-oauth2-client + ${spring.version} + + + org.springframework.security + spring-security-oauth2-jose + ${spring.version} + org.springframework.boot spring-boot-starter-web diff --git a/ui/src/app/login/login.tpl.html b/ui/src/app/login/login.tpl.html index 6e9d7d8977..409f5cd4ad 100644 --- a/ui/src/app/login/login.tpl.html +++ b/ui/src/app/login/login.tpl.html @@ -47,6 +47,7 @@ {{ 'login.login' | translate }} + OAUTH2 LOGIN From a563fdab3f1a1e37338190bbf933d55268551fec Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Fri, 1 May 2020 03:28:44 +0300 Subject: [PATCH 11/19] Added basic and custom OAuth2 user mappers --- .../ThingsboardOAuth2Configuration.java | 81 ------------ .../ThingsboardSecurityConfiguration.java | 14 +- .../server/controller/AuthController.java | 15 +++ .../Oauth2AuthenticationSuccessHandler.java | 125 ------------------ .../auth/oauth2/BaseOAuth2ClientMapper.java | 112 ++++++++++++++++ .../auth/oauth2/BasicOAuth2ClientMapper.java | 79 +++++++++++ .../auth/oauth2/CustomOAuth2ClientMapper.java | 47 +++++++ .../auth/oauth2/OAuth2ClientMapper.java | 24 ++++ .../oauth2/OAuth2ClientMapperProvider.java | 45 +++++++ .../Oauth2AuthenticationSuccessHandler.java | 70 ++++++++++ ...RestAwareAuthenticationSuccessHandler.java | 2 +- .../src/main/resources/thingsboard.yml | 77 +++++++---- .../server/dao/oauth2/OAuth2Service.java | 25 ++++ .../server/dao/oauth2/OAuth2User.java | 27 ++++ .../common/data/id/OAuth2IntegrationId.java | 35 +++++ .../common/data/oauth2/OAuth2ClientInfo.java | 45 +++++++ dao/pom.xml | 4 + .../server/dao/oauth2/OAuth2Client.java | 41 ++++++ .../dao/oauth2/OAuth2ClientMapperConfig.java | 44 ++++++ .../dao/oauth2/OAuth2Configuration.java | 80 +++++++++++ .../server/dao/oauth2/OAuth2ServiceImpl.java | 49 +++++++ pom.xml | 3 +- ui/src/app/api/login.service.js | 15 ++- ui/src/app/app.run.js | 35 ++++- ui/src/app/locale/locale.constant-cs_CZ.json | 6 +- ui/src/app/locale/locale.constant-en_US.json | 6 +- ui/src/app/locale/locale.constant-ru_RU.json | 4 +- ui/src/app/locale/locale.constant-uk_UA.json | 6 +- ui/src/app/login/login.scss | 36 +++++ ui/src/app/login/login.tpl.html | 23 +++- 30 files changed, 916 insertions(+), 259 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/config/ThingsboardOAuth2Configuration.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/security/auth/oauth/Oauth2AuthenticationSuccessHandler.java create mode 100644 application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/BaseOAuth2ClientMapper.java create mode 100644 application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/BasicOAuth2ClientMapper.java create mode 100644 application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/CustomOAuth2ClientMapper.java create mode 100644 application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/OAuth2ClientMapper.java create mode 100644 application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/OAuth2ClientMapperProvider.java create mode 100644 application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/Oauth2AuthenticationSuccessHandler.java create mode 100644 common/dao-api/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2Service.java create mode 100644 common/dao-api/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2User.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/id/OAuth2IntegrationId.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/oauth2/OAuth2ClientInfo.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2Client.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2ClientMapperConfig.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2Configuration.java create mode 100644 dao/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2ServiceImpl.java diff --git a/application/src/main/java/org/thingsboard/server/config/ThingsboardOAuth2Configuration.java b/application/src/main/java/org/thingsboard/server/config/ThingsboardOAuth2Configuration.java deleted file mode 100644 index 45048fddf3..0000000000 --- a/application/src/main/java/org/thingsboard/server/config/ThingsboardOAuth2Configuration.java +++ /dev/null @@ -1,81 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.config; - -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.security.oauth2.client.registration.ClientRegistration; -import org.springframework.security.oauth2.client.registration.ClientRegistrationRepository; -import org.springframework.security.oauth2.client.registration.InMemoryClientRegistrationRepository; -import org.springframework.security.oauth2.core.AuthorizationGrantType; -import org.springframework.security.oauth2.core.ClientAuthenticationMethod; - -import java.util.Collections; - -@ConditionalOnProperty(prefix = "security.oauth2", value = "enabled", havingValue = "true") -@Configuration -public class ThingsboardOAuth2Configuration { - - @Value("${security.oauth2.registrationId}") - private String registrationId; - @Value("${security.oauth2.userNameAttributeName}") - private String userNameAttributeName; - - @Value("${security.oauth2.client.clientId}") - private String clientId; - @Value("${security.oauth2.client.clientName}") - private String clientName; - @Value("${security.oauth2.client.clientSecret}") - private String clientSecret; - @Value("${security.oauth2.client.accessTokenUri}") - private String accessTokenUri; - @Value("${security.oauth2.client.authorizationUri}") - private String authorizationUri; - @Value("${security.oauth2.client.redirectUriTemplate}") - private String redirectUriTemplate; - @Value("${security.oauth2.client.scope}") - private String scope; - @Value("${security.oauth2.client.jwkSetUri}") - private String jwkSetUri; - @Value("${security.oauth2.client.authorizationGrantType}") - private String authorizationGrantType; - @Value("${security.oauth2.client.clientAuthenticationMethod}") - private String clientAuthenticationMethod; - - @Value("${security.oauth2.resource.userInfoUri}") - private String userInfoUri; - - @Bean - public ClientRegistrationRepository clientRegistrationRepository() { - ClientRegistration registration = ClientRegistration.withRegistrationId(registrationId) - .clientId(clientId) - .authorizationUri(authorizationUri) - .clientSecret(clientSecret) - .tokenUri(accessTokenUri) - .redirectUriTemplate(redirectUriTemplate) - .scope(scope.split(",")) - .clientName(clientName) - .authorizationGrantType(new AuthorizationGrantType(authorizationGrantType)) - .userInfoUri(userInfoUri) - .userNameAttributeName(userNameAttributeName) - .jwkSetUri(jwkSetUri) - .clientAuthenticationMethod(new ClientAuthenticationMethod(clientAuthenticationMethod)) - .build(); - return new InMemoryClientRegistrationRepository(Collections.singletonList(registration)); - } -} \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java b/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java index 36490b5166..beca0c37bb 100644 --- a/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java +++ b/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java @@ -18,8 +18,6 @@ package org.thingsboard.server.config; import com.fasterxml.jackson.databind.ObjectMapper; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.beans.factory.annotation.Required; -import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.autoconfigure.security.SecurityProperties; import org.springframework.context.annotation.Bean; @@ -41,6 +39,7 @@ import org.springframework.security.web.util.matcher.AntPathRequestMatcher; import org.springframework.web.cors.UrlBasedCorsConfigurationSource; import org.springframework.web.filter.CorsFilter; import org.thingsboard.server.dao.audit.AuditLogLevelFilter; +import org.thingsboard.server.dao.oauth2.OAuth2Configuration; import org.thingsboard.server.exception.ThingsboardErrorResponseHandler; import org.thingsboard.server.service.security.auth.jwt.JwtAuthenticationProvider; import org.thingsboard.server.service.security.auth.jwt.JwtTokenAuthenticationProcessingFilter; @@ -89,10 +88,7 @@ public class ThingsboardSecurityConfiguration extends WebSecurityConfigurerAdapt @Autowired private JwtAuthenticationProvider jwtAuthenticationProvider; @Autowired private RefreshTokenAuthenticationProvider refreshTokenAuthenticationProvider; - @Value("${security.oauth2.enabled}") - private boolean oauth2Enabled; - @Value("${security.oauth2.client.loginProcessingUrl}") - private String loginProcessingUrl; + @Autowired(required = false) OAuth2Configuration oauth2Configuration; @Autowired @Qualifier("jwtHeaderTokenExtractor") @@ -204,10 +200,12 @@ public class ThingsboardSecurityConfiguration extends WebSecurityConfigurerAdapt .addFilterBefore(buildRefreshTokenProcessingFilter(), UsernamePasswordAuthenticationFilter.class) .addFilterBefore(buildWsJwtTokenAuthenticationProcessingFilter(), UsernamePasswordAuthenticationFilter.class) .addFilterAfter(rateLimitProcessingFilter, UsernamePasswordAuthenticationFilter.class); - if (oauth2Enabled) { + if (oauth2Configuration.isEnabled()) { http.oauth2Login() - .loginProcessingUrl(loginProcessingUrl) + .loginPage("/oauth2Login") + .loginProcessingUrl(oauth2Configuration.getClients().values().iterator().next().getLoginProcessingUrl()) .successHandler(oauth2AuthenticationSuccessHandler); +// .and().oauth2Login().loginProcessingUrl(); } } diff --git a/application/src/main/java/org/thingsboard/server/controller/AuthController.java b/application/src/main/java/org/thingsboard/server/controller/AuthController.java index 42da043a91..3097b9c2b1 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AuthController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AuthController.java @@ -38,8 +38,10 @@ import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.oauth2.OAuth2ClientInfo; import org.thingsboard.server.common.data.security.UserCredentials; import org.thingsboard.server.dao.audit.AuditLogService; +import org.thingsboard.server.dao.oauth2.OAuth2Service; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.auth.jwt.RefreshTokenRepository; import org.thingsboard.server.service.security.auth.rest.RestAuthenticationDetails; @@ -55,6 +57,7 @@ import ua_parser.Client; import javax.servlet.http.HttpServletRequest; import java.net.URI; import java.net.URISyntaxException; +import java.util.List; @RestController @TbCoreComponent @@ -80,6 +83,9 @@ public class AuthController extends BaseController { @Autowired private AuditLogService auditLogService; + @Autowired + private OAuth2Service oauth2Service; + @PreAuthorize("isAuthenticated()") @RequestMapping(value = "/auth/user", method = RequestMethod.GET) public @ResponseBody User getUser() throws ThingsboardException { @@ -330,4 +336,13 @@ public class AuthController extends BaseController { } } + @RequestMapping(value = "/noauth/oauth2Clients", method = RequestMethod.POST) + @ResponseBody + public List getOath2Clients() throws ThingsboardException { + try { + return oauth2Service.getOAuth2Clients(); + } catch (Exception e) { + throw handleException(e); + } + } } diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/oauth/Oauth2AuthenticationSuccessHandler.java b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth/Oauth2AuthenticationSuccessHandler.java deleted file mode 100644 index e99e5896e1..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/oauth/Oauth2AuthenticationSuccessHandler.java +++ /dev/null @@ -1,125 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.security.auth.oauth; - -import com.fasterxml.jackson.databind.ObjectMapper; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.security.authentication.InsufficientAuthenticationException; -import org.springframework.security.authentication.LockedException; -import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; -import org.springframework.security.core.Authentication; -import org.springframework.security.core.userdetails.UsernameNotFoundException; -import org.springframework.security.web.authentication.SimpleUrlAuthenticationSuccessHandler; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.User; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.security.UserCredentials; -import org.thingsboard.server.dao.user.UserService; -import org.thingsboard.server.service.security.auth.jwt.RefreshTokenRepository; -import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.security.model.UserPrincipal; -import org.thingsboard.server.service.security.model.token.JwtToken; -import org.thingsboard.server.service.security.model.token.JwtTokenFactory; -import org.thingsboard.server.service.security.system.SystemSecurityService; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - -@Component(value="oauth2AuthenticationSuccessHandler") -@ConditionalOnProperty(prefix = "security.oauth2", value = "enabled", havingValue = "true") -public class Oauth2AuthenticationSuccessHandler extends SimpleUrlAuthenticationSuccessHandler { - - private final ObjectMapper mapper; - private final JwtTokenFactory tokenFactory; - private final RefreshTokenRepository refreshTokenRepository; - private final SystemSecurityService systemSecurityService; - private final UserService userService; - - @Autowired - public Oauth2AuthenticationSuccessHandler(final ObjectMapper mapper, - final JwtTokenFactory tokenFactory, - final RefreshTokenRepository refreshTokenRepository, - final UserService userService, - final SystemSecurityService systemSecurityService) { - this.mapper = mapper; - this.tokenFactory = tokenFactory; - this.refreshTokenRepository = refreshTokenRepository; - this.userService = userService; - this.systemSecurityService = systemSecurityService; - } - - @Override - public void onAuthenticationSuccess(HttpServletRequest request, HttpServletResponse response, Authentication authentication) throws IOException, ServletException { - Object object = authentication.getPrincipal(); - - System.out.println(object); - - // active user check - - UserPrincipal principal = new UserPrincipal(UserPrincipal.Type.USER_NAME, "tenant@thingsboard.org"); - SecurityUser securityUser = (SecurityUser) authenticateByUsernameAndPassword(principal,"tenant@thingsboard.org", "tenant").getPrincipal(); - - JwtToken accessToken = tokenFactory.createAccessJwtToken(securityUser); - JwtToken refreshToken = refreshTokenRepository.requestRefreshToken(securityUser); - - Map tokenMap = new HashMap(); - tokenMap.put("token", accessToken.getToken()); - tokenMap.put("refreshToken", refreshToken.getToken()); - -// response.setStatus(HttpStatus.OK.value()); -// response.setContentType(MediaType.APPLICATION_JSON_VALUE); -// mapper.writeValue(response.getWriter(), tokenMap); - - request.setAttribute("token", accessToken.getToken()); - response.addHeader("token", accessToken.getToken()); - - getRedirectStrategy().sendRedirect(request, response, "http://localhost:4200/?accessToken=" + accessToken.getToken() + "&refreshToken=" + refreshToken.getToken()); - } - - private Authentication authenticateByUsernameAndPassword(UserPrincipal userPrincipal, String username, String password) { - User user = userService.findUserByEmail(TenantId.SYS_TENANT_ID, username); - if (user == null) { - throw new UsernameNotFoundException("User not found: " + username); - } - - try { - - UserCredentials userCredentials = userService.findUserCredentialsByUserId(TenantId.SYS_TENANT_ID, user.getId()); - if (userCredentials == null) { - throw new UsernameNotFoundException("User credentials not found"); - } - - try { - systemSecurityService.validateUserCredentials(user.getTenantId(), userCredentials, username, password); - } catch (LockedException e) { - throw e; - } - - if (user.getAuthority() == null) - throw new InsufficientAuthenticationException("User has no authority assigned"); - - SecurityUser securityUser = new SecurityUser(user, userCredentials.isEnabled(), userPrincipal); - return new UsernamePasswordAuthenticationToken(securityUser, null, securityUser.getAuthorities()); - } catch (Exception e) { - throw e; - } - } -} \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/BaseOAuth2ClientMapper.java b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/BaseOAuth2ClientMapper.java new file mode 100644 index 0000000000..10334a8430 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/BaseOAuth2ClientMapper.java @@ -0,0 +1,112 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.security.auth.oauth2; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; +import org.springframework.security.core.userdetails.UsernameNotFoundException; +import org.springframework.util.StringUtils; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.TextPageLink; +import org.thingsboard.server.common.data.security.Authority; +import org.thingsboard.server.dao.customer.CustomerService; +import org.thingsboard.server.dao.oauth2.OAuth2User; +import org.thingsboard.server.dao.tenant.TenantService; +import org.thingsboard.server.dao.user.UserService; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.security.model.UserPrincipal; + +import java.util.List; +import java.util.Optional; + +@Slf4j +public abstract class BaseOAuth2ClientMapper { + + @Autowired + private UserService userService; + + @Autowired + private TenantService tenantService; + + @Autowired + private CustomerService customerService; + + protected SecurityUser getOrCreateSecurityUserFromOAuth2User(OAuth2User oauth2User, boolean allowUserCreation) { + UserPrincipal principal = new UserPrincipal(UserPrincipal.Type.USER_NAME, oauth2User.getEmail()); + + User user = userService.findUserByEmail(TenantId.SYS_TENANT_ID, oauth2User.getEmail()); + + if (user == null && !allowUserCreation) { + throw new UsernameNotFoundException("User not found: " + oauth2User.getEmail()); + } + + if (user == null) { + user = new User(); + if (StringUtils.isEmpty(oauth2User.getCustomerName())) { + user.setAuthority(Authority.TENANT_ADMIN); + } else { + user.setAuthority(Authority.CUSTOMER_USER); + } + user.setTenantId(getTenantId(oauth2User.getTenantName())); + user.setCustomerId(getCustomerId(user.getTenantId(), oauth2User.getCustomerName())); + user.setEmail(oauth2User.getEmail()); + user.setFirstName(oauth2User.getFirstName()); + user.setLastName(oauth2User.getLastName()); + user = userService.saveUser(user); + } + + try { + SecurityUser securityUser = new SecurityUser(user, true, principal); + return (SecurityUser) new UsernamePasswordAuthenticationToken(securityUser, null, securityUser.getAuthorities()).getPrincipal(); + } catch (Exception e) { + log.error("Can't get or create security user from oauth2 user", e); + throw e; + } + } + + private TenantId getTenantId(String tenantName) { + List tenants = tenantService.findTenants(new TextPageLink(1, tenantName)).getData(); + Tenant tenant; + if (tenants == null || tenants.isEmpty()) { + tenant = new Tenant(); + tenant.setTitle(tenantName); + tenant = tenantService.saveTenant(tenant); + } else { + tenant = tenants.get(0); + } + return tenant.getTenantId(); + } + + private CustomerId getCustomerId(TenantId tenantId, String customerName) { + if (StringUtils.isEmpty(customerName)) { + return null; + } + Optional customerOpt = customerService.findCustomerByTenantIdAndTitle(tenantId, customerName); + if (customerOpt.isPresent()) { + return customerOpt.get().getId(); + } else { + Customer customer = new Customer(); + customer.setTenantId(tenantId); + customer.setTitle(customerName); + return customerService.saveCustomer(customer).getId(); + } + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/BasicOAuth2ClientMapper.java b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/BasicOAuth2ClientMapper.java new file mode 100644 index 0000000000..2e87d57c72 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/BasicOAuth2ClientMapper.java @@ -0,0 +1,79 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.security.auth.oauth2; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.text.StrSubstitutor; +import org.springframework.security.oauth2.client.authentication.OAuth2AuthenticationToken; +import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; +import org.thingsboard.server.dao.oauth2.OAuth2ClientMapperConfig; +import org.thingsboard.server.dao.oauth2.OAuth2User; +import org.thingsboard.server.service.security.model.SecurityUser; + +import java.util.Map; + +@Service(value = "basicOAuth2ClientMapper") +@Slf4j +public class BasicOAuth2ClientMapper extends BaseOAuth2ClientMapper implements OAuth2ClientMapper { + + @Override + public SecurityUser getOrCreateUserByClientPrincipal(OAuth2AuthenticationToken token, OAuth2ClientMapperConfig config) { + OAuth2User oauth2User = new OAuth2User(); + Map attributes = token.getPrincipal().getAttributes(); + String email = getStringAttributeByKey(attributes, config.getBasic().getEmailAttributeKey()); + oauth2User.setEmail(email); + oauth2User.setTenantName(getTenantName(attributes, config)); + if (!StringUtils.isEmpty(config.getBasic().getLastNameAttributeKey())) { + String lastName = getStringAttributeByKey(attributes, config.getBasic().getLastNameAttributeKey()); + oauth2User.setLastName(lastName); + } + if (!StringUtils.isEmpty(config.getBasic().getFirstNameAttributeKey())) { + String firstName = getStringAttributeByKey(attributes, config.getBasic().getFirstNameAttributeKey()); + oauth2User.setFirstName(firstName); + } + if (!StringUtils.isEmpty(config.getBasic().getCustomerNameStrategyPattern())) { + StrSubstitutor sub = new StrSubstitutor(attributes, "${", "}"); + String customerName = sub.replace(config.getBasic().getCustomerNameStrategyPattern()); + oauth2User.setCustomerName(customerName); + } + return getOrCreateSecurityUserFromOAuth2User(oauth2User, config.getBasic().isAllowUserCreation()); + } + + private String getTenantName(Map attributes, OAuth2ClientMapperConfig config) { + switch (config.getBasic().getTenantNameStrategy()) { + case "domain": + String email = getStringAttributeByKey(attributes, config.getBasic().getEmailAttributeKey()); + return email.substring(email .indexOf("@") + 1); + case "custom": + StrSubstitutor sub = new StrSubstitutor(attributes, "${", "}"); + return sub.replace(config.getBasic().getTenantNameStrategyPattern()); + default: + throw new RuntimeException("Tenant Name Strategy with type " + config.getBasic().getTenantNameStrategy() + " is not supported!"); + } + } + + private String getStringAttributeByKey(Map attributes, String key) { + String result = null; + try { + result = (String) attributes.get(key); + + } catch (Exception e) { + log.warn("Can't convert attribute to String by key " + key); + } + return result; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/CustomOAuth2ClientMapper.java b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/CustomOAuth2ClientMapper.java new file mode 100644 index 0000000000..cada6a7958 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/CustomOAuth2ClientMapper.java @@ -0,0 +1,47 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.security.auth.oauth2; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.web.client.RestTemplateBuilder; +import org.springframework.security.oauth2.client.authentication.OAuth2AuthenticationToken; +import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; +import org.springframework.web.client.RestTemplate; +import org.thingsboard.server.dao.oauth2.OAuth2ClientMapperConfig; +import org.thingsboard.server.dao.oauth2.OAuth2User; +import org.thingsboard.server.service.security.model.SecurityUser; + +@Service(value = "customOAuth2ClientMapper") +@Slf4j +public class CustomOAuth2ClientMapper extends BaseOAuth2ClientMapper implements OAuth2ClientMapper { + + private RestTemplateBuilder restTemplateBuilder = new RestTemplateBuilder(); + + @Override + public SecurityUser getOrCreateUserByClientPrincipal(OAuth2AuthenticationToken token, OAuth2ClientMapperConfig config) { + OAuth2User oauth2User = getOAuth2User(token, config.getCustom()); + return getOrCreateSecurityUserFromOAuth2User(oauth2User, config.getBasic().isAllowUserCreation()); + } + + public OAuth2User getOAuth2User(OAuth2AuthenticationToken token, OAuth2ClientMapperConfig.CustomOAuth2ClientMapperConfig custom) { + if (!StringUtils.isEmpty(custom.getUsername()) && !StringUtils.isEmpty(custom.getPassword())) { + restTemplateBuilder = restTemplateBuilder.basicAuthentication(custom.getUsername(), custom.getPassword()); + } + RestTemplate restTemplate = restTemplateBuilder.build(); + return restTemplate.postForEntity(custom.getUrl(), token.getPrincipal(), OAuth2User.class).getBody(); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/OAuth2ClientMapper.java b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/OAuth2ClientMapper.java new file mode 100644 index 0000000000..196bfe7b50 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/OAuth2ClientMapper.java @@ -0,0 +1,24 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.security.auth.oauth2; + +import org.springframework.security.oauth2.client.authentication.OAuth2AuthenticationToken; +import org.thingsboard.server.dao.oauth2.OAuth2ClientMapperConfig; +import org.thingsboard.server.service.security.model.SecurityUser; + +public interface OAuth2ClientMapper { + SecurityUser getOrCreateUserByClientPrincipal(OAuth2AuthenticationToken token, OAuth2ClientMapperConfig config); +} diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/OAuth2ClientMapperProvider.java b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/OAuth2ClientMapperProvider.java new file mode 100644 index 0000000000..e1c5b694bb --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/OAuth2ClientMapperProvider.java @@ -0,0 +1,45 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.security.auth.oauth2; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Component; + +@Component +@Slf4j +public class OAuth2ClientMapperProvider { + + @Autowired + @Qualifier("basicOAuth2ClientMapper") + private OAuth2ClientMapper basicOAuth2ClientMapper; + + @Autowired + @Qualifier("customOAuth2ClientMapper") + private OAuth2ClientMapper customOAuth2ClientMapper; + + public OAuth2ClientMapper getOAuth2ClientMapperByType(String oauth2ClientType) { + switch (oauth2ClientType) { + case "custom": + return customOAuth2ClientMapper; + case "basic": + return basicOAuth2ClientMapper; + default: + throw new RuntimeException("OAuth2ClientMapper with type " + oauth2ClientType + " is not supported!"); + } + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/Oauth2AuthenticationSuccessHandler.java b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/Oauth2AuthenticationSuccessHandler.java new file mode 100644 index 0000000000..be8f7ca7c2 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/Oauth2AuthenticationSuccessHandler.java @@ -0,0 +1,70 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.security.auth.oauth2; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.security.core.Authentication; +import org.springframework.security.oauth2.client.authentication.OAuth2AuthenticationToken; +import org.springframework.security.web.authentication.SimpleUrlAuthenticationSuccessHandler; +import org.springframework.stereotype.Component; +import org.thingsboard.server.dao.oauth2.OAuth2Client; +import org.thingsboard.server.dao.oauth2.OAuth2Configuration; +import org.thingsboard.server.service.security.auth.jwt.RefreshTokenRepository; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.security.model.token.JwtToken; +import org.thingsboard.server.service.security.model.token.JwtTokenFactory; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; + +@Component(value = "oauth2AuthenticationSuccessHandler") +@ConditionalOnProperty(prefix = "security.oauth2", value = "enabled", havingValue = "true") +public class Oauth2AuthenticationSuccessHandler extends SimpleUrlAuthenticationSuccessHandler { + + private final JwtTokenFactory tokenFactory; + private final RefreshTokenRepository refreshTokenRepository; + private final OAuth2ClientMapperProvider oauth2ClientMapperProvider; + private final OAuth2Configuration oauth2Configuration; + + @Autowired + public Oauth2AuthenticationSuccessHandler(final JwtTokenFactory tokenFactory, + final RefreshTokenRepository refreshTokenRepository, + final OAuth2ClientMapperProvider oauth2ClientMapperProvider, + final OAuth2Configuration oauth2Configuration) { + this.tokenFactory = tokenFactory; + this.refreshTokenRepository = refreshTokenRepository; + this.oauth2ClientMapperProvider = oauth2ClientMapperProvider; + this.oauth2Configuration = oauth2Configuration; + } + + @Override + public void onAuthenticationSuccess(HttpServletRequest request, + HttpServletResponse response, + Authentication authentication) throws IOException { + OAuth2AuthenticationToken token = (OAuth2AuthenticationToken) authentication; + + OAuth2Client oauth2Client = oauth2Configuration.getClientByRegistrationId(token.getAuthorizedClientRegistrationId()); + OAuth2ClientMapper mapper = oauth2ClientMapperProvider.getOAuth2ClientMapperByType(oauth2Client.getMapperConfig().getType()); + SecurityUser securityUser = mapper.getOrCreateUserByClientPrincipal(token, oauth2Client.getMapperConfig()); + + JwtToken accessToken = tokenFactory.createAccessJwtToken(securityUser); + JwtToken refreshToken = refreshTokenRepository.requestRefreshToken(securityUser); + + getRedirectStrategy().sendRedirect(request, response, "/?accessToken=" + accessToken.getToken() + "&refreshToken=" + refreshToken.getToken()); + } +} \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAwareAuthenticationSuccessHandler.java b/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAwareAuthenticationSuccessHandler.java index d26b02174e..4983071c5f 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAwareAuthenticationSuccessHandler.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAwareAuthenticationSuccessHandler.java @@ -36,7 +36,7 @@ import java.io.IOException; import java.util.HashMap; import java.util.Map; -@Component(value="defaultAuthenticationSuccessHandler") +@Component(value = "defaultAuthenticationSuccessHandler") public class RestAwareAuthenticationSuccessHandler implements AuthenticationSuccessHandler { private final ObjectMapper mapper; private final JwtTokenFactory tokenFactory; diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 97e22fa2c6..e2808bc8f6 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -99,29 +99,13 @@ security: duration: "${SECURITY_CLAIM_DURATION:60000}" # 1 minute, note this value must equal claimDevices.timeToLiveInMinutes value basic: enabled: false - # oauth2: - # enabled: true - # registrationId: A - # userNameAttributeName: email - # client: - # clientName: Thingsboard Dev Test Q - # clientId: 5f5c0998-1d9b-4679-9610-6108fb91af2a - # clientSecret: h_kXVb7Ee1LgDDinix_nkAh_owWX7YCO783NNteF9AIOqlTWu2L03YoFjv5KL8yRVyx4uYAE-r_N3tFbupE8Kw - # accessTokenUri: https://federation-q.auth.schwarz/nidp/oauth/nam/token - # authorizationUri: https://federation-q.auth.schwarz/nidp/oauth/nam/authz - # scope: openid,profile,email,siam - # redirectUriTemplate: http://localhost:8080/login/oauth2/code/ - # loginProcessingUrl: /login/oauth2/code/ - # jwkSetUri: https://federation-q.auth.schwarz/nidp/oauth/nam/keys - # authorizationGrantType: authorization_code # authorization_code, implicit, refresh_token, client_credentials - # clientAuthenticationMethod: post # basic, post - # resource: - # userInfoUri: https://federation-q.auth.schwarz/nidp/oauth/nam/userinfo - oauth2: - enabled: true - registrationId: A - userNameAttributeName: email - client: + oauth2: + enabled: true + clients: + schwarz: + registrationId: A + loginButtonLabel: Auth0 # + loginButtonIcon: clientName: Test app clientId: dVH9reqyqiXIG7M2wmamb0ySue8zaM4g clientSecret: EYAfAGxwkwoeYnb2o2cDgaWZB5k97OStpZQPPvcMMD-SVH2BuughTGeBazXtF5I6 @@ -133,8 +117,53 @@ security: jwkSetUri: https://dev-r9m8ht0k.auth0.com/.well-known/jwks.json authorizationGrantType: authorization_code # authorization_code, implicit, refresh_token, client_credentials clientAuthenticationMethod: post # basic, post - resource: userInfoUri: https://dev-r9m8ht0k.auth0.com/userinfo + userNameAttributeName: email + mapperConfig: + type: custom # basic or custom + basic: + allowUserCreation: true # required + emailAttributeKey: email # required + firstNameAttributeKey: + lastNameAttributeKey: + tenantNameStrategy: domain # domain or custom + tenantNameStrategyPattern: + customerNameStrategyPattern: + custom: + url: http://localhost:9090/oauth2/mapper + username: admin + password: bababa + auth0: + registrationId: B + loginButtonLabel: Schwarz # + loginButtonIcon: mdi:google + clientName: Thingsboard Dev Test Q + clientId: 5f5c0998-1d9b-4679-9610-6108fb91af2a + clientSecret: h_kXVb7Ee1LgDDinix_nkAh_owWX7YCO783NNteF9AIOqlTWu2L03YoFjv5KL8yRVyx4uYAE-r_N3tFbupE8Kw + accessTokenUri: https://federation-q.auth.schwarz/nidp/oauth/nam/token + authorizationUri: https://federation-q.auth.schwarz/nidp/oauth/nam/authz + scope: openid,profile,email,siam + redirectUriTemplate: http://localhost:8080/login/oauth2/code/ + loginProcessingUrl: /login/oauth2/code/ + jwkSetUri: https://federation-q.auth.schwarz/nidp/oauth/nam/keys + authorizationGrantType: authorization_code # authorization_code, implicit, refresh_token, client_credentials + clientAuthenticationMethod: post # basic, post + userInfoUri: https://federation-q.auth.schwarz/nidp/oauth/nam/userinfo + userNameAttributeName: mail + mapperConfig: + type: basic # simple or custom + basic: + allowUserCreation: true # required + emailAttributeKey: CloudLoginName # required + firstNameAttributeKey: givenName + lastNameAttributeKey: sn + tenantNameStrategy: custom # domain or custom + tenantNameStrategyPattern: LOL ${region} + customerNameStrategyPattern: GGG ${countrycode} + custom: + url: http://localhost:9090/oauth2/mapper + username: test + password: test # Dashboard parameters dashboard: diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2Service.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2Service.java new file mode 100644 index 0000000000..d72b6ef98c --- /dev/null +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2Service.java @@ -0,0 +1,25 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.oauth2; + +import org.thingsboard.server.common.data.oauth2.OAuth2ClientInfo; + +import java.util.List; + +public interface OAuth2Service { + + List getOAuth2Clients(); +} diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2User.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2User.java new file mode 100644 index 0000000000..6337171369 --- /dev/null +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2User.java @@ -0,0 +1,27 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.oauth2; + +import lombok.Data; + +@Data +public class OAuth2User { + private String tenantName; + private String customerName; + private String email; + private String firstName; + private String lastName; +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/id/OAuth2IntegrationId.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/OAuth2IntegrationId.java new file mode 100644 index 0000000000..30fd55d204 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/OAuth2IntegrationId.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.id; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.UUID; + +public class OAuth2IntegrationId extends UUIDBased { + + private static final long serialVersionUID = 1L; + + @JsonCreator + public OAuth2IntegrationId(@JsonProperty("id") UUID id) { + super(id); + } + + public static OAuth2IntegrationId fromString(String oauth2IntegrationId) { + return new OAuth2IntegrationId(UUID.fromString(oauth2IntegrationId)); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/oauth2/OAuth2ClientInfo.java b/common/data/src/main/java/org/thingsboard/server/common/data/oauth2/OAuth2ClientInfo.java new file mode 100644 index 0000000000..0ee5832e63 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/oauth2/OAuth2ClientInfo.java @@ -0,0 +1,45 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.oauth2; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.BaseData; +import org.thingsboard.server.common.data.id.OAuth2IntegrationId; + +@EqualsAndHashCode(callSuper = true) +@Data +public class OAuth2ClientInfo extends BaseData { + + private String name; + private String icon; + private String url; + + public OAuth2ClientInfo() { + super(); + } + + public OAuth2ClientInfo(OAuth2IntegrationId id) { + super(id); + } + + public OAuth2ClientInfo(OAuth2ClientInfo oauth2ClientInfo) { + super(oauth2ClientInfo); + this.name = oauth2ClientInfo.getName(); + this.icon = oauth2ClientInfo.getIcon(); + this.url = oauth2ClientInfo.getUrl(); + } +} diff --git a/dao/pom.xml b/dao/pom.xml index ca81ee996c..ebee947f7a 100644 --- a/dao/pom.xml +++ b/dao/pom.xml @@ -115,6 +115,10 @@ org.springframework spring-web provided + + + org.springframework.security + spring-security-oauth2-client com.datastax.cassandra diff --git a/dao/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2Client.java b/dao/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2Client.java new file mode 100644 index 0000000000..1327e418cc --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2Client.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.oauth2; + +import lombok.Data; + +@Data +public class OAuth2Client { + + private String registrationId; + private String loginButtonLabel; + private String loginButtonIcon; + private String clientName; + private String clientId; + private String clientSecret; + private String accessTokenUri; + private String authorizationUri; + private String scope; + private String redirectUriTemplate; + private String jwkSetUri; + private String loginProcessingUrl; + private String authorizationGrantType; + private String clientAuthenticationMethod; + private String userInfoUri; + private String userNameAttributeName; + private OAuth2ClientMapperConfig mapperConfig; + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2ClientMapperConfig.java b/dao/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2ClientMapperConfig.java new file mode 100644 index 0000000000..2c8b7cbfc7 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2ClientMapperConfig.java @@ -0,0 +1,44 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.oauth2; + +import lombok.Data; + +@Data +public class OAuth2ClientMapperConfig { + + private String type; + private CustomOAuth2ClientMapperConfig custom; + private BasicOAuth2ClientMapperConfig basic; + + @Data + public static class BasicOAuth2ClientMapperConfig { + private boolean allowUserCreation; + private String emailAttributeKey; + private String firstNameAttributeKey; + private String lastNameAttributeKey; + private String tenantNameStrategy; + private String tenantNameStrategyPattern; + private String customerNameStrategyPattern; + } + + @Data + public static class CustomOAuth2ClientMapperConfig { + private String url; + private String username; + private String password; + } +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2Configuration.java b/dao/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2Configuration.java new file mode 100644 index 0000000000..fa51121d75 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2Configuration.java @@ -0,0 +1,80 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.oauth2; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.security.oauth2.client.registration.ClientRegistration; +import org.springframework.security.oauth2.client.registration.ClientRegistrationRepository; +import org.springframework.security.oauth2.client.registration.InMemoryClientRegistrationRepository; +import org.springframework.security.oauth2.core.AuthorizationGrantType; +import org.springframework.security.oauth2.core.ClientAuthenticationMethod; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Configuration +@ConditionalOnProperty(prefix = "security.oauth2", value = "enabled", havingValue = "true", matchIfMissing = true) +@ConfigurationProperties(prefix = "security.oauth2") +@Data +@Slf4j +public class OAuth2Configuration { + + private boolean enabled; + private Map clients = new HashMap<>(); + + @Bean + public ClientRegistrationRepository clientRegistrationRepository() { + List result = new ArrayList<>(); + for (OAuth2Client client : clients.values()) { + ClientRegistration registration = ClientRegistration.withRegistrationId(client.getRegistrationId()) + .clientId(client.getClientId()) + .authorizationUri(client.getAuthorizationUri()) + .clientSecret(client.getClientSecret()) + .tokenUri(client.getAccessTokenUri()) + .redirectUriTemplate(client.getRedirectUriTemplate()) + .scope(client.getScope().split(",")) + .clientName(client.getClientName()) + .authorizationGrantType(new AuthorizationGrantType(client.getAuthorizationGrantType())) + .userInfoUri(client.getUserInfoUri()) + .userNameAttributeName(client.getUserNameAttributeName()) + .jwkSetUri(client.getJwkSetUri()) + .clientAuthenticationMethod(new ClientAuthenticationMethod(client.getClientAuthenticationMethod())) + .build(); + result.add(registration); + } + return new InMemoryClientRegistrationRepository(result); + } + + public OAuth2Client getClientByRegistrationId(String registrationId) { + OAuth2Client result = null; + if (clients != null && !clients.isEmpty()) { + for (OAuth2Client client : clients.values()) { + if (client.getRegistrationId().equals(registrationId)) { + result = client; + break; + } + } + } + return result; + } +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2ServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2ServiceImpl.java new file mode 100644 index 0000000000..14aa390259 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/oauth2/OAuth2ServiceImpl.java @@ -0,0 +1,49 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.oauth2; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.oauth2.OAuth2ClientInfo; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +@Slf4j +@Service +public class OAuth2ServiceImpl implements OAuth2Service { + + @Autowired(required = false) + OAuth2Configuration oauth2Configuration; + + @Override + public List getOAuth2Clients() { + if (!oauth2Configuration.isEnabled()) { + return Collections.emptyList(); + } + List result = new ArrayList<>(); + for (OAuth2Client c : oauth2Configuration.getClients().values()) { + OAuth2ClientInfo client = new OAuth2ClientInfo(); + client.setName(c.getLoginButtonLabel()); + client.setUrl(String.format("/oauth2/authorization/%s", c.getRegistrationId())); + client.setIcon(c.getLoginButtonIcon()); + result.add(client); + } + return result; + } +} diff --git a/pom.xml b/pom.xml index f263307538..0ea3016b78 100755 --- a/pom.xml +++ b/pom.xml @@ -31,6 +31,7 @@ ${basedir} thingsboard 2.2.4.RELEASE + 2.1.2.RELEASE 5.2.2.RELEASE 5.2.2.RELEASE 2.2.4.RELEASE @@ -461,7 +462,7 @@ org.springframework.cloud spring-cloud-starter-oauth2 - ${spring-boot.version} + ${spring-oauth2.version} org.springframework.security diff --git a/ui/src/app/api/login.service.js b/ui/src/app/api/login.service.js index 0707070ed5..292268642f 100644 --- a/ui/src/app/api/login.service.js +++ b/ui/src/app/api/login.service.js @@ -18,7 +18,7 @@ export default angular.module('thingsboard.api.login', []) .name; /*@ngInject*/ -function LoginService($http, $q) { +function LoginService($http, $q, $rootScope) { var service = { activate: activate, @@ -28,6 +28,7 @@ function LoginService($http, $q) { publicLogin: publicLogin, resetPassword: resetPassword, sendResetPasswordLink: sendResetPasswordLink, + loadOAuth2Clients: loadOAuth2Clients } return service; @@ -109,4 +110,16 @@ function LoginService($http, $q) { }); return deferred.promise; } + + function loadOAuth2Clients(){ + var deferred = $q.defer(); + var url = '/api/noauth/oauth2Clients'; + $http.post(url).then(function success(response) { + $rootScope.oauth2Clients = response.data; + deferred.resolve(); + }, function fail() { + deferred.reject(); + }); + return deferred.promise; + } } diff --git a/ui/src/app/app.run.js b/ui/src/app/app.run.js index 5e56a83eaf..3255c0a917 100644 --- a/ui/src/app/app.run.js +++ b/ui/src/app/app.run.js @@ -17,7 +17,7 @@ import Flow from '@flowjs/ng-flow/dist/ng-flow-standalone.min'; import UrlHandler from './url.handler'; /*@ngInject*/ -export default function AppRun($rootScope, $window, $injector, $location, $log, $state, $mdDialog, $filter, loginService, userService, $translate) { +export default function AppRun($rootScope, $window, $injector, $location, $log, $state, $mdDialog, $filter, $q, loginService, userService, $translate) { $window.Flow = Flow; var frame = null; @@ -41,11 +41,13 @@ export default function AppRun($rootScope, $window, $injector, $location, $log, } initWatchers(); - + + var skipStateChange = false; + function initWatchers() { $rootScope.unauthenticatedHandle = $rootScope.$on('unauthenticated', function (event, doLogout) { if (doLogout) { - $state.go('login'); + gotoPublicModule('login'); } else { UrlHandler($injector, $location); } @@ -61,6 +63,11 @@ export default function AppRun($rootScope, $window, $injector, $location, $log, $rootScope.stateChangeStartHandle = $rootScope.$on('$stateChangeStart', function (evt, to, params) { + if (skipStateChange) { + skipStateChange = false; + return; + } + function waitForUserLoaded() { if ($rootScope.userLoadedHandle) { $rootScope.userLoadedHandle(); @@ -128,7 +135,10 @@ export default function AppRun($rootScope, $window, $injector, $location, $log, redirectParams.toName = to.name; redirectParams.params = params; userService.setRedirectParams(redirectParams); - $state.go('login', params); + gotoPublicModule('login', params); + } else { + evt.preventDefault(); + gotoPublicModule(to.name, params); } } } else { @@ -158,6 +168,23 @@ export default function AppRun($rootScope, $window, $injector, $location, $log, userService.gotoDefaultPlace(params); } + function gotoPublicModule(name, params) { + let tasks = []; + if (name === "login") { + tasks.push(loginService.loadOAuth2Clients()); + } + $q.all(tasks).then( + () => { + skipStateChange = true; + $state.go(name, params); + }, + () => { + skipStateChange = true; + $state.go(name, params); + } + ); + } + function showForbiddenDialog() { if (forbiddenDialog === null) { $translate(['access.access-forbidden', diff --git a/ui/src/app/locale/locale.constant-cs_CZ.json b/ui/src/app/locale/locale.constant-cs_CZ.json index 2a297cd9bb..13247f188a 100644 --- a/ui/src/app/locale/locale.constant-cs_CZ.json +++ b/ui/src/app/locale/locale.constant-cs_CZ.json @@ -1136,7 +1136,7 @@ "total": "celkem" }, "login": { - "login": "Přihlásit", + "login": "Přihlásit se", "request-password-reset": "Vyžádat reset hesla", "reset-password": "Reset hesla", "create-password": "Vytvořit heslo", @@ -1150,7 +1150,9 @@ "new-password": "Nové heslo", "new-password-again": "Nové heslo znovu", "password-link-sent-message": "Odkaz pro reset hesla byl úspěšně odeslán!", - "email": "Email" + "email": "Email", + "login-with": "Přihlásit se přes {{name}}", + "or": "nebo" }, "position": { "top": "Nahoře", diff --git a/ui/src/app/locale/locale.constant-en_US.json b/ui/src/app/locale/locale.constant-en_US.json index fa4689fc75..521669fd02 100644 --- a/ui/src/app/locale/locale.constant-en_US.json +++ b/ui/src/app/locale/locale.constant-en_US.json @@ -1317,7 +1317,7 @@ } }, "login": { - "login": "Login", + "login": "Log in", "request-password-reset": "Request Password Reset", "reset-password": "Reset Password", "create-password": "Create Password", @@ -1332,7 +1332,9 @@ "new-password": "New password", "new-password-again": "New password again", "password-link-sent-message": "Password reset link was successfully sent!", - "email": "Email" + "email": "Email", + "login-with": "Login with {{name}}", + "or": "or" }, "position": { "top": "Top", diff --git a/ui/src/app/locale/locale.constant-ru_RU.json b/ui/src/app/locale/locale.constant-ru_RU.json index e93c92124c..473698a091 100644 --- a/ui/src/app/locale/locale.constant-ru_RU.json +++ b/ui/src/app/locale/locale.constant-ru_RU.json @@ -1246,7 +1246,9 @@ "new-password": "Новый пароль", "new-password-again": "Повторите новый пароль", "password-link-sent-message": "Ссылка для сброса пароля была успешно отправлена!", - "email": "Эл. адрес" + "email": "Эл. адрес", + "login-with": "Войти через {{name}}", + "or": "или" }, "position": { "top": "Верх", diff --git a/ui/src/app/locale/locale.constant-uk_UA.json b/ui/src/app/locale/locale.constant-uk_UA.json index c19049bc11..79c2ece00f 100644 --- a/ui/src/app/locale/locale.constant-uk_UA.json +++ b/ui/src/app/locale/locale.constant-uk_UA.json @@ -1646,7 +1646,7 @@ } }, "login": { - "login": "Вхід", + "login": "Увійти", "request-password-reset": "Запит скидання пароля", "reset-password": "Скинути пароль", "create-password": "Створити пароль", @@ -1661,7 +1661,9 @@ "new-password": "Новий пароль", "new-password-again": "Повторіть новий пароль", "password-link-sent-message": "Посилання для скидання пароля було успішно надіслано!", - "email": "Електронна пошта" + "email": "Електронна пошта", + "login-with": "Увійти через {{name}}", + "or": "або" }, "position": { "top": "Угорі", diff --git a/ui/src/app/login/login.scss b/ui/src/app/login/login.scss index a83b9c09ac..16520544fb 100644 --- a/ui/src/app/login/login.scss +++ b/ui/src/app/login/login.scss @@ -22,6 +22,10 @@ md-card.tb-login-card { width: 450px !important; } + .tb-padding { + padding: 8px; + } + md-card-title { img.tb-login-logo { height: 50px; @@ -31,4 +35,36 @@ md-card.tb-login-card { md-card-content { margin-top: -50px; } + + md-input-container .md-errors-spacer { + display: none; + } + + .oauth-container{ + .container-divider { + display: flex; + flex-direction: row; + align-items: center; + justify-content: center; + width: 100%; + margin: 10px 0; + + .line { + flex: 1; + } + + .text { + padding-right: 10px; + padding-left: 10px; + } + } + + .material-icons{ + width: 20px; + min-width: 20px; + height: 20px; + min-height: 20px; + margin: 0 4px; + } + } } diff --git a/ui/src/app/login/login.tpl.html b/ui/src/app/login/login.tpl.html index 409f5cd4ad..e4f7f2559e 100644 --- a/ui/src/app/login/login.tpl.html +++ b/ui/src/app/login/login.tpl.html @@ -24,7 +24,7 @@ md-mode="indeterminate" ng-disabled="!$root.loading" ng-show="$root.loading">