Merge branch 'master' of https://github.com/thingsboard/thingsboard into UITests_AssetsProfiles

application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java

@@ -30,6 +30,7 @@ import org.springframework.security.config.annotation.authentication.builders.Au
 import org.springframework.security.config.annotation.method.configuration.EnableGlobalMethodSecurity;
 import org.springframework.security.config.annotation.web.builders.HttpSecurity;
 import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity;
+import org.springframework.security.config.annotation.web.configuration.WebSecurityCustomizer;
 import org.springframework.security.config.http.SessionCreationPolicy;
 import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder;
 import org.springframework.security.oauth2.client.web.OAuth2AuthorizationRequestResolver;
@@ -181,15 +182,8 @@ public class ThingsboardSecurityConfiguration {
     private OAuth2AuthorizationRequestResolver oAuth2AuthorizationRequestResolver;
 
     @Bean
-    @Order(0)
+    public WebSecurityCustomizer webSecurityCustomizer() {
-    SecurityFilterChain resources(HttpSecurity http) throws Exception {
+        return (web) -> web.ignoring().antMatchers("/*.js","/*.css","/*.ico","/assets/**","/static/**");
-        http
-                .requestMatchers((matchers) -> matchers.antMatchers("/*.js","/*.css","/*.ico","/assets/**","/static/**"))
-                .authorizeHttpRequests((authorize) -> authorize.anyRequest().permitAll())
-                .requestCache().disable()
-                .securityContext().disable()
-                .sessionManagement().disable();
-        return http.build();
     }
 
     @Bean

application/src/main/java/org/thingsboard/server/service/script/RuleNodeTbelScriptEngine.java

@@ -124,7 +124,7 @@ public class RuleNodeTbelScriptEngine extends RuleNodeScriptEngine<TbelInvokeSer
     protected Object[] prepareArgs(TbMsg msg) {
         Object[] args = new Object[3];
         if (msg.getData() != null) {
-            args[0] = JacksonUtil.fromString(msg.getData(), Map.class);
+            args[0] = JacksonUtil.fromString(msg.getData(), Object.class);
         } else {
             args[0] = new HashMap<>();
         }

rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/deduplication/DeduplicationData.java

@@ -0,0 +1,45 @@
+/**
+ * Copyright © 2016-2022 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.rule.engine.deduplication;
+
+import lombok.Data;
+import org.thingsboard.server.common.msg.TbMsg;
+
+import java.util.LinkedList;
+import java.util.List;
+
+@Data
+public class DeduplicationData {
+
+    private final List<TbMsg> msgList;
+    private boolean tickScheduled;
+
+    public DeduplicationData() {
+        msgList = new LinkedList<>();
+    }
+
+    public int size() {
+        return msgList.size();
+    }
+
+    public void add(TbMsg msg) {
+        msgList.add(msg);
+    }
+
+    public boolean isEmpty() {
+        return msgList.isEmpty();
+    }
+}

rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/deduplication/TbMsgDeduplicationNode.java

@@ -18,7 +18,6 @@ package org.thingsboard.rule.engine.deduplication;
 import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import lombok.extern.slf4j.Slf4j;
-import org.springframework.data.util.Pair;
 import org.thingsboard.common.util.JacksonUtil;
 import org.thingsboard.rule.engine.api.RuleNode;
 import org.thingsboard.rule.engine.api.TbContext;
@@ -37,7 +36,6 @@ import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.Iterator;
-import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -61,14 +59,14 @@ import java.util.concurrent.TimeUnit;
 public class TbMsgDeduplicationNode implements TbNode {
 
     private static final String TB_MSG_DEDUPLICATION_TIMEOUT_MSG = "TbMsgDeduplicationNodeMsg";
-    private static final int TB_MSG_DEDUPLICATION_TIMEOUT = 5000;
     public static final int TB_MSG_DEDUPLICATION_RETRY_DELAY = 10;
+    private static final String EMPTY_DATA = "";
+    private static final TbMsgMetaData EMPTY_META_DATA = new TbMsgMetaData();
 
     private TbMsgDeduplicationNodeConfiguration config;
 
-    private final Map<EntityId, List<TbMsg>> deduplicationMap;
+    private final Map<EntityId, DeduplicationData> deduplicationMap;
     private long deduplicationInterval;
-    private long lastScheduledTs;
     private DeduplicationId deduplicationId;
 
     public TbMsgDeduplicationNode() {
@@ -80,17 +78,12 @@ public class TbMsgDeduplicationNode implements TbNode {
         this.config = TbNodeUtils.convert(configuration, TbMsgDeduplicationNodeConfiguration.class);
         this.deduplicationInterval = TimeUnit.SECONDS.toMillis(config.getInterval());
         this.deduplicationId = config.getId();
-        scheduleTickMsg(ctx);
     }
 
     @Override
     public void onMsg(TbContext ctx, TbMsg msg) throws ExecutionException, InterruptedException, TbNodeException {
         if (TB_MSG_DEDUPLICATION_TIMEOUT_MSG.equals(msg.getType())) {
-            try {
+            processDeduplication(ctx, msg.getOriginator());
-                processDeduplication(ctx);
-            } finally {
-                scheduleTickMsg(ctx);
-            }
         } else {
             processOnRegularMsg(ctx, msg);
         }
@@ -103,11 +96,12 @@ public class TbMsgDeduplicationNode implements TbNode {
 
     private void processOnRegularMsg(TbContext ctx, TbMsg msg) {
         EntityId id = getDeduplicationId(ctx, msg);
-        List<TbMsg> deduplicationMsgs = deduplicationMap.computeIfAbsent(id, k -> new LinkedList<>());
+        DeduplicationData deduplicationMsgs = deduplicationMap.computeIfAbsent(id, k -> new DeduplicationData());
         if (deduplicationMsgs.size() < config.getMaxPendingMsgs()) {
             log.trace("[{}][{}] Adding msg: [{}][{}] to the pending msgs map ...", ctx.getSelfId(), id, msg.getId(), msg.getMetaDataTs());
             deduplicationMsgs.add(msg);
             ctx.ack(msg);
+            scheduleTickMsg(ctx, id, deduplicationMsgs);
         } else {
             log.trace("[{}] Max limit of pending messages reached for deduplication id: [{}]", ctx.getSelfId(), id);
             ctx.tellFailure(msg, new RuntimeException("[" + ctx.getSelfId() + "] Max limit of pending messages reached for deduplication id: [" + id + "]"));
@@ -127,22 +121,25 @@ public class TbMsgDeduplicationNode implements TbNode {
         }
     }
 
-    private void processDeduplication(TbContext ctx) {
+    private void processDeduplication(TbContext ctx, EntityId deduplicationId) {
-        if (deduplicationMap.isEmpty()) {
+        DeduplicationData data = deduplicationMap.get(deduplicationId);
+        if (data == null) {
+            return;
+        }
+        data.setTickScheduled(false);
+        if (data.isEmpty()) {
             return;
         }
-        List<TbMsg> deduplicationResults = new ArrayList<>();
         long deduplicationTimeoutMs = System.currentTimeMillis();
-        deduplicationMap.forEach((entityId, tbMsgs) -> {
+        try {
-            if (tbMsgs.isEmpty()) {
+            List<TbMsg> deduplicationResults = new ArrayList<>();
-                return;
+            List<TbMsg> msgList = data.getMsgList();
-            }
+            Optional<TbPair<Long, Long>> packBoundsOpt = findValidPack(msgList, deduplicationTimeoutMs);
-            Optional<TbPair<Long, Long>> packBoundsOpt = findValidPack(tbMsgs, deduplicationTimeoutMs);
             while (packBoundsOpt.isPresent()) {
                 TbPair<Long, Long> packBounds = packBoundsOpt.get();
                 if (DeduplicationStrategy.ALL.equals(config.getStrategy())) {
                     List<TbMsg> pack = new ArrayList<>();
-                    for (Iterator<TbMsg> iterator = tbMsgs.iterator(); iterator.hasNext(); ) {
+                    for (Iterator<TbMsg> iterator = msgList.iterator(); iterator.hasNext(); ) {
                         TbMsg msg = iterator.next();
                         long msgTs = msg.getMetaDataTs();
                         if (msgTs >= packBounds.getFirst() && msgTs < packBounds.getSecond()) {
@@ -153,13 +150,13 @@ public class TbMsgDeduplicationNode implements TbNode {
                     deduplicationResults.add(TbMsg.newMsg(
                             config.getQueueName(),
                             config.getOutMsgType(),
-                            entityId,
+                            deduplicationId,
                             getMetadata(),
                             getMergedData(pack)));
                 } else {
                     TbMsg resultMsg = null;
                     boolean searchMin = DeduplicationStrategy.FIRST.equals(config.getStrategy());
-                    for (Iterator<TbMsg> iterator = tbMsgs.iterator(); iterator.hasNext(); ) {
+                    for (Iterator<TbMsg> iterator = msgList.iterator(); iterator.hasNext(); ) {
                         TbMsg msg = iterator.next();
                         long msgTs = msg.getMetaDataTs();
                         if (msgTs >= packBounds.getFirst() && msgTs < packBounds.getSecond()) {
@@ -173,10 +170,21 @@ public class TbMsgDeduplicationNode implements TbNode {
                     }
                     deduplicationResults.add(resultMsg);
                 }
-                packBoundsOpt = findValidPack(tbMsgs, deduplicationTimeoutMs);
+                packBoundsOpt = findValidPack(msgList, deduplicationTimeoutMs);
             }
-        });
+            deduplicationResults.forEach(outMsg -> enqueueForTellNextWithRetry(ctx, outMsg, 0));
-        deduplicationResults.forEach(outMsg -> enqueueForTellNextWithRetry(ctx, outMsg, 0));
+        } finally {
+            if (!data.isEmpty()) {
+                scheduleTickMsg(ctx, deduplicationId, data);
+            }
+        }
+    }
+
+    private void scheduleTickMsg(TbContext ctx, EntityId deduplicationId, DeduplicationData data) {
+        if (!data.isTickScheduled()) {
+            scheduleTickMsg(ctx, deduplicationId);
+            data.setTickScheduled(true);
+        }
     }
 
     private Optional<TbPair<Long, Long>> findValidPack(List<TbMsg> msgs, long deduplicationTimeoutMs) {
@@ -206,15 +214,8 @@ public class TbMsgDeduplicationNode implements TbNode {
         }
     }
 
-    private void scheduleTickMsg(TbContext ctx) {
+    private void scheduleTickMsg(TbContext ctx, EntityId deduplicationId) {
-        long curTs = System.currentTimeMillis();
+        ctx.tellSelf(ctx.newMsg(null, TB_MSG_DEDUPLICATION_TIMEOUT_MSG, deduplicationId, EMPTY_META_DATA, EMPTY_DATA), deduplicationInterval + 1);
-        if (lastScheduledTs == 0L) {
-            lastScheduledTs = curTs;
-        }
-        lastScheduledTs += TB_MSG_DEDUPLICATION_TIMEOUT;
-        long curDelay = Math.max(0L, (lastScheduledTs - curTs));
-        TbMsg tickMsg = ctx.newMsg(null, TB_MSG_DEDUPLICATION_TIMEOUT_MSG, ctx.getSelfId(), new TbMsgMetaData(), "");
-        ctx.tellSelf(tickMsg, curDelay);
     }
 
     private String getMergedData(List<TbMsg> msgs) {

ui-ngx/src/app/modules/home/components/widget/dynamic-widget.component.ts

@@ -46,6 +46,7 @@ import { TranslateService } from '@ngx-translate/core';
 import { DomSanitizer } from '@angular/platform-browser';
 import { Router } from '@angular/router';
 import { TbInject } from '@shared/decorators/tb-inject';
+import { MillisecondsToTimeStringPipe } from '@shared/pipe/milliseconds-to-time-string.pipe';
 
 @Directive()
 // tslint:disable-next-line:directive-class-suffix
@@ -83,6 +84,7 @@ export class DynamicWidgetComponent extends PageComponent implements IDynamicWid
     this.ctx.resourceService = $injector.get(ResourceService);
     this.ctx.telemetryWsService = $injector.get(TelemetryWebsocketService);
     this.ctx.date = $injector.get(DatePipe);
+    this.ctx.milliSecondsToTimeString = $injector.get(MillisecondsToTimeStringPipe);
     this.ctx.translate = $injector.get(TranslateService);
     this.ctx.http = $injector.get(HttpClient);
     this.ctx.sanitizer = $injector.get(DomSanitizer);

ui-ngx/src/app/modules/home/components/widget/lib/markdown-widget.component.ts

@@ -42,7 +42,7 @@ interface MarkdownWidgetSettings {
   markdownCss: string;
 }
 
-type MarkdownTextFunction = (data: FormattedData[]) => string;
+type MarkdownTextFunction = (data: FormattedData[], ctx: WidgetContext) => string;
 
 @Component({
   selector: 'tb-markdown-widget ',
@@ -72,7 +72,8 @@ export class MarkdownWidgetComponent extends PageComponent implements OnInit {
   ngOnInit(): void {
     this.ctx.$scope.markdownWidget = this;
     this.settings = this.ctx.settings;
-    this.markdownTextFunction = this.settings.useMarkdownTextFunction ? parseFunction(this.settings.markdownTextFunction, ['data']) : null;
+    this.markdownTextFunction = this.settings.useMarkdownTextFunction ?
+      parseFunction(this.settings.markdownTextFunction, ['data', 'ctx']) : null;
     this.markdownClass = 'markdown-widget';
     const cssString = this.settings.markdownCss;
     if (isNotEmptyStr(cssString)) {
@@ -117,7 +118,7 @@ export class MarkdownWidgetComponent extends PageComponent implements OnInit {
     }
     const data = formattedDataFormDatasourceData(initialData);
     let markdownText = this.settings.useMarkdownTextFunction ?
-      safeExecute(this.markdownTextFunction, [data]) : this.settings.markdownTextPattern;
+      safeExecute(this.markdownTextFunction, [data, this.ctx]) : this.settings.markdownTextPattern;
     const allData: FormattedData = flatDataWithoutOverride(data);
     markdownText = createLabelFromPattern(markdownText, allData);
     if (this.markdownText !== markdownText) {

ui-ngx/src/app/modules/home/components/widget/lib/settings/cards/markdown-widget-settings.component.html

@@ -22,7 +22,7 @@
   <tb-js-func [fxShow]="markdownWidgetSettingsForm.get('useMarkdownTextFunction').value"
               formControlName="markdownTextFunction"
               [globalVariables]="functionScopeVariables"
-              [functionArgs]="['data']"
+              [functionArgs]="['data', 'ctx']"
               functionTitle="{{ 'widgets.markdown.markdown-text-function' | translate }}"
               helpId="widget/lib/markdown/markdown_text_fn">
   </tb-js-func>

ui-ngx/src/app/modules/home/models/services.map.ts

@@ -40,6 +40,7 @@ import { AuthService } from '@core/auth/auth.service';
 import { ResourceService } from '@core/http/resource.service';
 import { TwoFactorAuthenticationService } from '@core/http/two-factor-authentication.service';
 import { TelemetryWebsocketService } from '@core/ws/telemetry-websocket.service';
+import { MillisecondsToTimeStringPipe } from '@shared/pipe/milliseconds-to-time-string.pipe';
 
 export const ServicesMap = new Map<string, Type<any>>(
   [
@@ -57,6 +58,7 @@ export const ServicesMap = new Map<string, Type<any>>(
    ['dialogs', DialogService],
    ['customDialog', CustomDialogService],
    ['date', DatePipe],
+   ['milliSecondsToTimeString', MillisecondsToTimeStringPipe],
    ['utils', UtilsService],
    ['translate', TranslateService],
    ['http', HttpClient],

ui-ngx/src/app/modules/home/models/widget-component.models.ts

@@ -88,7 +88,7 @@ import * as RxJSOperators from 'rxjs/operators';
 import { TbPopoverComponent } from '@shared/components/popover.component';
 import { EntityId } from '@shared/models/id/entity-id';
 import { AlarmQuery, AlarmSearchStatus, AlarmStatus} from '@app/shared/models/alarm.models';
-import { TelemetrySubscriber } from '@app/shared/public-api';
+import { MillisecondsToTimeStringPipe, TelemetrySubscriber } from '@app/shared/public-api';
 
 export interface IWidgetAction {
   name: string;
@@ -182,6 +182,7 @@ export class WidgetContext {
   telemetryWsService: TelemetryWebsocketService;
   telemetrySubscribers?: TelemetrySubscriber[];
   date: DatePipe;
+  milliSecondsToTimeString: MillisecondsToTimeStringPipe;
   translate: TranslateService;
   http: HttpClient;
   sanitizer: DomSanitizer;

ui-ngx/src/app/shared/pipe/milliseconds-to-time-string.pipe.ts

@@ -25,33 +25,51 @@ export class MillisecondsToTimeStringPipe implements PipeTransform {
   constructor(private translate: TranslateService) {
   }
 
-  transform(millseconds: number, args?: any): string {
+  transform(millseconds: number, shortFormat = false): string {
     let seconds = Math.floor(millseconds / 1000);
     const days = Math.floor(seconds / 86400);
     let hours = Math.floor((seconds % 86400) / 3600);
     let minutes = Math.floor(((seconds % 86400) % 3600) / 60);
     seconds = seconds % 60;
     let timeString = '';
-    if (days > 0) {
+    if (shortFormat) {
-      timeString += this.translate.instant('timewindow.days', {days});
+      if (days > 0) {
-    }
+        timeString += this.translate.instant('timewindow.short.days', {days});
-    if (hours > 0) {
-      if (timeString.length === 0 && hours === 1) {
-        hours = 0;
       }
-      timeString += this.translate.instant('timewindow.hours', {hours});
+      if (hours > 0) {
-    }
+        timeString += this.translate.instant('timewindow.short.hours', {hours});
-    if (minutes > 0) {
-      if (timeString.length === 0 && minutes === 1) {
-        minutes = 0;
       }
-      timeString += this.translate.instant('timewindow.minutes', {minutes});
+      if (minutes > 0) {
-    }
+        timeString += this.translate.instant('timewindow.short.minutes', {minutes});
-    if (seconds > 0) {
+      }
-      if (timeString.length === 0 && seconds === 1) {
+      if (seconds > 0) {
-        seconds = 0;
+        timeString += this.translate.instant('timewindow.short.seconds', {seconds});
+      }
+      if (!timeString.length) {
+        timeString += this.translate.instant('timewindow.short.seconds', {seconds: 0});
+      }
+    } else {
+      if (days > 0) {
+        timeString += this.translate.instant('timewindow.days', {days});
+      }
+      if (hours > 0) {
+        if (timeString.length === 0 && hours === 1) {
+          hours = 0;
+        }
+        timeString += this.translate.instant('timewindow.hours', {hours});
+      }
+      if (minutes > 0) {
+        if (timeString.length === 0 && minutes === 1) {
+          minutes = 0;
+        }
+        timeString += this.translate.instant('timewindow.minutes', {minutes});
+      }
+      if (seconds > 0) {
+        if (timeString.length === 0 && seconds === 1) {
+          seconds = 0;
+        }
+        timeString += this.translate.instant('timewindow.seconds', {seconds});
       }
-      timeString += this.translate.instant('timewindow.seconds', {seconds});
     }
     return timeString;
   }

ui-ngx/src/assets/help/en_US/widget/lib/markdown/markdown_text_fn.md

@@ -3,7 +3,7 @@
 <div class="divider"></div>
 <br/>
 
-*function (data): string*
+*function (data, ctx): string*
 
 A JavaScript function used to calculate markdown or HTML content.
 
@@ -13,6 +13,9 @@ A JavaScript function used to calculate markdown or HTML content.
   <li><b>data:</b> <code><a href="https://github.com/thingsboard/thingsboard/blob/5bb6403407aa4898084832d6698aa9ea6d484889/ui-ngx/src/app/modules/home/components/widget/lib/maps/map-models.ts#L108" target="_blank">FormattedData[]</a></code> - An array of <a href="https://github.com/thingsboard/thingsboard/blob/5bb6403407aa4898084832d6698aa9ea6d484889/ui-ngx/src/app/modules/home/components/widget/lib/maps/map-models.ts#L108" target="_blank">FormattedData</a> objects resolved from configured datasources.<br/>
      Each object represents basic entity properties (ex. <code>entityId</code>, <code>entityName</code>)<br/>and provides access to other entity attributes/timeseries declared in widget datasource configuration.
   </li>
+  <li><b>ctx:</b> <code><a href="https://github.com/thingsboard/thingsboard/blob/5bb6403407aa4898084832d6698aa9ea6d484889/ui-ngx/src/app/modules/home/models/widget-component.models.ts#L107" target="_blank">WidgetContext</a></code> - A reference to <a href="https://github.com/thingsboard/thingsboard/blob/5bb6403407aa4898084832d6698aa9ea6d484889/ui-ngx/src/app/modules/home/models/widget-component.models.ts#L107" target="_blank">WidgetContext</a> that has all necessary API 
+     and data used by widget instance.
+  </li>
 </ul>
 
 **Returns:**

ui-ngx/src/assets/locale/locale.constant-en_US.json

@@ -3379,6 +3379,12 @@
         "hours": "{ hours, plural, 0 { hour } 1 {1 hour } other {# hours } }",
         "minutes": "{ minutes, plural, 0 { minute } 1 {1 minute } other {# minutes } }",
         "seconds": "{ seconds, plural, 0 { second } 1 {1 second } other {# seconds } }",
+        "short": {
+            "days": "{ days, plural, 1 {1 day } other {# days } }",
+            "hours": "{ hours, plural, 1 {1 hour } other {# hours } }",
+            "minutes": "{{minutes}} min ",
+            "seconds": "{{seconds}} sec "
+        },
         "realtime": "Realtime",
         "history": "History",
         "last-prefix": "last",

ui-ngx/src/styles.scss

@@ -850,6 +850,23 @@ mat-label {
   }
 }
 
+// Tooltipster
+
+.tooltipster-sidetip.tooltipster-tb {
+  .tooltipster-box {
+    background: rgba(3, 8, 40, 0.64);
+    border: none;
+    border-radius: 4px;
+    .tooltipster-content {
+      padding: 4px 8px;
+      font-size: 12px;
+      line-height: 16px;
+      font-weight: 500;
+      color: #ffffff;
+    }
+  }
+}
+
 .tb-default, .tb-dark {
 
   /*********************************