From 30acfd44e0fa1f06989541f34dfee2c012fd6193 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 5 Nov 2021 19:35:06 +0200 Subject: [PATCH 1/5] PaginateUpdater info log added on start updateEntities --- .../server/service/install/update/PaginatedUpdater.java | 1 + 1 file changed, 1 insertion(+) diff --git a/application/src/main/java/org/thingsboard/server/service/install/update/PaginatedUpdater.java b/application/src/main/java/org/thingsboard/server/service/install/update/PaginatedUpdater.java index 768d627a16..c10d1b4bd7 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/update/PaginatedUpdater.java +++ b/application/src/main/java/org/thingsboard/server/service/install/update/PaginatedUpdater.java @@ -28,6 +28,7 @@ public abstract class PaginatedUpdater { private int updated = 0; public void updateEntities(I id) { + log.info("{}: started...", getName()); updated = 0; PageLink pageLink = new PageLink(DEFAULT_LIMIT); boolean hasNext = true; From 3003ee6e08b817f703ed9ded81a25a94d2bceced Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 5 Nov 2021 19:36:21 +0200 Subject: [PATCH 2/5] TimePageLink callSuper = true for ToString and EqualsAndHashCode --- .../org/thingsboard/server/common/data/page/TimePageLink.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/page/TimePageLink.java b/common/data/src/main/java/org/thingsboard/server/common/data/page/TimePageLink.java index 879db38218..be4878b50e 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/page/TimePageLink.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/page/TimePageLink.java @@ -17,8 +17,12 @@ package org.thingsboard.server.common.data.page; import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.ToString; @Data +@ToString(callSuper = true) +@EqualsAndHashCode(callSuper = true) public class TimePageLink extends PageLink { private final Long startTime; From b477328dd3e180c3e0f5569ab68b230340ec7487 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Mon, 8 Nov 2021 08:41:26 +0200 Subject: [PATCH 3/5] updater: verbose updateTenantAlarmsCustomer processed count --- .../install/update/DefaultDataUpdateService.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java index 3172c37af6..b4e7794e2d 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java @@ -63,6 +63,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; import static org.apache.commons.lang3.StringUtils.isBlank; @@ -382,6 +383,8 @@ public class DefaultDataUpdateService implements DataUpdateService { private final PaginatedUpdater tenantsAlarmsCustomerUpdater = new PaginatedUpdater<>() { + final AtomicLong processed = new AtomicLong(); + @Override protected String getName() { return "Tenants alarms customer updater"; @@ -399,12 +402,12 @@ public class DefaultDataUpdateService implements DataUpdateService { @Override protected void updateEntity(Tenant tenant) { - updateTenantAlarmsCustomer(tenant.getId()); + updateTenantAlarmsCustomer(tenant.getId(), getName(), processed); } }; - private void updateTenantAlarmsCustomer(TenantId tenantId) { - AlarmQuery alarmQuery = new AlarmQuery(null, new TimePageLink(100), null, null, false); + private void updateTenantAlarmsCustomer(TenantId tenantId, String name, AtomicLong processed) { + AlarmQuery alarmQuery = new AlarmQuery(null, new TimePageLink(1000), null, null, false); PageData alarms = alarmDao.findAlarms(tenantId, alarmQuery); boolean hasNext = true; while (hasNext) { @@ -413,6 +416,9 @@ public class DefaultDataUpdateService implements DataUpdateService { alarm.setCustomerId(entityService.fetchEntityCustomerId(tenantId, alarm.getOriginator())); alarmDao.save(tenantId, alarm); } + if (processed.incrementAndGet() % 1000 == 0) { + log.info("{}: {} processed so far...", name, processed); + } } if (alarms.hasNext()) { alarmQuery.setPageLink(alarmQuery.getPageLink().nextPageLink()); From 27d745f1e3a95bc2537ea6cb1b89e508bf4ede06 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Mon, 8 Nov 2021 10:47:27 +0200 Subject: [PATCH 4/5] tests added for update service for convertDeviceProfileAlarmRulesForVersion330 --- .../update/DefaultDataUpdateService.java | 57 +++--- .../update/DefaultDataUpdateServiceTest.java | 97 +++++++++ .../src/test/resources/update/330/README.md | 3 + .../update/330/device_profile_001_in.json | 173 ++++++++++++++++ .../update/330/device_profile_001_out.json | 189 ++++++++++++++++++ 5 files changed, 492 insertions(+), 27 deletions(-) create mode 100644 application/src/test/java/org/thingsboard/server/service/install/update/DefaultDataUpdateServiceTest.java create mode 100644 application/src/test/resources/update/330/README.md create mode 100644 application/src/test/resources/update/330/device_profile_001_in.json create mode 100644 application/src/test/resources/update/330/device_profile_001_out.json diff --git a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java index b4e7794e2d..ebdf60e7b7 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java @@ -145,36 +145,39 @@ public class DefaultDataUpdateService implements DataUpdateService { @Override protected void updateEntity(DeviceProfileEntity deviceProfile) { - if (deviceProfile.getProfileData().has("alarms") && - !deviceProfile.getProfileData().get("alarms").isNull()) { - boolean isUpdated = false; - JsonNode alarms = deviceProfile.getProfileData().get("alarms"); - for (JsonNode alarm : alarms) { - if (alarm.has("createRules")) { - JsonNode createRules = alarm.get("createRules"); - for (AlarmSeverity severity : AlarmSeverity.values()) { - if (createRules.has(severity.name())) { - JsonNode spec = createRules.get(severity.name()).get("condition").get("spec"); - if (convertDeviceProfileAlarmRulesForVersion330(spec)) { - isUpdated = true; - } - } - } - } - if (alarm.has("clearRule") && !alarm.get("clearRule").isNull()) { - JsonNode spec = alarm.get("clearRule").get("condition").get("spec"); - if (convertDeviceProfileAlarmRulesForVersion330(spec)) { - isUpdated = true; - } - } - } - if (isUpdated) { - deviceProfileRepository.save(deviceProfile); - } + if (convertDeviceProfileForVersion330(deviceProfile.getProfileData())) { + deviceProfileRepository.save(deviceProfile); } } }; + boolean convertDeviceProfileForVersion330(JsonNode profileData) { + boolean isUpdated = false; + if (profileData.has("alarms") && !profileData.get("alarms").isNull()) { + JsonNode alarms = profileData.get("alarms"); + for (JsonNode alarm : alarms) { + if (alarm.has("createRules")) { + JsonNode createRules = alarm.get("createRules"); + for (AlarmSeverity severity : AlarmSeverity.values()) { + if (createRules.has(severity.name())) { + JsonNode spec = createRules.get(severity.name()).get("condition").get("spec"); + if (convertDeviceProfileAlarmRulesForVersion330(spec)) { + isUpdated = true; + } + } + } + } + if (alarm.has("clearRule") && !alarm.get("clearRule").isNull()) { + JsonNode spec = alarm.get("clearRule").get("condition").get("spec"); + if (convertDeviceProfileAlarmRulesForVersion330(spec)) { + isUpdated = true; + } + } + } + } + return isUpdated; + } + private final PaginatedUpdater tenantsDefaultRuleChainUpdater = new PaginatedUpdater<>() { @@ -429,7 +432,7 @@ public class DefaultDataUpdateService implements DataUpdateService { } } - private boolean convertDeviceProfileAlarmRulesForVersion330(JsonNode spec) { + boolean convertDeviceProfileAlarmRulesForVersion330(JsonNode spec) { if (spec != null) { if (spec.has("type") && spec.get("type").asText().equals("DURATION")) { if (spec.has("value")) { diff --git a/application/src/test/java/org/thingsboard/server/service/install/update/DefaultDataUpdateServiceTest.java b/application/src/test/java/org/thingsboard/server/service/install/update/DefaultDataUpdateServiceTest.java new file mode 100644 index 0000000000..2a37fa7c2a --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/install/update/DefaultDataUpdateServiceTest.java @@ -0,0 +1,97 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.install.update; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.context.ActiveProfiles; + +import java.io.IOException; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.BDDMockito.willCallRealMethod; + +@ActiveProfiles("install") +@SpringBootTest(classes = DefaultDataUpdateService.class) +class DefaultDataUpdateServiceTest { + + ObjectMapper mapper = new ObjectMapper(); + + @MockBean + DefaultDataUpdateService service; + + @BeforeEach + void setUp() { + willCallRealMethod().given(service).convertDeviceProfileAlarmRulesForVersion330(any()); + willCallRealMethod().given(service).convertDeviceProfileForVersion330(any()); + } + + JsonNode readFromResource(String resourceName) throws IOException { + return mapper.readTree(this.getClass().getClassLoader().getResourceAsStream(resourceName)); + } + + @Test + void convertDeviceProfileAlarmRulesForVersion330FirstRun() throws IOException { + JsonNode spec = readFromResource("update/330/device_profile_001_in.json"); + JsonNode expected = readFromResource("update/330/device_profile_001_out.json"); + + assertThat(service.convertDeviceProfileForVersion330(spec.get("profileData"))).isTrue(); + assertThat(spec.toPrettyString()).isEqualTo(expected.toPrettyString()); // use IDE feature + } + + @Test + void convertDeviceProfileAlarmRulesForVersion330SecondRun() throws IOException { + JsonNode spec = readFromResource("update/330/device_profile_001_out.json"); + JsonNode expected = readFromResource("update/330/device_profile_001_out.json"); + + assertThat(service.convertDeviceProfileForVersion330(spec.get("profileData"))).isFalse(); + assertThat(spec.toPrettyString()).isEqualTo(expected.toPrettyString()); // use IDE feature + } + + @Test + void convertDeviceProfileAlarmRulesForVersion330EmptyJson() throws JsonProcessingException { + JsonNode spec = mapper.readTree("{ }"); + JsonNode expected = mapper.readTree("{ }"); + + assertThat(service.convertDeviceProfileForVersion330(spec)).isFalse(); + assertThat(spec.toPrettyString()).isEqualTo(expected.toPrettyString()); + } + + @Test + void convertDeviceProfileAlarmRulesForVersion330AlarmNodeNull() throws JsonProcessingException { + JsonNode spec = mapper.readTree("{ \"alarms\" : null }"); + JsonNode expected = mapper.readTree("{ \"alarms\" : null }"); + + assertThat(service.convertDeviceProfileForVersion330(spec)).isFalse(); + assertThat(spec.toPrettyString()).isEqualTo(expected.toPrettyString()); + } + + @Test + void convertDeviceProfileAlarmRulesForVersion330NoAlarmNode() throws JsonProcessingException { + JsonNode spec = mapper.readTree("{ \"configuration\": { \"type\": \"DEFAULT\" } }"); + JsonNode expected = mapper.readTree("{ \"configuration\": { \"type\": \"DEFAULT\" } }"); + + assertThat(service.convertDeviceProfileForVersion330(spec)).isFalse(); + assertThat(spec.toPrettyString()).isEqualTo(expected.toPrettyString()); + } + +} diff --git a/application/src/test/resources/update/330/README.md b/application/src/test/resources/update/330/README.md new file mode 100644 index 0000000000..26c5e74898 --- /dev/null +++ b/application/src/test/resources/update/330/README.md @@ -0,0 +1,3 @@ +To get json from live Thingsboard instance use those methods: +1. Browser: F12 -> Network -> open device profile page -> copy raw response +2. Database: SELECT * FROM public.device_profile WHERE name = 'LORAWAN 001'; \ No newline at end of file diff --git a/application/src/test/resources/update/330/device_profile_001_in.json b/application/src/test/resources/update/330/device_profile_001_in.json new file mode 100644 index 0000000000..16d1f09e43 --- /dev/null +++ b/application/src/test/resources/update/330/device_profile_001_in.json @@ -0,0 +1,173 @@ +{ + "id": { + "entityType": "DEVICE_PROFILE", + "id": "b99fde7a-33dd-4d5d-a325-d0637f6acbe5" + }, + "createdTime": 1627268171906, + "tenantId": { + "entityType": "TENANT", + "id": "3db30ac6-db03-4788-98fe-6e024b422a15" + }, + "name": "LORAWAN 001", + "description": "Tektelic - 001", + "type": "DEFAULT", + "transportType": "DEFAULT", + "provisionType": "DISABLED", + "defaultRuleChainId": { + "entityType": "RULE_CHAIN", + "id": "9c50f4df-f41e-443f-bb7d-37b5ac97f3c3" + }, + "defaultQueueName": "LORAWAN", + "profileData": { + "configuration": { + "type": "DEFAULT" + }, + "transportConfiguration": { + "type": "DEFAULT" + }, + "provisionConfiguration": { + "type": "DISABLED", + "provisionDeviceSecret": null + }, + "alarms": [ + { + "id": "b86271fd-5fee-4bd5-975c-d9c18f610cd5", + "alarmType": "LORAWAN - Battery Alarm", + "createRules": { + "CRITICAL": { + "condition": { + "condition": [ + { + "key": { + "type": "TIME_SERIES", + "key": "batteryLevel" + }, + "valueType": "NUMERIC", + "value": null, + "predicate": { + "type": "NUMERIC", + "operation": "LESS", + "value": { + "defaultValue": 25.0, + "userValue": null, + "dynamicValue": null + } + } + } + ], + "spec": { + "type": "DURATION", + "unit": "DAYS", + "value": 1 + } + }, + "schedule": null, + "alarmDetails": null + } + }, + "clearRule": { + "condition": { + "condition": [ + { + "key": { + "type": "TIME_SERIES", + "key": "batteryLevel" + }, + "valueType": "NUMERIC", + "value": null, + "predicate": { + "type": "NUMERIC", + "operation": "GREATER_OR_EQUAL", + "value": { + "defaultValue": 25.0, + "userValue": null, + "dynamicValue": null + } + } + } + ], + "spec": { + "type": "DURATION", + "unit": "DAYS", + "value": 1 + } + }, + "schedule": null, + "alarmDetails": null + }, + "propagate": true, + "propagateRelationTypes": [ + "UC-0007 LORAWAN" + ] + }, + { + "id": "c70aef4e-65cf-4578-acd9-e1927c08b469", + "alarmType": "LORAWAN - No Data", + "createRules": { + "CRITICAL": { + "condition": { + "condition": [ + { + "key": { + "type": "TIME_SERIES", + "key": "active" + }, + "valueType": "BOOLEAN", + "value": null, + "predicate": { + "type": "BOOLEAN", + "operation": "EQUAL", + "value": { + "defaultValue": false, + "userValue": null, + "dynamicValue": null + } + } + } + ], + "spec": { + "type": "SIMPLE" + } + }, + "schedule": null, + "alarmDetails": null + } + }, + "clearRule": { + "condition": { + "condition": [ + { + "key": { + "type": "TIME_SERIES", + "key": "active" + }, + "valueType": "BOOLEAN", + "value": null, + "predicate": { + "type": "BOOLEAN", + "operation": "EQUAL", + "value": { + "defaultValue": true, + "userValue": null, + "dynamicValue": null + } + } + } + ], + "spec": { + "type": "SIMPLE" + } + }, + "schedule": null, + "alarmDetails": null + }, + "propagate": true, + "propagateRelationTypes": [ + "LORAWAN 001 related" + ] + } + ] + }, + "provisionDeviceKey": null, + "default": false +} diff --git a/application/src/test/resources/update/330/device_profile_001_out.json b/application/src/test/resources/update/330/device_profile_001_out.json new file mode 100644 index 0000000000..9a349c6638 --- /dev/null +++ b/application/src/test/resources/update/330/device_profile_001_out.json @@ -0,0 +1,189 @@ +{ + "id": { + "entityType": "DEVICE_PROFILE", + "id": "b99fde7a-33dd-4d5d-a325-d0637f6acbe5" + }, + "createdTime": 1627268171906, + "tenantId": { + "entityType": "TENANT", + "id": "3db30ac6-db03-4788-98fe-6e024b422a15" + }, + "name": "LORAWAN 001", + "description": "Tektelic - 001", + "type": "DEFAULT", + "transportType": "DEFAULT", + "provisionType": "DISABLED", + "defaultRuleChainId": { + "entityType": "RULE_CHAIN", + "id": "9c50f4df-f41e-443f-bb7d-37b5ac97f3c3" + }, + "defaultQueueName": "LORAWAN", + "profileData": { + "configuration": { + "type": "DEFAULT" + }, + "transportConfiguration": { + "type": "DEFAULT" + }, + "provisionConfiguration": { + "type": "DISABLED", + "provisionDeviceSecret": null + }, + "alarms": [ + { + "id": "b86271fd-5fee-4bd5-975c-d9c18f610cd5", + "alarmType": "LORAWAN - Battery Alarm", + "createRules": { + "CRITICAL": { + "condition": { + "condition": [ + { + "key": { + "type": "TIME_SERIES", + "key": "batteryLevel" + }, + "valueType": "NUMERIC", + "value": null, + "predicate": { + "type": "NUMERIC", + "operation": "LESS", + "value": { + "defaultValue": 25.0, + "userValue": null, + "dynamicValue": null + } + } + } + ], + "spec": { + "type": "DURATION", + "unit": "DAYS", + "predicate": { + "defaultValue": 1, + "userValue": null, + "dynamicValue": { + "sourceType": null, + "sourceAttribute": null, + "inherit": false + } + } + } + }, + "schedule": null, + "alarmDetails": null + } + }, + "clearRule": { + "condition": { + "condition": [ + { + "key": { + "type": "TIME_SERIES", + "key": "batteryLevel" + }, + "valueType": "NUMERIC", + "value": null, + "predicate": { + "type": "NUMERIC", + "operation": "GREATER_OR_EQUAL", + "value": { + "defaultValue": 25.0, + "userValue": null, + "dynamicValue": null + } + } + } + ], + "spec": { + "type": "DURATION", + "unit": "DAYS", + "predicate": { + "defaultValue": 1, + "userValue": null, + "dynamicValue": { + "sourceType": null, + "sourceAttribute": null, + "inherit": false + } + } + } + }, + "schedule": null, + "alarmDetails": null + }, + "propagate": true, + "propagateRelationTypes": [ + "UC-0007 LORAWAN" + ] + }, + { + "id": "c70aef4e-65cf-4578-acd9-e1927c08b469", + "alarmType": "LORAWAN - No Data", + "createRules": { + "CRITICAL": { + "condition": { + "condition": [ + { + "key": { + "type": "TIME_SERIES", + "key": "active" + }, + "valueType": "BOOLEAN", + "value": null, + "predicate": { + "type": "BOOLEAN", + "operation": "EQUAL", + "value": { + "defaultValue": false, + "userValue": null, + "dynamicValue": null + } + } + } + ], + "spec": { + "type": "SIMPLE" + } + }, + "schedule": null, + "alarmDetails": null + } + }, + "clearRule": { + "condition": { + "condition": [ + { + "key": { + "type": "TIME_SERIES", + "key": "active" + }, + "valueType": "BOOLEAN", + "value": null, + "predicate": { + "type": "BOOLEAN", + "operation": "EQUAL", + "value": { + "defaultValue": true, + "userValue": null, + "dynamicValue": null + } + } + } + ], + "spec": { + "type": "SIMPLE" + } + }, + "schedule": null, + "alarmDetails": null + }, + "propagate": true, + "propagateRelationTypes": [ + "LORAWAN 001 related" + ] + } + ] + }, + "provisionDeviceKey": null, + "default": false +} From 337f202812d6d0ad1e3bbbc525080ff3b748382f Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Mon, 8 Nov 2021 14:59:38 +0200 Subject: [PATCH 5/5] version upgrade: log detail added "alarm processed so far..." --- .../server/service/install/update/DefaultDataUpdateService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java index ebdf60e7b7..de483dfda8 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java @@ -420,7 +420,7 @@ public class DefaultDataUpdateService implements DataUpdateService { alarmDao.save(tenantId, alarm); } if (processed.incrementAndGet() % 1000 == 0) { - log.info("{}: {} processed so far...", name, processed); + log.info("{}: {} alarms processed so far...", name, processed); } } if (alarms.hasNext()) {